ci: limit provider health to public agents sdk routes

.github/workflows/ci.yml

@@ -179,18 +179,13 @@ jobs:
 
       - name: Check configured provider routes
         env:
-          CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }}
-          CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }}
-          XM_LLM_API_KEY: ${{ secrets.XM_LLM_API_KEY }}
-          XM_LLM_BASE_URL: ${{ secrets.XM_LLM_BASE_URL }}
           MM_BASE_URL: ${{ secrets.MM_BASE_URL }}
           MM_API_KEY: ${{ secrets.MM_API_KEY }}
           MIMO_TP_KEY: ${{ secrets.MIMO_TP_KEY }}
           KIMI_API_KEY: ${{ secrets.KIMI_API_KEY }}
-          NV_API_KEY: ${{ secrets.NV_API_KEY }}
         run: |
           .venv/bin/python scripts/dev/check_model_providers.py \
-            --mode all \
+            --profile agents-sdk-public \
             --dotenv "" \
             --require-all
 

scripts/dev/check_model_providers.py

@@ -32,7 +32,13 @@
 DEFAULT_CHAT_MAX_TOKENS = 512
 DEFAULT_TIMEOUT_S = 30.0
 
-ProbeMode = Literal["agents-sdk", "direct"]
+ProbeMode = Literal["agents-sdk", "provider"]
+
+PUBLIC_AGENT_SDK_ROUTE_IDS = (
+    "minimax-responses",
+    "mimo-tp-openai-chat",
+    "kimi-openai-chat",
+)
 
 
 @dataclass(frozen=True)
@@ -108,7 +114,7 @@ def build_agent_sdk_probes(
     return probes
 
 
-def build_direct_probes(
+def build_provider_probes(
     *,
     responses_max_tokens: int = DEFAULT_RESPONSES_MAX_OUTPUT_TOKENS,
     chat_max_tokens: int = DEFAULT_CHAT_MAX_TOKENS,
@@ -120,12 +126,14 @@ def build_direct_probes(
     mimo_inside = provider_route_spec("mimo-inside-openai-chat")
 
     return [
-        _direct_from_route("codex-router-responses", codex, max_tokens=responses_max_tokens),
-        _direct_from_route("mimo-mify-responses", mify_route, max_tokens=responses_max_tokens),
-        _direct_from_route("minimax-responses-m3", minimax_route, max_tokens=responses_max_tokens),
+        _provider_from_route("codex-router-responses", codex, max_tokens=responses_max_tokens),
+        _provider_from_route("mimo-mify-responses", mify_route, max_tokens=responses_max_tokens),
+        _provider_from_route(
+            "minimax-responses-m3", minimax_route, max_tokens=responses_max_tokens
+        ),
         ProbeSpec(
-            probe_id="direct:minimax-responses-m27",
-            mode="direct",
+            probe_id="provider:minimax-responses-m27",
+            mode="provider",
             route_id="minimax-responses",
             wire_api=WIRE_RESPONSES,
             model="MiniMax-M2.7-highspeed",
@@ -134,11 +142,11 @@ def build_direct_probes(
             max_tokens=responses_max_tokens,
             provider_note="MiniMax highspeed can spend early tokens on reasoning.",
         ),
-        _direct_from_route("mimo-tp-openai-chat", mimo, max_tokens=chat_max_tokens),
-        _direct_from_route("mimo-inside-openai-chat", mimo_inside, max_tokens=chat_max_tokens),
+        _provider_from_route("mimo-tp-openai-chat", mimo, max_tokens=chat_max_tokens),
+        _provider_from_route("mimo-inside-openai-chat", mimo_inside, max_tokens=chat_max_tokens),
         ProbeSpec(
-            probe_id="direct:kimi-coding-chat",
-            mode="direct",
+            probe_id="provider:kimi-coding-chat",
+            mode="provider",
             route_id="kimi-openai-chat",
             wire_api=WIRE_CHAT_COMPLETIONS,
             model="kimi-k2.7-code",
@@ -154,8 +162,8 @@ def build_direct_probes(
             ),
         ),
         ProbeSpec(
-            probe_id="direct:nvidia-chat",
-            mode="direct",
+            probe_id="provider:nvidia-chat",
+            mode="provider",
             wire_api=WIRE_CHAT_COMPLETIONS,
             model="nvidia/llama-3.1-nemotron-nano-vl-8b-v1",
             api_key_env="NV_API_KEY",
@@ -166,10 +174,10 @@ def build_direct_probes(
     ]
 
 
-def _direct_from_route(name: str, route: Any, *, max_tokens: int) -> ProbeSpec:
+def _provider_from_route(name: str, route: Any, *, max_tokens: int) -> ProbeSpec:
     return ProbeSpec(
-        probe_id=f"direct:{name}",
-        mode="direct",
+        probe_id=f"provider:{name}",
+        mode="provider",
         route_id=route.route_id,
         wire_api=route.wire_api,
         model=route.default_model_id,
@@ -213,7 +221,7 @@ def run_probe(
             output = _run_agents_sdk_probe(
                 spec, prompt=prompt, timeout_s=timeout_s, api_key=api_key
             )
-        elif spec.probe_id == "direct:kimi-coding-chat":
+        elif spec.probe_id == "provider:kimi-coding-chat":
             output = _run_kimi_coding_probe(
                 spec, prompt=prompt, timeout_s=timeout_s, api_key=api_key
             )
@@ -432,20 +440,23 @@ def kimi_coding_payload(*, prompt: str, model: str, max_tokens: int) -> dict[str
 
 def select_probes(args: argparse.Namespace) -> list[ProbeSpec]:
     probes: list[ProbeSpec] = []
-    if args.mode in {"agents-sdk", "all"}:
+    agent_sdk_probes = build_agent_sdk_probes(
+        responses_max_tokens=args.responses_max_output_tokens,
+        chat_max_tokens=args.chat_max_tokens,
+    )
+    provider_probes = build_provider_probes(
+        responses_max_tokens=args.responses_max_output_tokens,
+        chat_max_tokens=args.chat_max_tokens,
+    )
+    if args.profile == "agents-sdk-public":
         probes.extend(
-            build_agent_sdk_probes(
-                responses_max_tokens=args.responses_max_output_tokens,
-                chat_max_tokens=args.chat_max_tokens,
-            )
-        )
-    if args.mode in {"direct", "all"}:
-        probes.extend(
-            build_direct_probes(
-                responses_max_tokens=args.responses_max_output_tokens,
-                chat_max_tokens=args.chat_max_tokens,
-            )
+            probe for probe in agent_sdk_probes if probe.route_id in PUBLIC_AGENT_SDK_ROUTE_IDS
         )
+    else:
+        if args.mode in {"agents-sdk", "all"}:
+            probes.extend(agent_sdk_probes)
+        if args.mode in {"provider", "all"}:
+            probes.extend(provider_probes)
     if args.probe:
         wanted = set(args.probe)
         probes = [probe for probe in probes if probe.probe_id in wanted or probe.route_id in wanted]
@@ -494,10 +505,19 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
         "--mode",
-        choices=("all", "agents-sdk", "direct"),
+        choices=("all", "agents-sdk", "provider"),
         default="all",
         help="Which probe family to run.",
     )
+    parser.add_argument(
+        "--profile",
+        choices=("all", "agents-sdk-public"),
+        default="all",
+        help=(
+            "Probe profile to run. agents-sdk-public includes only public-network "
+            "OpenAI Agents SDK routes suitable for public CI."
+        ),
+    )
     parser.add_argument(
         "--probe",
         action="append",

tests/unit/providers/test_check_model_providers.py

@@ -24,16 +24,16 @@ def test_provider_health_defaults_leave_room_for_reasoning_tokens() -> None:
     assert script.DEFAULT_CHAT_MAX_TOKENS >= 128
 
 
-def test_direct_probe_defaults_cover_minimax_highspeed_and_kimi_payload() -> None:
+def test_provider_probe_defaults_cover_minimax_highspeed_and_kimi_payload() -> None:
     script = _load_script_module()
 
-    probes = {probe.probe_id: probe for probe in script.build_direct_probes()}
+    probes = {probe.probe_id: probe for probe in script.build_provider_probes()}
 
-    assert probes["direct:minimax-responses-m27"].max_tokens >= 256
-    assert probes["direct:mimo-tp-openai-chat"].max_tokens >= 128
-    assert probes["direct:mimo-inside-openai-chat"].model == "mimo-1000"
-    assert probes["direct:mimo-inside-openai-chat"].max_tokens >= 128
-    kimi = probes["direct:kimi-coding-chat"]
+    assert probes["provider:minimax-responses-m27"].max_tokens >= 256
+    assert probes["provider:mimo-tp-openai-chat"].max_tokens >= 128
+    assert probes["provider:mimo-inside-openai-chat"].model == "mimo-1000"
+    assert probes["provider:mimo-inside-openai-chat"].max_tokens >= 128
+    kimi = probes["provider:kimi-coding-chat"]
     payload = script.kimi_coding_payload(
         prompt="Health check. Reply exactly ok.",
         model=kimi.model,
@@ -45,12 +45,12 @@ def test_direct_probe_defaults_cover_minimax_highspeed_and_kimi_payload() -> Non
     assert "temperature" not in payload
 
 
-def test_direct_probe_defaults_exclude_unavailable_official_openai_route() -> None:
+def test_provider_probe_defaults_exclude_unavailable_official_openai_route() -> None:
     script = _load_script_module()
 
-    probes = {probe.probe_id: probe for probe in script.build_direct_probes()}
+    probes = {probe.probe_id: probe for probe in script.build_provider_probes()}
 
-    assert "direct:openai-responses" not in probes
+    assert "provider:openai-responses" not in probes
     assert all(probe.api_key_env != "OPENAI_API_KEY" for probe in probes.values())
 
 
@@ -73,7 +73,7 @@ def test_require_all_fails_on_skipped_probe(monkeypatch) -> None:
             )
         ],
     )
-    monkeypatch.setattr(script, "build_direct_probes", lambda **_kwargs: [])
+    monkeypatch.setattr(script, "build_provider_probes", lambda **_kwargs: [])
 
     assert script.main(["--mode", "agents-sdk", "--dotenv", "", "--require-all"]) == 1
 
@@ -107,7 +107,7 @@ def test_agents_sdk_probe_defaults_use_larger_responses_budget() -> None:
     assert not probes["agents-sdk:kimi-openai-chat"].unsupported_reason
 
 
-def test_direct_chat_probe_sends_thinking_through_extra_body(monkeypatch) -> None:
+def test_provider_chat_probe_sends_thinking_through_extra_body(monkeypatch) -> None:
     script = _load_script_module()
     captured: dict[str, object] = {}
     monkeypatch.setenv("MIMO_TP_KEY", "fake-mimo-key")
@@ -129,8 +129,8 @@ def __init__(self, **kwargs) -> None:
             self.chat = FakeChat()
 
     monkeypatch.setitem(sys.modules, "openai", type("FakeOpenAIModule", (), {"OpenAI": FakeOpenAI}))
-    probe = {probe.probe_id: probe for probe in script.build_direct_probes()}[
-        "direct:mimo-tp-openai-chat"
+    probe = {probe.probe_id: probe for probe in script.build_provider_probes()}[
+        "provider:mimo-tp-openai-chat"
     ]
 
     result = script.run_probe(probe, prompt="Health check. Reply exactly ok.", timeout_s=1.0)
@@ -239,11 +239,11 @@ def post(*_args, **kwargs):
     return captured
 
 
-def test_kimi_direct_probe_validates_provider_response_source(monkeypatch) -> None:
+def test_kimi_provider_probe_validates_provider_response_source(monkeypatch) -> None:
     script = _load_script_module()
     monkeypatch.setenv("KIMI_API_KEY", "fake-kimi-key")
-    probe = {probe.probe_id: probe for probe in script.build_direct_probes()}[
-        "direct:kimi-coding-chat"
+    probe = {probe.probe_id: probe for probe in script.build_provider_probes()}[
+        "provider:kimi-coding-chat"
     ]
     _install_fake_httpx(monkeypatch, response_text='["not-an-object"]')
 
@@ -252,14 +252,14 @@ def test_kimi_direct_probe_validates_provider_response_source(monkeypatch) -> No
     assert result.status == "FAIL"
     assert result.error_type == "ValueError"
     assert "Kimi coding provider response source must contain a JSON object" in result.error
-    assert "direct:kimi-coding-chat" in result.error
+    assert "provider:kimi-coding-chat" in result.error
 
 
-def test_kimi_direct_probe_validates_response_message_shape(monkeypatch) -> None:
+def test_kimi_provider_probe_validates_response_message_shape(monkeypatch) -> None:
     script = _load_script_module()
     monkeypatch.setenv("KIMI_API_KEY", "fake-kimi-key")
-    probe = {probe.probe_id: probe for probe in script.build_direct_probes()}[
-        "direct:kimi-coding-chat"
+    probe = {probe.probe_id: probe for probe in script.build_provider_probes()}[
+        "provider:kimi-coding-chat"
     ]
     _install_fake_httpx(monkeypatch, response_text=json.dumps({"choices": [{"message": []}]}))
 
@@ -268,14 +268,14 @@ def test_kimi_direct_probe_validates_response_message_shape(monkeypatch) -> None
     assert result.status == "FAIL"
     assert result.error_type == "RuntimeError"
     assert "choices[0].message must be a JSON object" in result.error
-    assert "direct:kimi-coding-chat" in result.error
+    assert "provider:kimi-coding-chat" in result.error
 
 
-def test_kimi_direct_probe_reads_reasoning_content_from_valid_response(monkeypatch) -> None:
+def test_kimi_provider_probe_reads_reasoning_content_from_valid_response(monkeypatch) -> None:
     script = _load_script_module()
     monkeypatch.setenv("KIMI_API_KEY", "fake-kimi-key")
-    probe = {probe.probe_id: probe for probe in script.build_direct_probes()}[
-        "direct:kimi-coding-chat"
+    probe = {probe.probe_id: probe for probe in script.build_provider_probes()}[
+        "provider:kimi-coding-chat"
     ]
     captured = _install_fake_httpx(
         monkeypatch,
@@ -297,18 +297,40 @@ def test_select_probe_can_limit_by_route_or_probe_id() -> None:
 
     assert {probe.probe_id for probe in selected} == {
         "agents-sdk:minimax-responses",
-        "direct:minimax-responses-m3",
-        "direct:minimax-responses-m27",
+        "provider:minimax-responses-m3",
+        "provider:minimax-responses-m27",
     }
 
 
-def test_select_probe_can_limit_kimi_route_across_sdk_and_direct_probes() -> None:
+def test_agents_sdk_public_profile_excludes_internal_routes() -> None:
+    script = _load_script_module()
+    args = script.parse_args(["--profile", "agents-sdk-public"])
+
+    selected = script.select_probes(args)
+
+    assert {probe.probe_id for probe in selected} == {
+        "agents-sdk:minimax-responses",
+        "agents-sdk:mimo-tp-openai-chat",
+        "agents-sdk:kimi-openai-chat",
+    }
+    assert not any(
+        probe.route_id
+        in {
+            "codex-router-responses",
+            "mimo-mify-responses",
+            "mimo-inside-openai-chat",
+        }
+        for probe in selected
+    )
+
+
+def test_select_probe_can_limit_kimi_route_across_sdk_and_provider_probes() -> None:
     script = _load_script_module()
     args = script.parse_args(["--mode", "all", "--probe", "kimi-openai-chat"])
 
     selected = script.select_probes(args)
 
     assert {probe.probe_id for probe in selected} == {
         "agents-sdk:kimi-openai-chat",
-        "direct:kimi-coding-chat",
+        "provider:kimi-coding-chat",
     }