From fdf2ce0a5d77b3cec5f71f1457f7ae529dea4fee Mon Sep 17 00:00:00 2001 From: MiaoDX Date: Mon, 22 Jun 2026 15:08:38 +0800 Subject: [PATCH] ci: limit provider health to public agents sdk routes Co-authored-by: Codex --- .github/workflows/ci.yml | 7 +- scripts/dev/check_model_providers.py | 80 ++++++++++++------- .../providers/test_check_model_providers.py | 80 ++++++++++++------- 3 files changed, 102 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc8616aff..5f54c67a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,18 +179,13 @@ jobs: - name: Check configured provider routes env: - CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }} - CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} - XM_LLM_API_KEY: ${{ secrets.XM_LLM_API_KEY }} - XM_LLM_BASE_URL: ${{ secrets.XM_LLM_BASE_URL }} MM_BASE_URL: ${{ secrets.MM_BASE_URL }} MM_API_KEY: ${{ secrets.MM_API_KEY }} MIMO_TP_KEY: ${{ secrets.MIMO_TP_KEY }} KIMI_API_KEY: ${{ secrets.KIMI_API_KEY }} - NV_API_KEY: ${{ secrets.NV_API_KEY }} run: | .venv/bin/python scripts/dev/check_model_providers.py \ - --mode all \ + --profile agents-sdk-public \ --dotenv "" \ --require-all diff --git a/scripts/dev/check_model_providers.py b/scripts/dev/check_model_providers.py index a92f1cced..47c4ee339 100644 --- a/scripts/dev/check_model_providers.py +++ b/scripts/dev/check_model_providers.py @@ -32,7 +32,13 @@ DEFAULT_CHAT_MAX_TOKENS = 512 DEFAULT_TIMEOUT_S = 30.0 -ProbeMode = Literal["agents-sdk", "direct"] +ProbeMode = Literal["agents-sdk", "provider"] + +PUBLIC_AGENT_SDK_ROUTE_IDS = ( + "minimax-responses", + "mimo-tp-openai-chat", + "kimi-openai-chat", +) @dataclass(frozen=True) @@ -108,7 +114,7 @@ def build_agent_sdk_probes( return probes -def build_direct_probes( +def build_provider_probes( *, responses_max_tokens: int = DEFAULT_RESPONSES_MAX_OUTPUT_TOKENS, chat_max_tokens: int = DEFAULT_CHAT_MAX_TOKENS, @@ -120,12 +126,14 @@ def build_direct_probes( mimo_inside = provider_route_spec("mimo-inside-openai-chat") return [ - _direct_from_route("codex-router-responses", codex, max_tokens=responses_max_tokens), - _direct_from_route("mimo-mify-responses", mify_route, max_tokens=responses_max_tokens), - _direct_from_route("minimax-responses-m3", minimax_route, max_tokens=responses_max_tokens), + _provider_from_route("codex-router-responses", codex, max_tokens=responses_max_tokens), + _provider_from_route("mimo-mify-responses", mify_route, max_tokens=responses_max_tokens), + _provider_from_route( + "minimax-responses-m3", minimax_route, max_tokens=responses_max_tokens + ), ProbeSpec( - probe_id="direct:minimax-responses-m27", - mode="direct", + probe_id="provider:minimax-responses-m27", + mode="provider", route_id="minimax-responses", wire_api=WIRE_RESPONSES, model="MiniMax-M2.7-highspeed", @@ -134,11 +142,11 @@ def build_direct_probes( max_tokens=responses_max_tokens, provider_note="MiniMax highspeed can spend early tokens on reasoning.", ), - _direct_from_route("mimo-tp-openai-chat", mimo, max_tokens=chat_max_tokens), - _direct_from_route("mimo-inside-openai-chat", mimo_inside, max_tokens=chat_max_tokens), + _provider_from_route("mimo-tp-openai-chat", mimo, max_tokens=chat_max_tokens), + _provider_from_route("mimo-inside-openai-chat", mimo_inside, max_tokens=chat_max_tokens), ProbeSpec( - probe_id="direct:kimi-coding-chat", - mode="direct", + probe_id="provider:kimi-coding-chat", + mode="provider", route_id="kimi-openai-chat", wire_api=WIRE_CHAT_COMPLETIONS, model="kimi-k2.7-code", @@ -154,8 +162,8 @@ def build_direct_probes( ), ), ProbeSpec( - probe_id="direct:nvidia-chat", - mode="direct", + probe_id="provider:nvidia-chat", + mode="provider", wire_api=WIRE_CHAT_COMPLETIONS, model="nvidia/llama-3.1-nemotron-nano-vl-8b-v1", api_key_env="NV_API_KEY", @@ -166,10 +174,10 @@ def build_direct_probes( ] -def _direct_from_route(name: str, route: Any, *, max_tokens: int) -> ProbeSpec: +def _provider_from_route(name: str, route: Any, *, max_tokens: int) -> ProbeSpec: return ProbeSpec( - probe_id=f"direct:{name}", - mode="direct", + probe_id=f"provider:{name}", + mode="provider", route_id=route.route_id, wire_api=route.wire_api, model=route.default_model_id, @@ -213,7 +221,7 @@ def run_probe( output = _run_agents_sdk_probe( spec, prompt=prompt, timeout_s=timeout_s, api_key=api_key ) - elif spec.probe_id == "direct:kimi-coding-chat": + elif spec.probe_id == "provider:kimi-coding-chat": output = _run_kimi_coding_probe( spec, prompt=prompt, timeout_s=timeout_s, api_key=api_key ) @@ -432,20 +440,23 @@ def kimi_coding_payload(*, prompt: str, model: str, max_tokens: int) -> dict[str def select_probes(args: argparse.Namespace) -> list[ProbeSpec]: probes: list[ProbeSpec] = [] - if args.mode in {"agents-sdk", "all"}: + agent_sdk_probes = build_agent_sdk_probes( + responses_max_tokens=args.responses_max_output_tokens, + chat_max_tokens=args.chat_max_tokens, + ) + provider_probes = build_provider_probes( + responses_max_tokens=args.responses_max_output_tokens, + chat_max_tokens=args.chat_max_tokens, + ) + if args.profile == "agents-sdk-public": probes.extend( - build_agent_sdk_probes( - responses_max_tokens=args.responses_max_output_tokens, - chat_max_tokens=args.chat_max_tokens, - ) - ) - if args.mode in {"direct", "all"}: - probes.extend( - build_direct_probes( - responses_max_tokens=args.responses_max_output_tokens, - chat_max_tokens=args.chat_max_tokens, - ) + probe for probe in agent_sdk_probes if probe.route_id in PUBLIC_AGENT_SDK_ROUTE_IDS ) + else: + if args.mode in {"agents-sdk", "all"}: + probes.extend(agent_sdk_probes) + if args.mode in {"provider", "all"}: + probes.extend(provider_probes) if args.probe: wanted = set(args.probe) probes = [probe for probe in probes if probe.probe_id in wanted or probe.route_id in wanted] @@ -494,10 +505,19 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--mode", - choices=("all", "agents-sdk", "direct"), + choices=("all", "agents-sdk", "provider"), default="all", help="Which probe family to run.", ) + parser.add_argument( + "--profile", + choices=("all", "agents-sdk-public"), + default="all", + help=( + "Probe profile to run. agents-sdk-public includes only public-network " + "OpenAI Agents SDK routes suitable for public CI." + ), + ) parser.add_argument( "--probe", action="append", diff --git a/tests/unit/providers/test_check_model_providers.py b/tests/unit/providers/test_check_model_providers.py index deda930fb..0181459eb 100644 --- a/tests/unit/providers/test_check_model_providers.py +++ b/tests/unit/providers/test_check_model_providers.py @@ -24,16 +24,16 @@ def test_provider_health_defaults_leave_room_for_reasoning_tokens() -> None: assert script.DEFAULT_CHAT_MAX_TOKENS >= 128 -def test_direct_probe_defaults_cover_minimax_highspeed_and_kimi_payload() -> None: +def test_provider_probe_defaults_cover_minimax_highspeed_and_kimi_payload() -> None: script = _load_script_module() - probes = {probe.probe_id: probe for probe in script.build_direct_probes()} + probes = {probe.probe_id: probe for probe in script.build_provider_probes()} - assert probes["direct:minimax-responses-m27"].max_tokens >= 256 - assert probes["direct:mimo-tp-openai-chat"].max_tokens >= 128 - assert probes["direct:mimo-inside-openai-chat"].model == "mimo-1000" - assert probes["direct:mimo-inside-openai-chat"].max_tokens >= 128 - kimi = probes["direct:kimi-coding-chat"] + assert probes["provider:minimax-responses-m27"].max_tokens >= 256 + assert probes["provider:mimo-tp-openai-chat"].max_tokens >= 128 + assert probes["provider:mimo-inside-openai-chat"].model == "mimo-1000" + assert probes["provider:mimo-inside-openai-chat"].max_tokens >= 128 + kimi = probes["provider:kimi-coding-chat"] payload = script.kimi_coding_payload( prompt="Health check. Reply exactly ok.", model=kimi.model, @@ -45,12 +45,12 @@ def test_direct_probe_defaults_cover_minimax_highspeed_and_kimi_payload() -> Non assert "temperature" not in payload -def test_direct_probe_defaults_exclude_unavailable_official_openai_route() -> None: +def test_provider_probe_defaults_exclude_unavailable_official_openai_route() -> None: script = _load_script_module() - probes = {probe.probe_id: probe for probe in script.build_direct_probes()} + probes = {probe.probe_id: probe for probe in script.build_provider_probes()} - assert "direct:openai-responses" not in probes + assert "provider:openai-responses" not in probes assert all(probe.api_key_env != "OPENAI_API_KEY" for probe in probes.values()) @@ -73,7 +73,7 @@ def test_require_all_fails_on_skipped_probe(monkeypatch) -> None: ) ], ) - monkeypatch.setattr(script, "build_direct_probes", lambda **_kwargs: []) + monkeypatch.setattr(script, "build_provider_probes", lambda **_kwargs: []) assert script.main(["--mode", "agents-sdk", "--dotenv", "", "--require-all"]) == 1 @@ -107,7 +107,7 @@ def test_agents_sdk_probe_defaults_use_larger_responses_budget() -> None: assert not probes["agents-sdk:kimi-openai-chat"].unsupported_reason -def test_direct_chat_probe_sends_thinking_through_extra_body(monkeypatch) -> None: +def test_provider_chat_probe_sends_thinking_through_extra_body(monkeypatch) -> None: script = _load_script_module() captured: dict[str, object] = {} monkeypatch.setenv("MIMO_TP_KEY", "fake-mimo-key") @@ -129,8 +129,8 @@ def __init__(self, **kwargs) -> None: self.chat = FakeChat() monkeypatch.setitem(sys.modules, "openai", type("FakeOpenAIModule", (), {"OpenAI": FakeOpenAI})) - probe = {probe.probe_id: probe for probe in script.build_direct_probes()}[ - "direct:mimo-tp-openai-chat" + probe = {probe.probe_id: probe for probe in script.build_provider_probes()}[ + "provider:mimo-tp-openai-chat" ] result = script.run_probe(probe, prompt="Health check. Reply exactly ok.", timeout_s=1.0) @@ -239,11 +239,11 @@ def post(*_args, **kwargs): return captured -def test_kimi_direct_probe_validates_provider_response_source(monkeypatch) -> None: +def test_kimi_provider_probe_validates_provider_response_source(monkeypatch) -> None: script = _load_script_module() monkeypatch.setenv("KIMI_API_KEY", "fake-kimi-key") - probe = {probe.probe_id: probe for probe in script.build_direct_probes()}[ - "direct:kimi-coding-chat" + probe = {probe.probe_id: probe for probe in script.build_provider_probes()}[ + "provider:kimi-coding-chat" ] _install_fake_httpx(monkeypatch, response_text='["not-an-object"]') @@ -252,14 +252,14 @@ def test_kimi_direct_probe_validates_provider_response_source(monkeypatch) -> No assert result.status == "FAIL" assert result.error_type == "ValueError" assert "Kimi coding provider response source must contain a JSON object" in result.error - assert "direct:kimi-coding-chat" in result.error + assert "provider:kimi-coding-chat" in result.error -def test_kimi_direct_probe_validates_response_message_shape(monkeypatch) -> None: +def test_kimi_provider_probe_validates_response_message_shape(monkeypatch) -> None: script = _load_script_module() monkeypatch.setenv("KIMI_API_KEY", "fake-kimi-key") - probe = {probe.probe_id: probe for probe in script.build_direct_probes()}[ - "direct:kimi-coding-chat" + probe = {probe.probe_id: probe for probe in script.build_provider_probes()}[ + "provider:kimi-coding-chat" ] _install_fake_httpx(monkeypatch, response_text=json.dumps({"choices": [{"message": []}]})) @@ -268,14 +268,14 @@ def test_kimi_direct_probe_validates_response_message_shape(monkeypatch) -> None assert result.status == "FAIL" assert result.error_type == "RuntimeError" assert "choices[0].message must be a JSON object" in result.error - assert "direct:kimi-coding-chat" in result.error + assert "provider:kimi-coding-chat" in result.error -def test_kimi_direct_probe_reads_reasoning_content_from_valid_response(monkeypatch) -> None: +def test_kimi_provider_probe_reads_reasoning_content_from_valid_response(monkeypatch) -> None: script = _load_script_module() monkeypatch.setenv("KIMI_API_KEY", "fake-kimi-key") - probe = {probe.probe_id: probe for probe in script.build_direct_probes()}[ - "direct:kimi-coding-chat" + probe = {probe.probe_id: probe for probe in script.build_provider_probes()}[ + "provider:kimi-coding-chat" ] captured = _install_fake_httpx( monkeypatch, @@ -297,12 +297,34 @@ def test_select_probe_can_limit_by_route_or_probe_id() -> None: assert {probe.probe_id for probe in selected} == { "agents-sdk:minimax-responses", - "direct:minimax-responses-m3", - "direct:minimax-responses-m27", + "provider:minimax-responses-m3", + "provider:minimax-responses-m27", } -def test_select_probe_can_limit_kimi_route_across_sdk_and_direct_probes() -> None: +def test_agents_sdk_public_profile_excludes_internal_routes() -> None: + script = _load_script_module() + args = script.parse_args(["--profile", "agents-sdk-public"]) + + selected = script.select_probes(args) + + assert {probe.probe_id for probe in selected} == { + "agents-sdk:minimax-responses", + "agents-sdk:mimo-tp-openai-chat", + "agents-sdk:kimi-openai-chat", + } + assert not any( + probe.route_id + in { + "codex-router-responses", + "mimo-mify-responses", + "mimo-inside-openai-chat", + } + for probe in selected + ) + + +def test_select_probe_can_limit_kimi_route_across_sdk_and_provider_probes() -> None: script = _load_script_module() args = script.parse_args(["--mode", "all", "--probe", "kimi-openai-chat"]) @@ -310,5 +332,5 @@ def test_select_probe_can_limit_kimi_route_across_sdk_and_direct_probes() -> Non assert {probe.probe_id for probe in selected} == { "agents-sdk:kimi-openai-chat", - "direct:kimi-coding-chat", + "provider:kimi-coding-chat", }