From cb99e1b21ab2213b622fc91a13dbbd056c5cf99e Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Mon, 6 Jul 2026 22:24:02 +0200 Subject: [PATCH 1/2] fix docs for sglang and vllm --- doc/workloads/sglang.rst | 18 +++++++++++++----- doc/workloads/vllm.rst | 18 +++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/doc/workloads/sglang.rst b/doc/workloads/sglang.rst index 9250e65b0..85f35d34c 100644 --- a/doc/workloads/sglang.rst +++ b/doc/workloads/sglang.rst @@ -65,11 +65,9 @@ Test-in-Scenario example docker_image_url = "lmsysorg/sglang:dev-cu13" model = "Qwen/Qwen3-8B" - [Tests.bench_cmd_args] - random_input = 16 - random_output = 128 - max_concurrency = 16 - num_prompts = 30 +Workload-specific test definition sections, such as ``bench_cmd_args`` and ``semantic_eval_cmd_args``, cannot be +defined under ``[[Tests]]`` in a test scenario. Define them in a test definition TOML and reference that test with +``test_name`` when custom benchmark or semantic-evaluation arguments are needed. Semantic Validation @@ -96,6 +94,16 @@ The ``cli`` string supports ``{model}``, ``{host}``, ``{port}``, ``{url}``, ``{o placeholders. +Reporting +--------- +After a run completes, CloudAI parses ``sglang-bench.jsonl`` and reports serving latency, successful prompt count, +completion rate, throughput, TPS per user, and TPS per GPU. If ``semantic_eval_cmd_args`` is configured, CloudAI also +reports the semantic validation ``accuracy`` metric. + +CloudAI also provides the scenario-level ``sglang_comparison`` report. It compares SGLang test runs in the scenario and +uses ``bench_cmd_args`` values as comparison labels. + + Readiness health checks ----------------------- Healthcheck fields: diff --git a/doc/workloads/vllm.rst b/doc/workloads/vllm.rst index a57486773..dfaa9e5ea 100644 --- a/doc/workloads/vllm.rst +++ b/doc/workloads/vllm.rst @@ -65,11 +65,9 @@ Test-in-Scenario example docker_image_url = "nvcr.io#nvidia/ai-dynamo/vllm-runtime:0.7.0" model = "Qwen/Qwen3-0.6B" - [Tests.bench_cmd_args] - random_input_len = 16 - random_output_len = 128 - max_concurrency = 16 - num_prompts = 30 +Workload-specific test definition sections, such as ``bench_cmd_args`` and ``semantic_eval_cmd_args``, cannot be +defined under ``[[Tests]]`` in a test scenario. Define them in a test definition TOML and reference that test with +``test_name`` when custom benchmark or semantic-evaluation arguments are needed. Semantic Validation @@ -91,6 +89,16 @@ The ``cli`` string supports ``{model}``, ``{host}``, ``{port}``, ``{url}``, ``{o placeholders. +Reporting +--------- +After a run completes, CloudAI parses ``vllm-bench.json`` and reports serving latency, successful prompt count, +completion rate, throughput, TPS per user, and TPS per GPU. If ``semantic_eval_cmd_args`` is configured, CloudAI also +reports the semantic validation ``accuracy`` metric. + +CloudAI also provides the scenario-level ``vllm_comparison`` report. It compares vLLM test runs in the scenario and +uses ``bench_cmd_args`` values as comparison labels. + + Controlling the Number of GPUs ------------------------------- GPU selection priority, from lowest to highest: From 579543e4c5caf01d4ecd97d088a9c5ab3edee370 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Mon, 6 Jul 2026 22:28:10 +0200 Subject: [PATCH 2/2] make wording consistent with the rest of the docs --- doc/workloads/sglang.rst | 15 +++++++++------ doc/workloads/vllm.rst | 11 +++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/doc/workloads/sglang.rst b/doc/workloads/sglang.rst index 85f35d34c..6d62f6ee9 100644 --- a/doc/workloads/sglang.rst +++ b/doc/workloads/sglang.rst @@ -8,8 +8,8 @@ SGLang is a high-throughput and memory-efficient inference engine for LLMs. This Usage Examples -------------- -Test + Scenario example -~~~~~~~~~~~~~~~~~~~~~~~ +Test and Scenario Examples +~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: toml :caption: test.toml (test definition) @@ -65,8 +65,8 @@ Test-in-Scenario example docker_image_url = "lmsysorg/sglang:dev-cu13" model = "Qwen/Qwen3-8B" -Workload-specific test definition sections, such as ``bench_cmd_args`` and ``semantic_eval_cmd_args``, cannot be -defined under ``[[Tests]]`` in a test scenario. Define them in a test definition TOML and reference that test with +Workload-specific test definition sections, such as ``bench_cmd_args`` and ``semantic_eval_cmd_args``, are not +supported under ``[[Tests]]`` in a test scenario. Define them in a test definition TOML and reference that test with ``test_name`` when custom benchmark or semantic-evaluation arguments are needed. @@ -96,9 +96,12 @@ placeholders. Reporting --------- -After a run completes, CloudAI parses ``sglang-bench.jsonl`` and reports serving latency, successful prompt count, +After a run completes, CloudAI parses ``sglang-bench.jsonl`` and prints serving latency, successful prompt count, completion rate, throughput, TPS per user, and TPS per GPU. If ``semantic_eval_cmd_args`` is configured, CloudAI also -reports the semantic validation ``accuracy`` metric. +reports semantic validation accuracy. + +The reported metric (``default``) is throughput. Additional supported metrics are ``throughput``, ``tps-per-user``, +``tps-per-gpu``, and ``accuracy``. CloudAI also provides the scenario-level ``sglang_comparison`` report. It compares SGLang test runs in the scenario and uses ``bench_cmd_args`` values as comparison labels. diff --git a/doc/workloads/vllm.rst b/doc/workloads/vllm.rst index dfaa9e5ea..009270b32 100644 --- a/doc/workloads/vllm.rst +++ b/doc/workloads/vllm.rst @@ -65,8 +65,8 @@ Test-in-Scenario example docker_image_url = "nvcr.io#nvidia/ai-dynamo/vllm-runtime:0.7.0" model = "Qwen/Qwen3-0.6B" -Workload-specific test definition sections, such as ``bench_cmd_args`` and ``semantic_eval_cmd_args``, cannot be -defined under ``[[Tests]]`` in a test scenario. Define them in a test definition TOML and reference that test with +Workload-specific test definition sections, such as ``bench_cmd_args`` and ``semantic_eval_cmd_args``, are not +supported under ``[[Tests]]`` in a test scenario. Define them in a test definition TOML and reference that test with ``test_name`` when custom benchmark or semantic-evaluation arguments are needed. @@ -91,9 +91,12 @@ placeholders. Reporting --------- -After a run completes, CloudAI parses ``vllm-bench.json`` and reports serving latency, successful prompt count, +After a run completes, CloudAI parses ``vllm-bench.json`` and prints serving latency, successful prompt count, completion rate, throughput, TPS per user, and TPS per GPU. If ``semantic_eval_cmd_args`` is configured, CloudAI also -reports the semantic validation ``accuracy`` metric. +reports semantic validation accuracy. + +The reported metric (``default``) is throughput. Additional supported metrics are ``throughput``, ``tps-per-user``, +``tps-per-gpu``, and ``accuracy``. CloudAI also provides the scenario-level ``vllm_comparison`` report. It compares vLLM test runs in the scenario and uses ``bench_cmd_args`` values as comparison labels.