Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ black . && ruff check . # format + lint (CI enforced)

```bash
# OpenAI-compatible serving
python -m atom.entrypoints.openai_server --model <model> --kv_cache_dtype fp8 -tp 8
python -m atom.entrypoints.openai_server --model <model> --kv-cache-dtype fp8 -tp 8

# Offline inference
python -m atom.examples.simple_inference --model <model> --kv_cache_dtype fp8
python -m atom.examples.simple_inference --model <model> --kv-cache-dtype fp8
```

- Accuracy validation: see `/ci-pr-guide` for `lm_eval` setup and CI thresholds
Expand Down
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ hf auth login
The default optimization level is 3 (piecewise torch.compile with CUDA graphs).

```bash
python -m atom.examples.simple_inference --model meta-llama/Meta-Llama-3-8B --kv_cache_dtype fp8
python -m atom.examples.simple_inference --model meta-llama/Meta-Llama-3-8B --kv-cache-dtype fp8
```

> **Note:** First-time execution may take approximately 10 minutes for model compilation.
Expand All @@ -137,13 +137,13 @@ Start an OpenAI-compatible server:

```bash
# Single GPU
python -m atom.entrypoints.openai_server --model Qwen/Qwen3-0.6B --kv_cache_dtype fp8
python -m atom.entrypoints.openai_server --model Qwen/Qwen3-0.6B --kv-cache-dtype fp8

# Multi-GPU with tensor parallelism
python -m atom.entrypoints.openai_server --model deepseek-ai/DeepSeek-R1 --kv_cache_dtype fp8 -tp 8
python -m atom.entrypoints.openai_server --model deepseek-ai/DeepSeek-R1 --kv-cache-dtype fp8 -tp 8

# With MTP speculative decoding
python -m atom.entrypoints.openai_server --model deepseek-ai/DeepSeek-R1 --kv_cache_dtype fp8 -tp 8 \
python -m atom.entrypoints.openai_server --model deepseek-ai/DeepSeek-R1 --kv-cache-dtype fp8 -tp 8 \
--method mtp --num-speculative-tokens 3
```

Expand Down Expand Up @@ -191,7 +191,7 @@ Launch the server with `--torch-profiler-dir` and `--mark-trace`:

```bash
python -m atom.entrypoints.openai_server \
--model deepseek-ai/DeepSeek-R1 --kv_cache_dtype fp8 -tp 8 \
--model deepseek-ai/DeepSeek-R1 --kv-cache-dtype fp8 -tp 8 \
--torch-profiler-dir ./trace --mark-trace
```

Expand Down
2 changes: 2 additions & 0 deletions atom/model_engine/arg_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,9 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
help="Engine internal port",
)
parser.add_argument(
"--kv-cache-dtype",
"--kv_cache_dtype",
dest="kv_cache_dtype",
choices=["bf16", "fp8"],
type=str,
default="bf16",
Expand Down
20 changes: 20 additions & 0 deletions tests/test_arg_utils_spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,29 @@
{"NONE": 0, "PIECEWISE": 1, "FULL": 2, "FULL_AND_PIECEWISE": 3},
)

import argparse # noqa: E402

from atom.model_engine.arg_utils import EngineArgs # noqa: E402


class TestKVCacheDtypeCliAlias:
"""--kv-cache-dtype and --kv_cache_dtype must both set kv_cache_dtype."""

def _parse(self, argv):
parser = argparse.ArgumentParser()
EngineArgs.add_cli_args(parser)
return parser.parse_args(argv)

def test_dashed_form(self):
assert self._parse(["--kv-cache-dtype", "fp8"]).kv_cache_dtype == "fp8"

def test_underscore_form(self):
assert self._parse(["--kv_cache_dtype", "fp8"]).kv_cache_dtype == "fp8"

def test_default(self):
assert self._parse([]).kv_cache_dtype == "bf16"


class TestEngineArgsSpeculativeValidation:
"""Regression tests for speculative-config construction in _get_engine_kwargs."""

Expand Down