From fcc07f5767d4dc837b88d49bc947fb28de51e8c5 Mon Sep 17 00:00:00 2001 From: shreyaskommuri Date: Mon, 29 Jun 2026 10:25:03 -0700 Subject: [PATCH] Make live CloudAI smoke evidence trustworthy Flush the recorded command before CloudAI inherits the log descriptor, and discover canonical summary artifacts inside CloudAI's timestamped scenario directory. Constraint: Preserve the existing run directory and report precedence contracts without adding dependencies. Rejected: Recursively scan every descendant | CloudAI's current contract adds one scenario-directory level and a broad scan could select workload-internal artifacts. Confidence: high Scope-risk: narrow Directive: Keep CloudAI artifact discovery aligned with the real output directory layout and deterministic report precedence. Tested: .venv/bin/python -m pytest tests/test_runner.py -q (8 passed); .venv/bin/python -m pytest -q (70 passed); compileall; git diff --check Not-tested: Machine-readable summary generation from PR NVIDIA/cloudai#919 because it remains unmerged. --- autotune/runner.py | 5 +++++ tests/test_runner.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/autotune/runner.py b/autotune/runner.py index bebe404..c1fe3ad 100644 --- a/autotune/runner.py +++ b/autotune/runner.py @@ -80,6 +80,7 @@ def run(self, config_path: Path | str, run_id: str) -> RunResult: failure_reason: Optional[str] = None with open(stdout_path, "w") as stdout_file: stdout_file.write(f"autotune: command: {' '.join(shlex.quote(part) for part in cmd)}\n") + stdout_file.flush() try: proc = subprocess.run( cmd, @@ -143,4 +144,8 @@ def _find_report(expected_path: Path) -> Optional[Path]: candidate = expected_path.parent / name if candidate.exists(): return candidate + for name in REPORT_CANDIDATES: + candidates = sorted(expected_path.parent.glob(f"*/{name}")) + if candidates: + return candidates[-1] return None diff --git a/tests/test_runner.py b/tests/test_runner.py index 1f68650..e9db439 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -30,6 +30,7 @@ def test_run_returns_failed_result_when_command_exits_nonzero(tmp_path): assert result.returncode == 7 assert result.stdout_path.exists() log = result.stdout_path.read_text() + assert log.splitlines()[0].startswith("autotune: command:") assert "backend exploded" in log assert "autotune: CloudAI exited with code 7" in log assert result.failure_reason == "CloudAI exited with code 7" @@ -115,6 +116,36 @@ def test_run_prefers_cloudai_summary_report(tmp_path): assert result.report_path == tmp_path / "0006_test" / "cloudai-summary.json" +def test_run_detects_cloudai_summary_in_scenario_output_directory(tmp_path): + script = tmp_path / "fake_cloudai.sh" + script.write_text( + "#!/bin/sh\n" + "out=''\n" + "while [ \"$#\" -gt 0 ]; do\n" + " if [ \"$1\" = '--output-dir' ]; then\n" + " shift\n" + " out=\"$1\"\n" + " fi\n" + " shift\n" + "done\n" + "mkdir -p \"$out/scenario_timestamp\"\n" + "printf '{\"metrics\":{\"throughput_tokens_per_sec\": 123}}' " + "> \"$out/scenario_timestamp/cloudai-summary.json\"\n" + ) + script.chmod(0o755) + runner = CloudAIRunner( + cloudai_bin=str(script), + runs_dir=tmp_path, + dry_run=True, + system_config=tmp_path / "system.toml", + ) + + result = runner.run(tmp_path / "config.toml", "0007_test") + + assert result.succeeded + assert result.report_path == tmp_path / "0007_test" / "scenario_timestamp" / "cloudai-summary.json" + + def test_command_string_uses_current_cloudai_cli_when_system_config_is_set(tmp_path): runner = CloudAIRunner( cloudai_bin="cloudai",