From 3463406a831e40d8ee1ba875d85ab64f45aaad3c Mon Sep 17 00:00:00 2001 From: Laura Date: Sun, 12 Jul 2026 00:24:40 -0400 Subject: [PATCH 1/2] Add minimum-compatibility tests for the TREC AutoJudge integration Adapted from the auto-judge-starter-kit test suite for this repo's layout: the workflow.yml must parse with a judge_class and described evaluation_measures; the judge class is verified against the source file (its judges.ir_axioms module path exists only inside the Docker container); and locally installed autojudge-base/tira are checked against the upstream template's minimum pins (skipped when the container base image provides them instead). Co-Authored-By: Claude Fable 5 --- tests/test_trec_auto_judge_workflow.py | 66 ++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/test_trec_auto_judge_workflow.py diff --git a/tests/test_trec_auto_judge_workflow.py b/tests/test_trec_auto_judge_workflow.py new file mode 100644 index 0000000..fcf1ce3 --- /dev/null +++ b/tests/test_trec_auto_judge_workflow.py @@ -0,0 +1,66 @@ +"""Minimum-compatibility checks for the TREC AutoJudge integration. + +Adapted from the auto-judge-starter-kit test suite for this repo's layout: +the judge lives in trec-auto-judge/ and its `judges.ir_axioms.*` module path +only exists inside the Docker container (see trec-auto-judge/Dockerfile), so +the class reference is checked against the source file rather than imported. +Framework packages (autojudge-base, tira) are provided by the container base +image; when they happen to be installed locally, their versions are checked +against the upstream template's minimum pins. +""" + +import re +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path + +import pytest + +yaml = pytest.importorskip("yaml", reason="pyyaml needed for workflow checks") + +REPO = Path(__file__).parent.parent +WORKFLOW = REPO / "trec-auto-judge" / "workflow.yml" + +# Mirrors the pins in the upstream template (trec-auto-judge/auto-judge-starter-kit +# pyproject.toml); bump when the template raises its requirements. +TEMPLATE_MINIMUMS = {"autojudge-base": "0.3.18", "tira": "0.0.100"} + + +def test_workflow_parses(): + """workflow.yml parses and declares a judge_class — what auto-judge run needs.""" + cfg = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + assert cfg.get("judge_class"), "workflow.yml declares no judge_class" + measures = cfg.get("settings", {}).get("evaluation_measures", {}) + assert measures, "workflow.yml declares no evaluation_measures" + undescribed = [name for name, spec in measures.items() + if not (spec or {}).get("description")] + assert not undescribed, f"measures without description: {undescribed}" + + +def test_judge_class_source_exists(): + """The class named in judge_class exists in the judge source file. + + The module path (judges.ir_axioms.axiom_judge) is container-only, so the + check resolves against trec-auto-judge/axiom_judge.py instead of importing. + """ + cfg = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + module_path, _, class_name = cfg["judge_class"].partition(":") + source = REPO / "trec-auto-judge" / (module_path.rsplit(".", 1)[-1] + ".py") + assert source.exists(), f"judge source {source.name} not found next to workflow.yml" + assert re.search(rf"^class {re.escape(class_name)}\b", + source.read_text(encoding="utf-8"), re.M), ( + f"{class_name} (from judge_class) not defined in {source.name}" + ) + + +@pytest.mark.parametrize("package,minimum", sorted(TEMPLATE_MINIMUMS.items())) +def test_framework_version_if_installed(package, minimum): + """Locally installed framework packages meet the upstream template's pins.""" + packaging_version = pytest.importorskip("packaging.version") + try: + installed = packaging_version.Version(version(package)) + except PackageNotFoundError: + pytest.skip(f"{package} not installed locally (provided by the container base image)") + assert installed >= packaging_version.Version(minimum), ( + f"installed {package} {installed} < {minimum} required by the upstream " + f"starter-kit template — run: pip install --upgrade {package}" + ) From 075aec9f62c43ba927f9a15402e5d610259099c3 Mon Sep 17 00:00:00 2001 From: Laura Date: Sun, 12 Jul 2026 00:58:33 -0400 Subject: [PATCH 2/2] Version checks prefer live template pins over mirrored constants When a starterkit/upstream git remote exists, the minimum versions are read from the template's pyproject.toml directly; the hardcoded constants remain as offline/no-remote fallback, with the remote-add command documented. Co-Authored-By: Claude Fable 5 --- tests/test_trec_auto_judge_workflow.py | 29 ++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/test_trec_auto_judge_workflow.py b/tests/test_trec_auto_judge_workflow.py index fcf1ce3..2e2c5d6 100644 --- a/tests/test_trec_auto_judge_workflow.py +++ b/tests/test_trec_auto_judge_workflow.py @@ -20,11 +20,35 @@ REPO = Path(__file__).parent.parent WORKFLOW = REPO / "trec-auto-judge" / "workflow.yml" -# Mirrors the pins in the upstream template (trec-auto-judge/auto-judge-starter-kit -# pyproject.toml); bump when the template raises its requirements. +# Fallback pins mirroring the upstream template (trec-auto-judge/auto-judge-starter-kit +# pyproject.toml); bump when the template raises its requirements. A live value is +# preferred when a `starterkit` (or `upstream`) git remote exists — add one with: +# git remote add starterkit git@github.com:trec-auto-judge/auto-judge-starter-kit.git TEMPLATE_MINIMUMS = {"autojudge-base": "0.3.18", "tira": "0.0.100"} +def _template_minimum(package: str, fallback: str) -> str: + """The template's current pin from a starterkit/upstream remote, else the fallback.""" + import subprocess + for remote in ("starterkit", "upstream"): + try: + subprocess.run( + ["git", "fetch", "--quiet", remote, "main"], + cwd=REPO, capture_output=True, timeout=10, check=False, + ) + out = subprocess.run( + ["git", "show", f"{remote}/main:pyproject.toml"], + cwd=REPO, capture_output=True, text=True, check=True, + ).stdout + m = re.search(rf'"{re.escape(package)}\s*>=\s*([0-9][0-9a-zA-Z.]*)"', out) + if m: + return m.group(1) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, + FileNotFoundError, OSError): + continue + return fallback + + def test_workflow_parses(): """workflow.yml parses and declares a judge_class — what auto-judge run needs.""" cfg = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) @@ -60,6 +84,7 @@ def test_framework_version_if_installed(package, minimum): installed = packaging_version.Version(version(package)) except PackageNotFoundError: pytest.skip(f"{package} not installed locally (provided by the container base image)") + minimum = _template_minimum(package, minimum) assert installed >= packaging_version.Version(minimum), ( f"installed {package} {installed} < {minimum} required by the upstream " f"starter-kit template — run: pip install --upgrade {package}"