chore: cleanup

Signed-off-by: Nick Goncharenko <ngoncharenko@nvidia.com>

Author: ngoncharenko

packages/nemo_evaluator_sdk/examples/gym/inspect_results.py

@@ -103,8 +103,20 @@ def per_task_attempts(
 
 
 def load_bundle(bundle: Path) -> AgentEvalSummary:
-    """Load the persisted summary, including native and runner aggregates and per-task attempts."""
-    return AgentEvalSummary.model_validate(json.loads((bundle / "summary.json").read_text(encoding="utf-8")))
+    """Load the persisted summary, including native and runner aggregates and per-task attempts.
+
+    Rejects a bundle written before ``task_metric_attempts`` existed rather than reading one. The
+    field defaults to empty, so an older bundle would otherwise load cleanly and simply show no
+    per-task section — the reader would conclude the run had no per-task outcomes rather than that
+    this script cannot see them.
+    """
+    payload = json.loads((bundle / "summary.json").read_text(encoding="utf-8"))
+    if "task_metric_attempts" not in payload:
+        raise SystemExit(
+            f"{bundle / 'summary.json'} predates summary.task_metric_attempts, which this script reads "
+            "per-task outcomes from. Re-run the eval to produce a current bundle."
+        )
+    return AgentEvalSummary.model_validate(payload)
 
 
 # --------------------------------------------------------------------------------------------------

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py

@@ -34,7 +34,7 @@
     serialize_value,
     summary_aggregate_record,
 )
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_serializer
 
 #: Metric-output value schemas retained in the ordered per-task attempt mapping.
 _TASK_METRIC_VALUE_SCHEMAS = (ContinuousScore, DiscreteScore, BooleanValue)
@@ -86,6 +86,20 @@ class AgentEvalAttemptValue(BaseModel):
         ),
     )
 
+    @field_serializer("value")
+    def serialize_nan(self, value: float | None) -> float | str | None:
+        """Emit NaN as the string ``"NaN"``, matching :class:`MetricOutput`.
+
+        A metric may legitimately score an attempt NaN, and this is the first summary field to carry
+        a raw metric value rather than a filtered aggregate. ``json.dumps`` would write it as a bare
+        ``NaN`` token, which is valid Python but not valid JSON, so any strict reader of
+        ``summary.json`` would reject the whole bundle. Pydantic coerces the string back to a float
+        on load, so the round trip is lossless.
+        """
+        if isinstance(value, float) and math.isnan(value):
+            return "NaN"
+        return value
+
 
 class AgentEvalSummary(BaseModel):
     """Aggregated scores, coverage, per-task attempt values, and run counts for an agent-eval run."""
@@ -744,9 +758,10 @@ def _task_metric_attempts(
     reuses one costs pass@k nothing.
     """
     output_keys: dict[str, set[tuple[str, str]]] = {}
-    # Declared under a schema this mapping does not retain. Tracked so an emitted numeric value cannot
-    # add back what the spec filter just excluded.
-    excluded: set[tuple[str, str]] = set()
+    # Per task, the outputs it declared under a schema this mapping does not retain. Tracked so an
+    # emitted numeric value cannot add back what that task's spec filter just excluded -- and keyed by
+    # task because tasks in one run need not declare the same output under the same schema.
+    excluded: dict[str, set[tuple[str, str]]] = {}
     if tasks is not None:
         for task in tasks:
             task_keys = output_keys.setdefault(task.id, set())
@@ -756,16 +771,17 @@ def _task_metric_attempts(
                     if issubclass(spec.value_schema, _TASK_METRIC_VALUE_SCHEMAS):
                         task_keys.add((metric_type, spec.name))
                     else:
-                        excluded.add((metric_type, spec.name))
+                        excluded.setdefault(task.id, set()).add((metric_type, spec.name))
 
     scores_by_task_metric: dict[tuple[str, str], list[AgentEvalTaskScore]] = {}
     for score in scores:
         scores_by_task_metric.setdefault((score.task_id, score.metric_type), []).append(score)
         task_keys = output_keys.setdefault(score.task_id, set())
         if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL):
             continue
+        task_excluded = excluded.get(score.task_id, frozenset())
         for output in score.outputs:
-            if (score.metric_type, output.name) in excluded:
+            if (score.metric_type, output.name) in task_excluded:
                 continue
             if _semantic_value(output) is not None:
                 task_keys.add((score.metric_type, output.name))
@@ -809,11 +825,15 @@ def _task_pass_at_k_scores(
     denominator is never silent. (Tasks excluded from a given ``k`` merely for having fewer than ``k``
     attempts are *not* counted there — that is the estimator working as defined, not missing data.)
 
-    "No usable attempt" includes a task that was never scored at all: a runner that returns no trial
-    for a requested task (Harbor logs a warning and carries on) leaves it declaring the metric with an
-    empty attempt list, and it lands in ``nan_count`` like any other unmeasured task. That is
-    deliberate — it is the same missing coverage whether the trial died or was never produced, and
-    excluding it would report pass@k over a denominator quietly smaller than the task set asked for.
+    "No usable attempt" includes a task that was never scored at all: it declares the metric, holds an
+    empty attempt list, and lands in ``nan_count`` like any other unmeasured task. That is the same
+    missing coverage whether the trial died or was never produced, and excluding it would report
+    pass@k over a denominator quietly smaller than the task set asked for.
+
+    Note this is reachable only through :meth:`AgentEvalSummary.from_scores` called directly with a
+    task list wider than the scores — a caller re-aggregating a subset, say. A full run cannot get
+    here: :meth:`AgentEvaluator._score_trials` refuses to score at all when a task produced no trial,
+    so a runner that drops one fails the run rather than reporting it as missing coverage.
     """
     scorelike = _scorelike_outputs(tasks)
     if not scorelike:

packages/nemo_evaluator_sdk/tests/agent_eval/test_task_metric_attempts.py

@@ -3,6 +3,8 @@
 
 from __future__ import annotations
 
+import json
+import math
 from collections.abc import Iterator
 from pathlib import Path
 
@@ -184,9 +186,11 @@ def test_failed_trials_are_attempts_but_metric_failures_are_unmeasured() -> None
 
 
 def test_a_task_that_produced_no_trial_is_unmeasured_and_counted_in_pass_at_k_nan() -> None:
-    # A runner may return no trial at all for a requested task (Harbor warns and carries on). The task
-    # still declares the metric, so it holds an empty attempt list and counts as missing coverage --
-    # excluding it would report pass@k over a denominator smaller than the task set that was asked for.
+    # from_scores can be handed a task list wider than the scores -- a caller re-aggregating a subset.
+    # The task still declares the metric, so it holds an empty attempt list and counts as missing
+    # coverage: excluding it would report pass@k over a denominator smaller than the task set asked
+    # for. A full run cannot reach this state; AgentEvaluator._score_trials refuses to score when a
+    # task produced no trial, which test_evaluator.py::test_run_rejects_tasks_without_trials pins.
     reward = _Metric("reward", MetricOutputSpec.continuous_score("score"))
     tasks = [_task("scored", reward), _task("never-ran", reward)]
     scores = [_score("scored", "attempt-0", "reward", "score", 1.0)]
@@ -223,6 +227,50 @@ def test_outputs_declared_under_an_unretained_schema_stay_out_even_when_numeric(
     }
 
 
+def test_one_tasks_schema_exclusion_does_not_suppress_another_tasks_output() -> None:
+    # The spec filter is per task: tasks in one run need not declare the same output under the same
+    # schema. Task-a declaring usage.prompt_tokens as a free model must not strip it from task-b,
+    # which never declared it and whose only evidence is the numeric value it actually emitted.
+    tasks = [
+        _task("task-a", _Metric("usage", MetricOutputSpec.model("prompt_tokens", _TokenCount))),
+        _task("task-b", _Metric("reward", MetricOutputSpec.continuous_score("score"))),
+    ]
+    scores = [
+        _score("task-a", "attempt-0", "usage", "prompt_tokens", 100),
+        _score("task-b", "attempt-0", "reward", "score", 1.0),
+        _score("task-b", "attempt-0", "usage", "prompt_tokens", 250),  # undeclared on task-b
+    ]
+
+    attempts = AgentEvalSummary.from_scores(scores, tasks=tasks).task_metric_attempts
+
+    # task-a declared it under an unretained schema, so it is not a key there at all -- not even an
+    # empty one -- and the numeric value it emitted cannot add it back.
+    assert attempts["task-a"] == {}
+    # task-b never declared it, so its emitted numeric value is the only evidence and it is kept.
+    assert sorted(attempts["task-b"]) == ["reward.score", "usage.prompt_tokens"]
+    assert _pairs(attempts)["task-b"]["usage.prompt_tokens"] == [("attempt-0", 250.0)]
+
+
+def test_nan_attempt_values_survive_json_as_a_string() -> None:
+    # A metric may legitimately score NaN. json.dumps would write a bare NaN token, which is not
+    # valid JSON, so summary.json must carry the string form -- and read it back as a float.
+    tasks = [_task("task-a", _Metric("reward", MetricOutputSpec.continuous_score("score")))]
+    summary = AgentEvalSummary.from_scores(
+        [_score("task-a", "attempt-0", "reward", "score", float("nan"))], tasks=tasks
+    )
+
+    payload = summary.model_dump(mode="json")
+    assert payload["task_metric_attempts"]["task-a"]["reward.score"][0]["value"] == "NaN"
+
+    # Strict JSON: no bare NaN/Infinity tokens anywhere in the serialized bundle.
+    def _reject(constant: str) -> float:
+        raise AssertionError(f"summary.json contains a bare {constant} token")
+
+    reloaded = json.loads(json.dumps(payload), parse_constant=_reject)
+    value = AgentEvalSummary.model_validate(reloaded).task_metric_attempts["task-a"]["reward.score"][0].value
+    assert value is not None and math.isnan(value)
+
+
 def test_without_tasks_there_is_no_spec_to_filter_on() -> None:
     # No tasks means no declared schemas to consult, so every numeric output observed is retained.
     scores = [_score("task-a", "attempt-0", "usage", "prompt_tokens", 1234)]
@@ -389,6 +437,18 @@ def test_vendored_results_module_is_a_verbatim_copy_of_this_one() -> None:
     )
 
 
+def test_gym_example_rejects_a_bundle_written_before_task_metric_attempts(tmp_path: Path) -> None:
+    # The field defaults to empty, so an older bundle would load cleanly and simply show no per-task
+    # section -- a reader would take that as "no per-task outcomes" rather than "this script cannot
+    # see them". Fail with a version message instead.
+    from packages.nemo_evaluator_sdk.examples.gym.inspect_results import load_bundle
+
+    (tmp_path / "summary.json").write_text(json.dumps({"task_count": 2}), encoding="utf-8")
+
+    with pytest.raises(SystemExit, match="predates summary.task_metric_attempts"):
+        load_bundle(tmp_path)
+
+
 def test_gym_example_reads_task_outcomes_from_summary() -> None:
     from packages.nemo_evaluator_sdk.examples.gym.inspect_results import per_task_attempts, per_task_outcomes