diff --git a/.gitignore b/.gitignore index 3df6be0..239b914 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,6 @@ venv/ # OS .DS_Store -Thumbs.db \ No newline at end of file +Thumbs.db +# Generated evaluation reports (regenerate via the scoring scripts) +examples/**/reports/ diff --git a/examples/aegis/gen_validators_confirm.py b/examples/aegis/gen_validators_confirm.py new file mode 100644 index 0000000..e1aa2c3 --- /dev/null +++ b/examples/aegis/gen_validators_confirm.py @@ -0,0 +1,101 @@ +"""Add the LLM confirmation layer to AEGIS's already-stored deterministic +``non_llm_validators`` and write augmented result files to ``aegis_findings_confirm/``. + +AEGIS's launcher already runs the validators (each finding carries ``culprit_agent`` ++ per-idx ``evidence`` agents), so — unlike TraceElephant — we do NOT re-run +``run_on_trace``; we only attach ``llm_confirmation`` (~1 call per finding-bearing +trace). The confirmer needs the trace history, which the result files do not store, +so we reload it from the source JSONL, matched on ``sample_id`` (the stable global +row index — the AEGIS ``id`` is NOT unique). We hand the confirmer the exact object +the validators ran on (``rec["input"]``) so ``build_raw_spans`` re-derives the same +span idxs the findings cite. + +AEGIS gold is agent-only (no step index), so the confirmer's only useful job here is +benign-pruning; appointing (``corrected_idx``) is irrelevant for scoring. + +Resumable: existing output files are skipped. All 600 files are written through +(only the finding-bearing ones incur an LLM call) so the scorer reads one folder. + +Usage: + python gen_validators_confirm.py + python gen_validators_confirm.py --limit 1 # smoke test on the first bearing trace +""" + +from __future__ import annotations + +import argparse +import asyncio +import glob +import json +import os +import sys +from pathlib import Path + +from dotenv import load_dotenv + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +load_dotenv(ROOT / ".env") +load_dotenv(THIS_DIR / ".env") + +from maseval.validators.llm_confirm import confirm_trace_async # noqa: E402 + +# Source AEGIS JSONL (conversation_history). Override per machine with $AEGIS_SRC. +DEFAULT_SRC = os.environ.get("AEGIS_SRC", "your/path/test.jsonl") + + +def _n(nlv: dict) -> int: + return sum(len(m.get("findings", [])) for m in (nlv.get("metrics") or {}).values()) + + +async def main(model: str, src: str, limit: int | None) -> None: + if not os.getenv("OPENROUTER_API_KEY"): + raise RuntimeError("OPENROUTER_API_KEY is not set (source the root .env).") + + records = [json.loads(l) for l in open(src, encoding="utf-8") if l.strip()] + print(f"Loaded {len(records)} AEGIS source records from {src}") + + files = sorted(glob.glob(str(THIS_DIR / "aegis_findings" / "*.json"))) + out = THIS_DIR / "aegis_findings_confirm" + out.mkdir(parents=True, exist_ok=True) + totals = {"bearing": 0, "confirmed": 0, "benign": 0, "uncertain": 0, "appointed": 0} + done = 0 + for fp in files: + name = Path(fp).name + out_f = out / name + if out_f.exists(): + continue + data = json.loads(Path(fp).read_text(encoding="utf-8")) + nlv = data.get("non_llm_validators") or {} + if _n(nlv): + sid = int(data["sample_id"]) + trace = (records[sid].get("input") or records[sid]) + try: + await confirm_trace_async(trace, nlv, model) # full reading view + except Exception as exc: # noqa: BLE001 + print(f" [{name}] confirm error: {type(exc).__name__}: {exc}") + s = nlv.get("llm_confirmation_summary", {}) + totals["bearing"] += 1 + for k in ("confirmed", "benign", "uncertain", "appointed"): + totals[k] += int(s.get(k, 0) or 0) + print(f" [{name}] sid={sid} {_n(nlv)} det-findings -> {s}") + done += 1 + data["non_llm_validators"] = nlv + out_f.write_text(json.dumps(data, indent=2, default=str), encoding="utf-8") + if limit is not None and done >= limit: + print(f"(stopped after {done} confirmed traces per --limit)") + break + print(f"totals: {totals}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Add LLM confirmation to AEGIS validators.") + parser.add_argument("--model", default="google/gemini-2.5-flash") + parser.add_argument("--src", default=DEFAULT_SRC, help="AEGIS source JSONL (conversation_history).") + parser.add_argument("--limit", type=int, default=None, help="Confirm at most N bearing traces (smoke test).") + args = parser.parse_args() + asyncio.run(main(model=args.model, src=args.src, limit=args.limit)) diff --git a/examples/aegis/launch_aegis.py b/examples/aegis/launch_aegis.py index d2f092a..7cd09be 100644 --- a/examples/aegis/launch_aegis.py +++ b/examples/aegis/launch_aegis.py @@ -24,6 +24,7 @@ import asyncio import json import os +import time from pathlib import Path from dotenv import load_dotenv @@ -43,6 +44,7 @@ from maseval.metrics import EvidenceVerifier, MetricType, create_metric from maseval.models import RawTraceInput from maseval.reporting import build_evaluation_report +from maseval.run_stats import aggregate_task_stats, extract_usage from maseval.validators import run_on_trace LLM_METRICS_TO_TEST = [ @@ -110,21 +112,40 @@ def _format_indexed_raw_trace(steps: list, query: str) -> str: return "\n".join(lines) -async def _run_all_metrics(model, eval_input: RawTraceInput) -> dict: +async def _run_all_metrics(model, eval_input: RawTraceInput) -> tuple[dict, dict]: """Run all LLM evaluators for one trace sequentially (mirrors the WW launcher; - avoids bursting OpenRouter rate limits with concurrent calls).""" + avoids bursting OpenRouter rate limits with concurrent calls). + + Returns ``(findings_results, metric_status)`` — the latter carries per-metric + timing + token usage for the ``task_stats`` rollup. + """ findings_results: dict = {} + metric_status: dict = {} for metric_type in LLM_METRICS_TO_TEST: - print(f" -- LLM metric: {metric_type.value}") + name = metric_type.value + print(f" -- LLM metric: {name}") + metric = None + t0 = time.perf_counter() try: metric = create_metric(metric_type, model) result = await metric.evaluate(eval_input) - findings_results[metric_type.value] = result - print(f" findings: {len(result.findings)}") + inp, out, tot = extract_usage(getattr(metric, "last_usage", None)) + findings_results[name] = result + metric_status[name] = { + "status": "ok", "duration_s": round(time.perf_counter() - t0, 3), + "input_tokens": inp, "output_tokens": out, "total_tokens": tot, + } + print(f" findings: {len(result.findings)} | in={inp} out={out}") except Exception as e: # noqa: BLE001 - isolate a single metric failure - print(f" [skip] {metric_type.value}: {e}") + inp, out, tot = extract_usage(getattr(metric, "last_usage", None)) + metric_status[name] = { + "status": "failed", "detail": f"{type(e).__name__}: {e}"[:300], + "duration_s": round(time.perf_counter() - t0, 3), + "input_tokens": inp, "output_tokens": out, "total_tokens": tot, + } + print(f" [skip] {name}: {e}") continue - return findings_results + return findings_results, metric_status def _serialize_findings(findings_results: dict) -> dict: @@ -174,7 +195,9 @@ async def main(input_path: str, out_dir: str, model_name: str, from_idx: int, li eval_input = RawTraceInput(trace=_format_indexed_raw_trace(steps, query)) - findings_results = await _run_all_metrics(model, eval_input) + _t0 = time.perf_counter() + findings_results, metric_status = await _run_all_metrics(model, eval_input) + task_stats = aggregate_task_stats(metric_status, time.perf_counter() - _t0) evidence_results = EvidenceVerifier().verify_all(findings_results, eval_input) gt_answer = (rec.get("ground_truth") or {}).get("correct_answer") @@ -189,6 +212,7 @@ async def main(input_path: str, out_dir: str, model_name: str, from_idx: int, li # can consume the file identically to the WW pipeline. payload: dict = {} payload.update(_serialize_findings(findings_results)) + payload["task_stats"] = task_stats payload["evidence_verification"] = _serialize_evidence(evidence_results) payload["non_llm_validators"] = run_on_trace(rec.get("input") or rec) payload["reference_answer"] = gt_answer diff --git a/examples/aegis/score_none_plus_validators.py b/examples/aegis/score_none_plus_validators.py new file mode 100644 index 0000000..1c545da --- /dev/null +++ b/examples/aegis/score_none_plus_validators.py @@ -0,0 +1,255 @@ +"""Score AEGIS agent localization for verifier=``none``, folding in the +deterministic ``non_llm_validators`` (+ LLM confirmation) — the AEGIS analogue of +the Who&When / TraceElephant ``score_none_plus_validators``. + +AEGIS gold is **agent-only** (``ground_truth_faulty_agents``; no step index), so: + +* the reported metrics are agent Top-1 / Hit / Exact-Set **and** the set-based + micro/macro precision/recall/F1 (the AutoJudge AEGIS reference math, reused from + ``verifier_ablation.calculate_metrics``); +* the confirmer's only useful job is **benign-pruning** — appointing + (``corrected_idx``) is irrelevant with no step gold, so there is no locus choice. + +Each validator finding already names its ``culprit_agent`` (no idx→agent map +needed, unlike TraceElephant). We OR that agent into the verifier=``none`` agent +set under three verdict filters: + +* ``all`` -- every regex finding (no confirmer gating); +* ``conf+unc`` -- confirmer verdict confirmed OR uncertain; +* ``confirmed`` -- confirmed only (**the honest column** — folding in guesses + raises recall but costs precision; the confirmed filter is the + precision protection, so F1 is the honest arbiter here). + +Only 49/600 traces are finding-bearing, so the fold-in is invisible in the 600-trace +overall; we therefore also report the **bearing-only** subset, where the signal lives. + +Usage: + python score_none_plus_validators.py + python score_none_plus_validators.py --pred-dir aegis_findings_confirm +""" + +from __future__ import annotations + +import argparse +import glob +import json +import sys +from pathlib import Path + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from maseval.diagnostic_accuracy import _normalize_agent, read_prediction_file # noqa: E402 + +from verifier_ablation import calculate_metrics # noqa: E402 (AEGIS reference P/R/F1) + +FILTERS = ("none_only", "all", "conf_unc", "confirmed") +FILTER_LABELS = { + "none_only": "none (LLM only)", + "all": "none + val (all)", + "conf_unc": "none + val (conf+unc)", + "confirmed": "none + val (confirmed)", +} + + +def _gt_agents(rec: dict) -> list[str]: + out = [] + for a in rec.get("ground_truth_faulty_agents") or []: + name = a.get("agent_name") if isinstance(a, dict) else a + if name: + out.append(str(name)) + return out + + +def _iter_val_findings(nlv: dict): + for m in (nlv.get("metrics") or {}).values(): + for f in m.get("findings", []): + yield f + + +def _val_agents(finding: dict) -> list[str]: + """The agent(s) a validator finding blames: its ``culprit_agent`` plus any + per-idx evidence agents (deduped, order preserved).""" + out = [] + c = finding.get("culprit_agent") + if c: + out.append(str(c)) + for ev in finding.get("evidence") or []: + a = ev.get("agent") + if a and str(a) not in out: + out.append(str(a)) + return out + + +def _passes(finding: dict, filt: str) -> bool: + if filt == "all": + return True + verdict = (finding.get("llm_confirmation") or {}).get("verdict") + if filt == "conf_unc": + return verdict in ("confirmed", "uncertain") + if filt == "confirmed": + return verdict == "confirmed" + return False + + +def score(pred_dir: str) -> dict: + pred_dir_p = THIS_DIR / pred_dir if not Path(pred_dir).is_absolute() else Path(pred_dir) + files = sorted(glob.glob(str(pred_dir_p / "*.json"))) + if not files: + raise FileNotFoundError(f"No prediction files in {pred_dir_p}") + + # Per-filter accumulators: agent bool lists + (gold,pred) sets for P/R/F1. + acc = {f: {"top1": [], "hit": [], "exact": [], "gold_sets": [], "pred_sets": []} + for f in FILTERS} + acc_bearing = {f: {"top1": [], "hit": [], "exact": [], "gold_sets": [], "pred_sets": []} + for f in FILTERS} + matched = 0 + bearing = 0 + counts = {"confirmed": 0, "benign": 0, "uncertain": 0} + + for fp in files: + data = json.loads(Path(fp).read_text(encoding="utf-8")) + gold = {_normalize_agent(a) for a in _gt_agents(data)} + if not gold: + continue + matched += 1 + + pred = read_prediction_file(fp, verifier_mode="none") + base_agents = list(pred.agents) + base_primary = pred.primary_agent + + nlv = data.get("non_llm_validators") or {} + val_hits = list(_iter_val_findings(nlv)) + is_bearing = bool(val_hits) + if is_bearing: + bearing += 1 + s = nlv.get("llm_confirmation_summary") or {} + for k in ("confirmed", "benign", "uncertain"): + counts[k] += int(s.get(k, 0) or 0) + + for filt in FILTERS: + agents = list(base_agents) + primary = base_primary + if filt != "none_only": + for finding in val_hits: + if not _passes(finding, filt): + continue + for a in _val_agents(finding): + if a not in agents: + agents.append(a) + if primary is None: + primary = a + + pred_set = {_normalize_agent(a) for a in agents} + primary_norm = _normalize_agent(primary) if primary else None + top1 = bool(primary_norm and primary_norm in gold) + hit = any(a in gold for a in pred_set) + exact = pred_set == gold + for store, active in ((acc, True), (acc_bearing, is_bearing)): + if not active: + continue + store[filt]["top1"].append(top1) + store[filt]["hit"].append(hit) + store[filt]["exact"].append(exact) + store[filt]["gold_sets"].append(gold) + store[filt]["pred_sets"].append(pred_set) + + def summarize(store: dict) -> dict: + out = {} + for f in FILTERS: + d = store[f] + n = len(d["top1"]) + prf = calculate_metrics(d["gold_sets"], d["pred_sets"]) if n else {} + out[f] = { + "agent_top1": (sum(d["top1"]) / n) if n else None, + "agent_hit": (sum(d["hit"]) / n) if n else None, + "agent_exact": (sum(d["exact"]) / n) if n else None, + "precision_micro": prf.get("precision_micro"), + "recall_micro": prf.get("recall_micro"), + "f1_micro": prf.get("f1_micro"), + "f1_macro": prf.get("f1_macro"), + } + return out + + return { + "matched": matched, + "bearing": bearing, + "counts": counts, + "overall": summarize(acc), + "bearing_only": summarize(acc_bearing), + } + + +def _pct(v): + return f"{v * 100:5.1f}%" if isinstance(v, (int, float)) else " —" + + +ROWS = [ + ("Agent Top-1", "agent_top1"), ("Agent Hit", "agent_hit"), ("Agent Exact-Set", "agent_exact"), + ("Precision (micro)", "precision_micro"), ("Recall (micro)", "recall_micro"), + ("F1 (micro)", "f1_micro"), ("F1 (macro)", "f1_macro"), +] + + +def _render(title: str, summ: dict, n: int) -> str: + header = f"{'Metric':<18}" + "".join(FILTER_LABELS[f].rjust(22) for f in FILTERS) + lines = [f"=== {title} (n={n}) ===", "", header, "-" * len(header)] + for label, key in ROWS: + lines.append(f"{label:<18}" + "".join(_pct(summ[f][key]).rjust(22) for f in FILTERS)) + return "\n".join(lines) + + +def _markdown(res: dict, pred_dir: str) -> str: + md = [ + "# AEGIS — none + non_llm_validators (LLM-confirm) fold-in", + "", + f"- Predictions: `{pred_dir}`", + f"- Matched (has gold): {res['matched']}; finding-bearing: {res['bearing']}", + f"- Confirmer: confirmed={res['counts']['confirmed']}, " + f"uncertain={res['counts']['uncertain']}, benign={res['counts']['benign']}", + "- Baseline: verifier=`none` over the 11 LLM evaluators.", + "- Fold-in agent = validator `culprit_agent` (+ per-idx evidence agents). " + "AEGIS gold is agent-only, so no step metrics and no locus/appointing.", + "- Filters: `all` = every regex finding; `conf+unc` = confirmed|uncertain; " + "`confirmed` = confirmed only. **F1 is the honest arbiter** (fold-in trades " + "precision for recall; the confirmed filter protects precision).", + "", + ] + for title, key in (("Overall (all traces with gold)", "overall"), + ("Bearing-only (traces with ≥1 validator finding)", "bearing_only")): + n = res["matched"] if key == "overall" else res["bearing"] + md.append(f"## {title} (n={n})") + md.append("") + md.append("| Metric | " + " | ".join(FILTER_LABELS[f] for f in FILTERS) + " |") + md.append("|---|" + "---:|" * len(FILTERS)) + s = res[key] + for label, mk in ROWS: + md.append("| " + label + " | " + " | ".join(_pct(s[f][mk]).strip() for f in FILTERS) + " |") + md.append("") + return "\n".join(md) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="AEGIS none + validators (llm-confirm) fold-in.") + parser.add_argument("--pred-dir", default="aegis_findings_confirm") + args = parser.parse_args() + res = score(args.pred_dir) + + print(_render("AEGIS — Overall (all traces with gold)", res["overall"], res["matched"])) + print() + print(f"confirmer: confirmed={res['counts']['confirmed']} " + f"uncertain={res['counts']['uncertain']} benign={res['counts']['benign']}") + print() + print(_render("AEGIS — Bearing-only", res["bearing_only"], res["bearing"])) + + reports_dir = THIS_DIR / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + (reports_dir / "none_plus_validators.json").write_text( + json.dumps({"pred_dir": args.pred_dir, "filter_labels": FILTER_LABELS, **res}, + indent=2), encoding="utf-8") + (reports_dir / "none_plus_validators.md").write_text( + _markdown(res, args.pred_dir), encoding="utf-8") + print(f"\nSaved: {reports_dir / 'none_plus_validators.md'}") diff --git a/examples/aegis/verifier_ablation.py b/examples/aegis/verifier_ablation.py new file mode 100644 index 0000000..199e793 --- /dev/null +++ b/examples/aegis/verifier_ablation.py @@ -0,0 +1,209 @@ +"""EvidenceVerifier ablation for AEGIS agent localization. + +Scores the same AEGIS result files under three verifier gating settings and +writes one JSON report per setting, plus a printed comparison table: + +* ``none`` -- no verifier: every LLM finding counts; +* ``strict`` -- only ``verified`` findings count (weak + invalid -> review); +* ``soft`` -- ``verified``/``weak`` count, ``invalid`` -> review (prior default). + +``non_llm_validators`` are NOT counted in any setting (LLM findings only). + +Unlike the Who&When ablation (which reads gold from an HF parquet), AEGIS gold +culprit agents are embedded in each result file (``ground_truth_faulty_agents``); +we build the annotation table from them, keyed on ``sample_id`` (the stable global +row index — the AEGIS ``id`` is NOT unique). AEGIS gold has no step index, so only +the AGENT-level metrics are populated. + +Debugger-friendly: call ``run(...)`` directly, or run as a script. + + run(pred_glob="aegis_findings") +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Sequence + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from maseval.diagnostic_accuracy import ( # noqa: E402 + evaluate_agent_step_accuracy, + _normalize_agent, +) + +MODES = ("none", "strict", "soft") + + +def calculate_metrics(true_labels_list, pred_labels_list): + """Set-based micro/macro precision/recall/F1 — verbatim port of the AutoJudge + AEGIS ``calculate.py`` reference math (kept inline so this ablation is + self-contained on ``main``, where ``calculate.py`` lives only on the aegis branch). + """ + all_classes = set() + for labels in true_labels_list: + all_classes.update(labels) + for labels in pred_labels_list: + all_classes.update(labels) + all_classes = list(all_classes) + + micro_tp = micro_fp = micro_fn = 0 + per_class_stats = {cls: {"tp": 0, "fp": 0, "fn": 0} for cls in all_classes} + + for true_set, pred_set in zip(true_labels_list, pred_labels_list): + tp_set = true_set.intersection(pred_set) + fp_set = pred_set.difference(true_set) + fn_set = true_set.difference(pred_set) + micro_tp += len(tp_set) + micro_fp += len(fp_set) + micro_fn += len(fn_set) + for cls in tp_set: + per_class_stats[cls]["tp"] += 1 + for cls in fp_set: + per_class_stats[cls]["fp"] += 1 + for cls in fn_set: + per_class_stats[cls]["fn"] += 1 + + def _prf(tp, fp, fn): + p = tp / (tp + fp) if (tp + fp) > 0 else 0.0 + r = tp / (tp + fn) if (tp + fn) > 0 else 0.0 + f = 2 * p * r / (p + r) if (p + r) > 0 else 0.0 + return p, r, f + + micro_p, micro_r, micro_f1 = _prf(micro_tp, micro_fp, micro_fn) + + macro_p_list, macro_r_list, macro_f1_list = [], [], [] + for cls in all_classes: + s = per_class_stats[cls] + p, r, f = _prf(s["tp"], s["fp"], s["fn"]) + macro_p_list.append(p) + macro_r_list.append(r) + macro_f1_list.append(f) + + _avg = lambda xs: sum(xs) / len(xs) if xs else 0.0 + return { + "precision_micro": micro_p, + "recall_micro": micro_r, + "f1_micro": micro_f1, + "precision_macro": _avg(macro_p_list), + "recall_macro": _avg(macro_r_list), + "f1_macro": _avg(macro_f1_list), + } + + +# AEGIS is agent-level only (gold has no step index). The *_acc metrics come from +# the WW code path; the P/R/F1 keys are the set-based micro/macro scores computed +# by ``calculate_metrics`` (the AutoJudge AEGIS reference math), folded in per mode. +METRIC_KEYS = [ + ("agent_top1_acc", "Agent Top-1"), + ("agent_hit_acc", "Agent Hit"), + ("agent_exact_set_acc", "Agent Exact-Set"), + ("precision_micro", "Precision (micro)"), + ("recall_micro", "Recall (micro)"), + ("f1_micro", "F1 (micro)"), + ("f1_macro", "F1 (macro)"), +] + + +def _agent_f1(per_example: list[dict]) -> dict: + """Set-based micro/macro P/R/F1 over normalized agent names, per calculate.py.""" + true_list, pred_list = [], [] + for row in per_example: + true_list.append({_normalize_agent(a) for a in row.get("gold_agents") or []}) + pred_list.append({_normalize_agent(a) for a in row.get("predicted_agents") or []}) + return calculate_metrics(true_list, pred_list) + + +def _gt_agent_names(rec: dict) -> list[str]: + out = [] + for a in rec.get("ground_truth_faulty_agents") or []: + name = a.get("agent_name") if isinstance(a, dict) else a + if name: + out.append(str(name)) + return out + + +def run( + pred_glob: str | Path = THIS_DIR / "aegis_findings", + *, + split: str = "aegis", + dataset_name: str = "AEGIS", + reports_dir: str | Path = THIS_DIR / "reports", +) -> dict[str, dict]: + """Score the 3 verifier modes, write per-mode reports, return the summaries.""" + files = sorted(Path(pred_glob).glob("*.json")) + if not files: + raise SystemExit(f"No result files in {pred_glob}") + + # Build the annotation table from the ground truth embedded in each result file. + ann_rows = [] + for fp in files: + rec = json.load(open(fp, encoding="utf-8")) + ann_rows.append({"sample_id": rec.get("sample_id"), "faulty_agents": _gt_agent_names(rec)}) + + ann_path = Path(pred_glob).parent / "aegis_annotations.jsonl" + with open(ann_path, "w", encoding="utf-8") as f: + for row in ann_rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + reports_dir = Path(reports_dir) + reports_dir.mkdir(parents=True, exist_ok=True) + + file_strs = [str(fp) for fp in files] + summaries: dict[str, dict] = {} + for mode in MODES: + result = evaluate_agent_step_accuracy( + prediction_paths=file_strs, + annotation_path=str(ann_path), + id_column="sample_id", + agent_columns=["faulty_agents"], + build_missing_report=True, + verifier_mode=mode, + ) + summary = dict(result["summary"]) + summary.update(_agent_f1(result["per_example"])) + summaries[mode] = summary + result["summary"] = summary + with open(reports_dir / f"verifier_ablation_{split}_{mode}.json", "w", encoding="utf-8") as f: + json.dump(result, f, indent=2, ensure_ascii=False) + + print(_render(split, dataset_name, summaries)) + return summaries + + +def _pct(v): + return f"{v * 100:5.1f}%" if isinstance(v, (int, float)) else " —" + + +def _render(split: str, dataset_name: str, summaries: dict[str, dict]) -> str: + matched = summaries["soft"].get("matched_annotations") + lines = [ + f"=== Verifier ablation — {split.upper()} (matched {matched}) ===", + f"Dataset: {dataset_name}", + "LLM findings only; non_llm_validators not counted. Agent-level (AEGIS gold has no step).", + "", + f"{'Metric':<18}{'none':>9}{'strict':>9}{'soft':>9}", + "-" * 45, + ] + for key, label in METRIC_KEYS: + row = "".join(_pct(summaries[m].get(key)).rjust(9) for m in MODES) + lines.append(f"{label:<18}{row}") + return "\n".join(lines) + + +if __name__ == "__main__": + import argparse + + here = Path(__file__).resolve().parent + parser = argparse.ArgumentParser(description="AEGIS EvidenceVerifier ablation.") + parser.add_argument("--pred-glob", default=str(here / "aegis_findings"), + help="Directory of AEGIS result JSON files.") + args = parser.parse_args() + + run(pred_glob=args.pred_glob) diff --git a/examples/agentrx/reports/agentrx_magentic_all.md b/examples/agentrx/reports/agentrx_magentic_all.md deleted file mode 100644 index 5657f0f..0000000 --- a/examples/agentrx/reports/agentrx_magentic_all.md +++ /dev/null @@ -1,108 +0,0 @@ -# AgentRx magentic — gold=all - -Generated: `2026-07-07 19:30:22` - -## Notes - -AgentRx / magentic, gold_scope='all'. Spans keyed by the native 1-based step index (== gold step_number). non_llm_validators are not used. Step Top-1 uses first_idx_mode='top_ranked' (top_ranked = the model's #1-ranked span), comparable to the paper's single-root-cause Step Acc; step_top1_pm1/pm3/pm5 add ±1/±3/±5 tolerance to match Table 6's Acc@±k. - -## Experiment setup - -| Field | Value | -|---|---| -| Experiment | AgentRx magentic — gold=all | -| Model / judge | google/gemini-2.5-flash | -| Dataset | AgentRx / Magentic-One | -| Run label | AgentRx findings (all gold) | -| Prediction glob | /mnt/c/Users/barak/MASeval/examples/agentrx/agentrx_magentic_findings | -| Prediction files | 44 | -| HF annotations | microsoft/AgentRx :: magentic (all) | -| Annotation rows | 44 | -| Matched annotations | 44 | -| Unmatched predictions | 0 | -| ID column | trajectory_id | -| Agent annotation columns | mistake_agents | -| Step annotation columns | mistake_steps | -| Step tolerance | ±1 | -| Build missing report | True | -| Verifier mode (ablation) | soft (stored) | -| EvidenceVerifier policy | verified/weak are counted; invalid findings are excluded and left for review | - -## Main results - -| Group | Metric | Value | Count | Meaning | -|---|---:|---:|---:|---| -| Agent | Top-1 Acc | 72.7% | 44 | Primary culprit agent matches the annotation. | -| Agent | Hit Acc | 97.7% | 44 | At least one predicted culprit agent matches the annotation. | -| Agent | Exact Set Acc | 34.1% | 44 | Predicted agent set exactly equals the gold agent set. | -| Step | Top-1 Acc | 43.2% | 44 | First predicted problematic idx equals the gold idx. | -| Step | Hit Acc | 88.6% | 44 | At least one predicted idx exactly matches a gold idx. | -| Step | Hit Acc ±1 | 93.2% | 44 | At least one predicted numeric idx is within ±1 of a gold idx. | - -## Diagnostic quality summary - -| Field | Value | -|---|---:| -| Invalid findings excluded from main predictions | 10 | -| Agent examples | 44 | -| Step examples | 44 | - -## Interpretation - -- Agent localization: Hit Acc is 97.7%, Top-1 Acc is 72.7% (gap 25.0%). A large gap means the correct agent is often present but not ranked first. -- Step localization: exact Hit Acc is 88.6%, relaxed Hit Acc ±1 is 93.2% (gap 4.5%). A large gap suggests off-by-one or indexing mismatch. -- First-idx ranking: Step Top-1 Acc is 43.2%. If this is much lower than Step Hit Acc, use the full problematic_idxs list rather than only first_problem_idx. - -## Example mismatches / review targets (top 13) - -| # | File | Gold agents | Predicted agents | Agent hit | Gold idxs | Predicted idxs | Step hit ±1 | Invalid findings | -|---:|---|---|---|---:|---|---|---:|---:| -| 1 | findings_25.json | Assistant | | ❌ | 17 | | ❌ | 0 | -| 2 | findings_2.json | Websurfer | WebSurfer, Orchestrator | ✅ | 17 | 59, 63, 67, 75, 79, 83, … (+12) | ❌ | 0 | -| 3 | findings_4.json | Orchestrator | Orchestrator, WebSurfer | ✅ | 4 | 17, 2, 10, 13, 15, 1, … (+2) | ❌ | 0 | -| 4 | findings_29.json | Assistant | Assistant | ✅ | 12 | 13, 16, 1, 11 | ✅ | 0 | -| 5 | findings_8.json | Orchestrator, WebSurfer | Orchestrator, WebSurfer | ✅ | 7, 9, 10 | 25, 37, 23, 21, 11, 15, … (+3) | ✅ | 0 | -| 6 | findings_12.json | WebSurfer, Orchestrator | Orchestrator, WebSurfer | ✅ | 9, 10, 13, 14, 17, 18, … (+1) | 13, 11, 15, 14, 26, 32, … (+12) | ✅ | 2 | -| 7 | findings_21.json | WebSurfer, Orchestrator | WebSurfer, Orchestrator, Assistant | ✅ | 5, 13, 17, 25, 29, 33, … (+16) | 93, 115, 100, 126, 42, 53, … (+40) | ✅ | 2 | -| 8 | findings_16.json | WebSurfer, Orchestrator | Orchestrator, WebSurfer, Assistant | ✅ | 9, 10 | 30, 9, 25, 29, 1, 21, … (+5) | ✅ | 1 | -| 9 | findings_19.json | WebSurfer, Orchestrator | Orchestrator, WebSurfer, Assistant | ✅ | 6, 7, 10, 14, 18, 22, … (+20) | 113, 23, 27, 15, 104, 26, … (+24) | ✅ | 1 | -| 10 | findings_22.json | WebSurfer | Orchestrator, WebSurfer | ✅ | 5, 12, 16, 20, 24, 28, … (+13) | 25, 37, 56, 14, 64, 13, … (+20) | ✅ | 1 | -| 11 | findings_24.json | WebSurfer, Assistant | Orchestrator, WebSurfer | ✅ | 9, 21, 25 | 9, 2, 21, 22, 5 | ✅ | 1 | -| 12 | findings_28.json | Orchestrator | Orchestrator | ✅ | 2 | 2, 1, 5 | ✅ | 1 | -| 13 | findings_41.json | WebSurfer, Orchestrator | Orchestrator, WebSurfer, Assistant | ✅ | 17, 21, 31 | 13, 10, 21, 22, 31, 14, … (+11) | ✅ | 1 | - -## Reproducibility - -```json -{ - "experiment_name": "AgentRx magentic — gold=all", - "model_name": "google/gemini-2.5-flash", - "dataset_name": "AgentRx / Magentic-One", - "run_label": "AgentRx findings (all gold)", - "pred_glob": "/mnt/c/Users/barak/MASeval/examples/agentrx/agentrx_magentic_findings", - "hf_annotations": "microsoft/AgentRx :: magentic (all)", - "output_json_path": "/mnt/c/Users/barak/MASeval/examples/agentrx/reports/agentrx_magentic_all.json", - "output_md_path": "/mnt/c/Users/barak/MASeval/examples/agentrx/reports/agentrx_magentic_all.md", - "id_column": "trajectory_id", - "agent_columns": [ - "mistake_agents" - ], - "step_columns": [ - "mistake_steps" - ], - "step_tolerance": 1, - "build_missing_report": true, - "top_error_examples": 20, - "notes": "AgentRx / magentic, gold_scope='all'. Spans keyed by the native 1-based step index (== gold step_number). non_llm_validators are not used. Step Top-1 uses first_idx_mode='top_ranked' (top_ranked = the model's #1-ranked span), comparable to the paper's single-root-cause Step Acc; step_top1_pm1/pm3/pm5 add ±1/±3/±5 tolerance to match Table 6's Acc@±k." -} -``` - -## Paper comparison — Table 6 (root-cause Step Acc, top-1) - -Top-1 span (first_idx_mode=top_ranked) vs gold at ±0/±1/±3/±5. The paper reports a single predicted root-cause step, so `gold_scope=root_cause` is the closest analog. Their AGENTRX is a trained method; ours is an untrained LLM judge — treat this as orientation, not parity. - -| Source | Step Acc | Acc@±1 | Acc@±3 | Acc@±5 | -|---|---|---|---|---| -| Ours (magentic / all) | 43.2% | 52.3% | 65.9% | 72.7% | -| Paper AGENTRX — Magentic-One | 31.8% | 40.9% | 47.7% | 50.8% | -| Paper AGENTRX — Magentic-One* | 46.9% | 61.7% | 72.8% | 79.0% | diff --git a/examples/agentrx/reports/agentrx_magentic_root_cause.md b/examples/agentrx/reports/agentrx_magentic_root_cause.md deleted file mode 100644 index fcb5dd7..0000000 --- a/examples/agentrx/reports/agentrx_magentic_root_cause.md +++ /dev/null @@ -1,115 +0,0 @@ -# AgentRx magentic — gold=root_cause - -Generated: `2026-07-07 19:31:55` - -## Notes - -AgentRx / magentic, gold_scope='root_cause'. Spans keyed by the native 1-based step index (== gold step_number). non_llm_validators are not used. Step Top-1 uses first_idx_mode='top_ranked' (top_ranked = the model's #1-ranked span), comparable to the paper's single-root-cause Step Acc; step_top1_pm1/pm3/pm5 add ±1/±3/±5 tolerance to match Table 6's Acc@±k. - -## Experiment setup - -| Field | Value | -|---|---| -| Experiment | AgentRx magentic — gold=root_cause | -| Model / judge | google/gemini-2.5-flash | -| Dataset | AgentRx / Magentic-One | -| Run label | AgentRx findings (root_cause gold) | -| Prediction glob | /mnt/c/Users/barak/MASeval/examples/agentrx/agentrx_magentic_findings | -| Prediction files | 44 | -| HF annotations | microsoft/AgentRx :: magentic (root_cause) | -| Annotation rows | 44 | -| Matched annotations | 44 | -| Unmatched predictions | 0 | -| ID column | trajectory_id | -| Agent annotation columns | root_cause_agent | -| Step annotation columns | root_cause_step | -| Step tolerance | ±1 | -| Build missing report | True | -| Verifier mode (ablation) | soft (stored) | -| EvidenceVerifier policy | verified/weak are counted; invalid findings are excluded and left for review | - -## Main results - -| Group | Metric | Value | Count | Meaning | -|---|---:|---:|---:|---| -| Agent | Top-1 Acc | 63.6% | 44 | Primary culprit agent matches the annotation. | -| Agent | Hit Acc | 97.7% | 44 | At least one predicted culprit agent matches the annotation. | -| Agent | Exact Set Acc | 4.5% | 44 | Predicted agent set exactly equals the gold agent set. | -| Step | Top-1 Acc | 15.9% | 44 | First predicted problematic idx equals the gold idx. | -| Step | Hit Acc | 56.8% | 44 | At least one predicted idx exactly matches a gold idx. | -| Step | Hit Acc ±1 | 77.3% | 44 | At least one predicted numeric idx is within ±1 of a gold idx. | - -## Diagnostic quality summary - -| Field | Value | -|---|---:| -| Invalid findings excluded from main predictions | 10 | -| Agent examples | 44 | -| Step examples | 44 | - -## Interpretation - -- Agent localization: Hit Acc is 97.7%, Top-1 Acc is 63.6% (gap 34.1%). A large gap means the correct agent is often present but not ranked first. -- Step localization: exact Hit Acc is 56.8%, relaxed Hit Acc ±1 is 77.3% (gap 20.5%). A large gap suggests off-by-one or indexing mismatch. -- First-idx ranking: Step Top-1 Acc is 15.9%. If this is much lower than Step Hit Acc, use the full problematic_idxs list rather than only first_problem_idx. - -## Example mismatches / review targets (top 20) - -| # | File | Gold agents | Predicted agents | Agent hit | Gold idxs | Predicted idxs | Step hit ±1 | Invalid findings | -|---:|---|---|---|---:|---|---|---:|---:| -| 1 | findings_25.json | Assistant | | ❌ | 17 | | ❌ | 0 | -| 2 | findings_21.json | WebSurfer | WebSurfer, Orchestrator, Assistant | ✅ | 5 | 93, 115, 100, 126, 42, 53, … (+40) | ❌ | 2 | -| 3 | findings_15.json | WebSurfer | WebSurfer, Orchestrator, FileSurfer | ✅ | 13 | 106, 110, 114, 118, 98, 102, … (+24) | ❌ | 0 | -| 4 | findings_17.json | Orchestrator | FileSurfer, Orchestrator, WebSurfer | ✅ | 15 | 21, 36, 51, 29, 18, 22, … (+6) | ❌ | 0 | -| 5 | findings_18.json | Orchestrator | WebSurfer, Orchestrator | ✅ | 10 | 13, 21, 25, 22, 14, 26, … (+1) | ❌ | 0 | -| 6 | findings_2.json | Websurfer | WebSurfer, Orchestrator | ✅ | 17 | 59, 63, 67, 75, 79, 83, … (+12) | ❌ | 0 | -| 7 | findings_27.json | Orchestrator | Orchestrator, WebSurfer | ✅ | 7 | 10, 12, 1, 4, 5, 9 | ❌ | 0 | -| 8 | findings_4.json | Orchestrator | Orchestrator, WebSurfer | ✅ | 4 | 17, 2, 10, 13, 15, 1, … (+2) | ❌ | 0 | -| 9 | findings_42.json | Orchestrator | Orchestrator, WebSurfer | ✅ | 4 | 21, 25, 26, 52, 9, 13, … (+12) | ❌ | 0 | -| 10 | findings_7.json | WebSurfer | Orchestrator, WebSurfer | ✅ | 5 | 29, 33, 55, 67, 90, 113, … (+22) | ❌ | 0 | -| 11 | findings_12.json | Orchestrator | Orchestrator, WebSurfer | ✅ | 10 | 13, 11, 15, 14, 26, 32, … (+12) | ✅ | 2 | -| 12 | findings_19.json | Orchestrator | Orchestrator, WebSurfer, Assistant | ✅ | 7 | 113, 23, 27, 15, 104, 26, … (+24) | ✅ | 1 | -| 13 | findings_22.json | WebSurfer | Orchestrator, WebSurfer | ✅ | 5 | 25, 37, 56, 14, 64, 13, … (+20) | ✅ | 1 | -| 14 | findings_1.json | Orchestrator | Orchestrator, WebSurfer | ✅ | 10 | 89, 11, 13, 15, 17, 19, … (+5) | ✅ | 0 | -| 15 | findings_13.json | Orchestrator | Orchestrator, FileSurfer, WebSurfer, Assistant | ✅ | 4 | 19, 91, 6, 14, 18, 62, … (+44) | ✅ | 0 | -| 16 | findings_20.json | Orchestrator | Orchestrator, WebSurfer, Assistant, ComputerTerminal | ✅ | 27 | 102, 68, 72, 106, 107, 14, … (+34) | ✅ | 0 | -| 17 | findings_29.json | Assistant | Assistant | ✅ | 12 | 13, 16, 1, 11 | ✅ | 0 | -| 18 | findings_5.json | Orchestrator | Orchestrator, WebSurfer | ✅ | 7 | 5, 6, 8 | ✅ | 0 | -| 19 | findings_8.json | Orchestrator | Orchestrator, WebSurfer | ✅ | 10 | 25, 37, 23, 21, 11, 15, … (+3) | ✅ | 0 | -| 20 | findings_16.json | Orchestrator | Orchestrator, WebSurfer, Assistant | ✅ | 10 | 30, 9, 25, 29, 1, 21, … (+5) | ✅ | 1 | - -## Reproducibility - -```json -{ - "experiment_name": "AgentRx magentic — gold=root_cause", - "model_name": "google/gemini-2.5-flash", - "dataset_name": "AgentRx / Magentic-One", - "run_label": "AgentRx findings (root_cause gold)", - "pred_glob": "/mnt/c/Users/barak/MASeval/examples/agentrx/agentrx_magentic_findings", - "hf_annotations": "microsoft/AgentRx :: magentic (root_cause)", - "output_json_path": "/mnt/c/Users/barak/MASeval/examples/agentrx/reports/agentrx_magentic_root_cause.json", - "output_md_path": "/mnt/c/Users/barak/MASeval/examples/agentrx/reports/agentrx_magentic_root_cause.md", - "id_column": "trajectory_id", - "agent_columns": [ - "root_cause_agent" - ], - "step_columns": [ - "root_cause_step" - ], - "step_tolerance": 1, - "build_missing_report": true, - "top_error_examples": 20, - "notes": "AgentRx / magentic, gold_scope='root_cause'. Spans keyed by the native 1-based step index (== gold step_number). non_llm_validators are not used. Step Top-1 uses first_idx_mode='top_ranked' (top_ranked = the model's #1-ranked span), comparable to the paper's single-root-cause Step Acc; step_top1_pm1/pm3/pm5 add ±1/±3/±5 tolerance to match Table 6's Acc@±k." -} -``` - -## Paper comparison — Table 6 (root-cause Step Acc, top-1) - -Top-1 span (first_idx_mode=top_ranked) vs gold at ±0/±1/±3/±5. The paper reports a single predicted root-cause step, so `gold_scope=root_cause` is the closest analog. Their AGENTRX is a trained method; ours is an untrained LLM judge — treat this as orientation, not parity. - -| Source | Step Acc | Acc@±1 | Acc@±3 | Acc@±5 | -|---|---|---|---|---| -| Ours (magentic / root_cause) | 15.9% | 20.5% | 31.8% | 34.1% | -| Paper AGENTRX — Magentic-One | 31.8% | 40.9% | 47.7% | 50.8% | -| Paper AGENTRX — Magentic-One* | 46.9% | 61.7% | 72.8% | 79.0% | diff --git a/examples/agentrx/reports/agentrx_tau_all.md b/examples/agentrx/reports/agentrx_tau_all.md deleted file mode 100644 index 0b8b446..0000000 --- a/examples/agentrx/reports/agentrx_tau_all.md +++ /dev/null @@ -1,113 +0,0 @@ -# AgentRx tau — gold=all - -Generated: `2026-07-07 19:33:26` - -## Notes - -AgentRx / tau, gold_scope='all'. Spans keyed by the native 1-based step index (== gold step_number). non_llm_validators are not used. Step Top-1 uses first_idx_mode='top_ranked' (top_ranked = the model's #1-ranked span), comparable to the paper's single-root-cause Step Acc; step_top1_pm1/pm3/pm5 add ±1/±3/±5 tolerance to match Table 6's Acc@±k. - -## Experiment setup - -| Field | Value | -|---|---| -| Experiment | AgentRx tau — gold=all | -| Model / judge | google/gemini-2.5-flash | -| Dataset | AgentRx / tau-bench retail | -| Run label | AgentRx findings (all gold) | -| Prediction glob | /mnt/c/Users/barak/MASeval/examples/agentrx/agentrx_tau_findings | -| Prediction files | 29 | -| HF annotations | microsoft/AgentRx :: tau (all) | -| Annotation rows | 29 | -| Matched annotations | 29 | -| Unmatched predictions | 0 | -| ID column | trajectory_id | -| Agent annotation columns | mistake_agents | -| Step annotation columns | mistake_steps | -| Step tolerance | ±1 | -| Build missing report | True | -| Verifier mode (ablation) | soft (stored) | -| EvidenceVerifier policy | verified/weak are counted; invalid findings are excluded and left for review | - -## Main results - -| Group | Metric | Value | Count | Meaning | -|---|---:|---:|---:|---| -| Agent | Top-1 Acc | 48.3% | 29 | Primary culprit agent matches the annotation. | -| Agent | Hit Acc | 48.3% | 29 | At least one predicted culprit agent matches the annotation. | -| Agent | Exact Set Acc | 0.0% | 29 | Predicted agent set exactly equals the gold agent set. | -| Step | Top-1 Acc | 6.9% | 29 | First predicted problematic idx equals the gold idx. | -| Step | Hit Acc | 44.8% | 29 | At least one predicted idx exactly matches a gold idx. | -| Step | Hit Acc ±1 | 44.8% | 29 | At least one predicted numeric idx is within ±1 of a gold idx. | - -## Diagnostic quality summary - -| Field | Value | -|---|---:| -| Invalid findings excluded from main predictions | 3 | -| Agent examples | 29 | -| Step examples | 29 | - -## Interpretation - -- Agent localization: Hit Acc is 48.3%, Top-1 Acc is 48.3% (gap 0.0%). A large gap means the correct agent is often present but not ranked first. -- Step localization: exact Hit Acc is 44.8%, relaxed Hit Acc ±1 is 44.8% (gap 0.0%). A large gap suggests off-by-one or indexing mismatch. -- First-idx ranking: Step Top-1 Acc is 6.9%. If this is much lower than Step Hit Acc, use the full problematic_idxs list rather than only first_problem_idx. - -## Example mismatches / review targets (top 19) - -| # | File | Gold agents | Predicted agents | Agent hit | Gold idxs | Predicted idxs | Step hit ±1 | Invalid findings | -|---:|---|---|---|---:|---|---|---:|---:| -| 1 | findings_22.json | User | | ❌ | 20 | | ❌ | 1 | -| 2 | findings_0.json | Assistant | | ❌ | 3, 7 | | ❌ | 0 | -| 3 | findings_1.json | Assistant | | ❌ | 3, 7 | | ❌ | 0 | -| 4 | findings_11.json | Assistant | | ❌ | 17 | | ❌ | 0 | -| 5 | findings_13.json | Assistant | | ❌ | 19 | | ❌ | 0 | -| 6 | findings_14.json | User | | ❌ | 32 | | ❌ | 0 | -| 7 | findings_15.json | User | | ❌ | 14 | | ❌ | 0 | -| 8 | findings_2.json | Assistant | | ❌ | 15 | | ❌ | 0 | -| 9 | findings_20.json | User | | ❌ | 20 | | ❌ | 0 | -| 10 | findings_21.json | Assistant | | ❌ | 26 | | ❌ | 0 | -| 11 | findings_23.json | Assistant | | ❌ | 37 | | ❌ | 0 | -| 12 | findings_24.json | User | assistant | ❌ | 34 | 21, 22, user, 2, 29 | ❌ | 0 | -| 13 | findings_27.json | Assistant | | ❌ | 57 | | ❌ | 0 | -| 14 | findings_8.json | Assistant | | ❌ | 11 | | ❌ | 0 | -| 15 | findings_10.json | Assistant | assistant, Retail agent | ✅ | 37 | 21, 15, 1, 19, 22 | ❌ | 0 | -| 16 | findings_26.json | Assistant | assistant, Retail agent | ✅ | 35 | 45, 42, 43, 26, 41, 44 | ❌ | 0 | -| 17 | findings_16.json | User | assistant, Retail agent policy, Retail agent | ❌ | 32 | 23, 33, 1, 18, 32 | ✅ | 0 | -| 18 | findings_3.json | Assistant | assistant, Retail agent | ✅ | 19 | 23, 22, 19, 1, 12, 15 | ✅ | 1 | -| 19 | findings_7.json | Assistant | assistant, Retail agent | ✅ | 31 | 26, 32, 31, 33, 1, 28, … (+1) | ✅ | 1 | - -## Reproducibility - -```json -{ - "experiment_name": "AgentRx tau — gold=all", - "model_name": "google/gemini-2.5-flash", - "dataset_name": "AgentRx / tau-bench retail", - "run_label": "AgentRx findings (all gold)", - "pred_glob": "/mnt/c/Users/barak/MASeval/examples/agentrx/agentrx_tau_findings", - "hf_annotations": "microsoft/AgentRx :: tau (all)", - "output_json_path": "/mnt/c/Users/barak/MASeval/examples/agentrx/reports/agentrx_tau_all.json", - "output_md_path": "/mnt/c/Users/barak/MASeval/examples/agentrx/reports/agentrx_tau_all.md", - "id_column": "trajectory_id", - "agent_columns": [ - "mistake_agents" - ], - "step_columns": [ - "mistake_steps" - ], - "step_tolerance": 1, - "build_missing_report": true, - "top_error_examples": 20, - "notes": "AgentRx / tau, gold_scope='all'. Spans keyed by the native 1-based step index (== gold step_number). non_llm_validators are not used. Step Top-1 uses first_idx_mode='top_ranked' (top_ranked = the model's #1-ranked span), comparable to the paper's single-root-cause Step Acc; step_top1_pm1/pm3/pm5 add ±1/±3/±5 tolerance to match Table 6's Acc@±k." -} -``` - -## Paper comparison — Table 6 (root-cause Step Acc, top-1) - -Top-1 span (first_idx_mode=top_ranked) vs gold at ±0/±1/±3/±5. The paper reports a single predicted root-cause step, so `gold_scope=root_cause` is the closest analog. Their AGENTRX is a trained method; ours is an untrained LLM judge — treat this as orientation, not parity. - -| Source | Step Acc | Acc@±1 | Acc@±3 | Acc@±5 | -|---|---|---|---|---| -| Ours (tau / all) | 6.9% | 17.2% | 27.6% | 34.5% | -| Paper AGENTRX — τ-Bench | 54.0% | 59.8% | 72.4% | 83.9% | diff --git a/examples/agentrx/reports/agentrx_tau_root_cause.md b/examples/agentrx/reports/agentrx_tau_root_cause.md deleted file mode 100644 index 02708ae..0000000 --- a/examples/agentrx/reports/agentrx_tau_root_cause.md +++ /dev/null @@ -1,114 +0,0 @@ -# AgentRx tau — gold=root_cause - -Generated: `2026-07-07 19:34:57` - -## Notes - -AgentRx / tau, gold_scope='root_cause'. Spans keyed by the native 1-based step index (== gold step_number). non_llm_validators are not used. Step Top-1 uses first_idx_mode='top_ranked' (top_ranked = the model's #1-ranked span), comparable to the paper's single-root-cause Step Acc; step_top1_pm1/pm3/pm5 add ±1/±3/±5 tolerance to match Table 6's Acc@±k. - -## Experiment setup - -| Field | Value | -|---|---| -| Experiment | AgentRx tau — gold=root_cause | -| Model / judge | google/gemini-2.5-flash | -| Dataset | AgentRx / tau-bench retail | -| Run label | AgentRx findings (root_cause gold) | -| Prediction glob | /mnt/c/Users/barak/MASeval/examples/agentrx/agentrx_tau_findings | -| Prediction files | 29 | -| HF annotations | microsoft/AgentRx :: tau (root_cause) | -| Annotation rows | 29 | -| Matched annotations | 29 | -| Unmatched predictions | 0 | -| ID column | trajectory_id | -| Agent annotation columns | root_cause_agent | -| Step annotation columns | root_cause_step | -| Step tolerance | ±1 | -| Build missing report | True | -| Verifier mode (ablation) | soft (stored) | -| EvidenceVerifier policy | verified/weak are counted; invalid findings are excluded and left for review | - -## Main results - -| Group | Metric | Value | Count | Meaning | -|---|---:|---:|---:|---| -| Agent | Top-1 Acc | 41.4% | 29 | Primary culprit agent matches the annotation. | -| Agent | Hit Acc | 41.4% | 29 | At least one predicted culprit agent matches the annotation. | -| Agent | Exact Set Acc | 0.0% | 29 | Predicted agent set exactly equals the gold agent set. | -| Step | Top-1 Acc | 0.0% | 29 | First predicted problematic idx equals the gold idx. | -| Step | Hit Acc | 34.5% | 29 | At least one predicted idx exactly matches a gold idx. | -| Step | Hit Acc ±1 | 34.5% | 29 | At least one predicted numeric idx is within ±1 of a gold idx. | - -## Diagnostic quality summary - -| Field | Value | -|---|---:| -| Invalid findings excluded from main predictions | 3 | -| Agent examples | 29 | -| Step examples | 29 | - -## Interpretation - -- Agent localization: Hit Acc is 41.4%, Top-1 Acc is 41.4% (gap 0.0%). A large gap means the correct agent is often present but not ranked first. -- Step localization: exact Hit Acc is 34.5%, relaxed Hit Acc ±1 is 34.5% (gap 0.0%). A large gap suggests off-by-one or indexing mismatch. -- First-idx ranking: Step Top-1 Acc is 0.0%. If this is much lower than Step Hit Acc, use the full problematic_idxs list rather than only first_problem_idx. - -## Example mismatches / review targets (top 20) - -| # | File | Gold agents | Predicted agents | Agent hit | Gold idxs | Predicted idxs | Step hit ±1 | Invalid findings | -|---:|---|---|---|---:|---|---|---:|---:| -| 1 | findings_22.json | User | | ❌ | 20 | | ❌ | 1 | -| 2 | findings_0.json | Assistant | | ❌ | 7 | | ❌ | 0 | -| 3 | findings_1.json | Assistant | | ❌ | 7 | | ❌ | 0 | -| 4 | findings_11.json | Assistant | | ❌ | 17 | | ❌ | 0 | -| 5 | findings_13.json | Assistant | | ❌ | 19 | | ❌ | 0 | -| 6 | findings_14.json | User | | ❌ | 32 | | ❌ | 0 | -| 7 | findings_15.json | User | | ❌ | 14 | | ❌ | 0 | -| 8 | findings_17.json | User | assistant, Retail agent | ❌ | 24 | 35, 27, 1, 30, 18, 31 | ❌ | 0 | -| 9 | findings_2.json | Assistant | | ❌ | 15 | | ❌ | 0 | -| 10 | findings_20.json | User | | ❌ | 20 | | ❌ | 0 | -| 11 | findings_21.json | Assistant | | ❌ | 26 | | ❌ | 0 | -| 12 | findings_23.json | Assistant | | ❌ | 37 | | ❌ | 0 | -| 13 | findings_24.json | User | assistant | ❌ | 34 | 21, 22, user, 2, 29 | ❌ | 0 | -| 14 | findings_27.json | Assistant | | ❌ | 57 | | ❌ | 0 | -| 15 | findings_5.json | User | assistant, Retail agent | ❌ | 28 | 41, 1, 44, 40, 39, 43, … (+1) | ❌ | 0 | -| 16 | findings_8.json | Assistant | | ❌ | 11 | | ❌ | 0 | -| 17 | findings_10.json | Assistant | assistant, Retail agent | ✅ | 37 | 21, 15, 1, 19, 22 | ❌ | 0 | -| 18 | findings_26.json | Assistant | assistant, Retail agent | ✅ | 35 | 45, 42, 43, 26, 41, 44 | ❌ | 0 | -| 19 | findings_4.json | Assistant | assistant, Retail agent | ✅ | 21 | 61, 50, 49, 62, 1, 55, … (+4) | ❌ | 0 | -| 20 | findings_16.json | User | assistant, Retail agent policy, Retail agent | ❌ | 32 | 23, 33, 1, 18, 32 | ✅ | 0 | - -## Reproducibility - -```json -{ - "experiment_name": "AgentRx tau — gold=root_cause", - "model_name": "google/gemini-2.5-flash", - "dataset_name": "AgentRx / tau-bench retail", - "run_label": "AgentRx findings (root_cause gold)", - "pred_glob": "/mnt/c/Users/barak/MASeval/examples/agentrx/agentrx_tau_findings", - "hf_annotations": "microsoft/AgentRx :: tau (root_cause)", - "output_json_path": "/mnt/c/Users/barak/MASeval/examples/agentrx/reports/agentrx_tau_root_cause.json", - "output_md_path": "/mnt/c/Users/barak/MASeval/examples/agentrx/reports/agentrx_tau_root_cause.md", - "id_column": "trajectory_id", - "agent_columns": [ - "root_cause_agent" - ], - "step_columns": [ - "root_cause_step" - ], - "step_tolerance": 1, - "build_missing_report": true, - "top_error_examples": 20, - "notes": "AgentRx / tau, gold_scope='root_cause'. Spans keyed by the native 1-based step index (== gold step_number). non_llm_validators are not used. Step Top-1 uses first_idx_mode='top_ranked' (top_ranked = the model's #1-ranked span), comparable to the paper's single-root-cause Step Acc; step_top1_pm1/pm3/pm5 add ±1/±3/±5 tolerance to match Table 6's Acc@±k." -} -``` - -## Paper comparison — Table 6 (root-cause Step Acc, top-1) - -Top-1 span (first_idx_mode=top_ranked) vs gold at ±0/±1/±3/±5. The paper reports a single predicted root-cause step, so `gold_scope=root_cause` is the closest analog. Their AGENTRX is a trained method; ours is an untrained LLM judge — treat this as orientation, not parity. - -| Source | Step Acc | Acc@±1 | Acc@±3 | Acc@±5 | -|---|---|---|---|---| -| Ours (tau / root_cause) | 0.0% | 10.3% | 20.7% | 27.6% | -| Paper AGENTRX — τ-Bench | 54.0% | 59.8% | 72.4% | 83.9% | diff --git a/examples/trace_elephant/README.md b/examples/trace_elephant/README.md new file mode 100644 index 0000000..b78ba10 --- /dev/null +++ b/examples/trace_elephant/README.md @@ -0,0 +1,72 @@ +# TraceElephant — running the MASeval judge on their benchmark + +Runs the MASeval LLM findings pipeline on the **TraceElephant** failure-attribution +benchmark ([HF dataset](https://huggingface.co/datasets/TraceElephant/TraceElephant), +[paper](https://arxiv.org/abs/2604.22708), *"Seeing the Whole Elephant"*, ACL 2026). + +This mirrors the `agentrx` example: our multi-evaluator judge + `EvidenceVerifier` ++ report aggregation produce a per-trace agent/step localization, which we score +against TraceElephant's gold `mistake_agent` / `mistake_step`. + +## Benchmark + +220 annotated **failure** traces (of 380 total executions) from three multi-agent +systems, each fully observable (agent inputs *and* outputs, inter-agent messages, +tool calls): + +| System | Runs dirs | Traces | +|---|---|--:| +| Captain-Agent | `captain-runs-{assistantbench,gaia}` | 85 | +| Magentic-One | `magentic-runs-{assistant-bench,gaia}` | 91 | +| SWE-Agent | `swe-agent-runs-swe-bench` | 44 | + +Each trace is annotated with one decisive failure: the **responsible agent** and +the **decisive step** (earliest inevitable error, 1-based over the history). + +## Data + +The dataset ships as a single `data.zip` on HuggingFace. Download + extract into +`./data` (git-ignored): + +```python +from huggingface_hub import hf_hub_download +import zipfile +z = hf_hub_download("TraceElephant/TraceElephant", "data.zip", repo_type="dataset") +zipfile.ZipFile(z).extractall("examples/trace_elephant") # -> examples/trace_elephant/data/ +``` + +`trace_elephant_data.py` walks `data/{system}-runs-*/{task}/` and normalizes both +on-disk shapes (`trace_metadata.json`+`step_records.json`, or `summary.json`+ +`history.json`) into a `history` of `{name, content}` steps. The gold `mistake_step` +is the 1-based history position, so spans are keyed by that index (evaluators cite +it in `evidence[i].idx`). + +## Run + +```bash +# 1) generate findings (11 LLM evaluators + EvidenceVerifier + report + FinalAnswerVerifier) +python launch_findings_judges.py --system all --model google/gemini-2.5-flash +# or per system: --system captain | magentic | swe + +# 2) score agent/step localization vs TraceElephant gold +python calculate_agent_step_accuracy.py --system all --step-tolerance 1 +``` + +`FinalAnswerVerifier` runs in **no-ground-truth mode**: with `gt=None` it routes to +the MAS Task Completion judge (assesses completion from the trace alone) and feeds +the report's `answer_status`. TraceElephant has no gold final answer for the judge, +so this is the right mode — same as `agentrx`. + +## Metric + +Who&When-parity single-target scoring (TraceElephant gold is one agent + one step): + +- **Agent Top-1** — the report's primary culprit is the gold `mistake_agent` + (names matched via the repo's `_normalize_agent` convention). +- **Step Top-1** (`first_idx_mode=top_ranked`) — the top-ranked predicted span is + the gold `mistake_step`; `--step-tolerance k` allows ±k. +- **Agent/Step Hit** — gold appears anywhere in the predicted set (reference). + +> Note: TraceElephant's own `evaluate.py` uses a lenient `str(gold_step) in +> predicted_step` membership check, not exact/±k — keep our numbers in their own +> column rather than lining them up against the Who&When ±k tables. diff --git a/examples/trace_elephant/calculate_agent_step_accuracy.py b/examples/trace_elephant/calculate_agent_step_accuracy.py new file mode 100644 index 0000000..13b0e08 --- /dev/null +++ b/examples/trace_elephant/calculate_agent_step_accuracy.py @@ -0,0 +1,151 @@ +"""Agent / step localization accuracy for TraceElephant findings runs. + +Compares per-trace findings JSON (``report.diagnostic_report``) against the +TraceElephant gold (``mistake_agent`` / ``mistake_step``, one decisive failure +per trace), using the shared ``maseval.diagnostic_accuracy`` scorer. + +TraceElephant gold is a single (agent, step), so this is Who&When-parity +scoring: agent Top-1 = predicted primary is the gold agent; step Top-1 = the +top-ranked predicted span is the gold step (``--step-tolerance`` adds ±k). We +also report Hit (gold in the predicted set) for reference. + +Prediction files (``findings_{i}.json``) match gold row ``i`` by filename index +(and by ``task_name`` when present). ``--system`` must match the launch run's +``--system`` so the gold subset is filtered and re-indexed identically. + +Usage: + python calculate_agent_step_accuracy.py --system all + python calculate_agent_step_accuracy.py --system captain --step-tolerance 1 +""" + +from __future__ import annotations + +import argparse +import json +import sys +import tempfile +from pathlib import Path +from typing import Sequence + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from maseval.diagnostic_accuracy import evaluate_agent_step_accuracy # noqa: E402 + +import trace_elephant_data as ted # noqa: E402 + +SYSTEMS = ("captain", "magentic", "swe") + + +def _subset(data_dir: str, system: str): + """The examples for ``system``, re-indexed exactly as launch_findings_judges does.""" + examples = ted.load_examples(data_dir) + if system != "all": + examples = [e for e in examples if e.system_category == system] + for i, e in enumerate(examples): + e.row_index = i + return examples + + +def _resolve_pred_paths(pred_glob) -> list[Path]: + import glob + + specs = pred_glob if isinstance(pred_glob, (list, tuple)) else [pred_glob] + files: list[Path] = [] + for spec in specs: + p = Path(spec) + if p.is_dir(): + files.extend(sorted(p.glob("*.json"))) + elif any(ch in str(spec) for ch in "*?["): + files.extend(sorted(Path(x) for x in glob.glob(str(spec)))) + elif p.exists(): + files.append(p) + return files + + +def main( + system: str = "all", + pred_glob: str | Path | Sequence[str | Path] | None = None, + *, + data_dir: str | None = None, + step_tolerance: int = 1, + verifier_mode: str | None = None, + first_idx_mode: str = "top_ranked", + output_path: str | Path | None = None, + print_summary: bool = True, +) -> dict: + data_dir = data_dir or str(THIS_DIR / "data") + if pred_glob is None: + pred_glob = THIS_DIR / f"trace_elephant_{system}_findings" + pred_paths = _resolve_pred_paths(pred_glob) + if not pred_paths: + raise FileNotFoundError(f"No prediction JSON files matched: {pred_glob!r}") + + examples = _subset(data_dir, system) + + with tempfile.TemporaryDirectory(prefix="trace_elephant_gold_") as tmp: + gold_path = ted.write_gold_jsonl(examples, str(Path(tmp) / "gold.jsonl")) + result = evaluate_agent_step_accuracy( + pred_paths, + gold_path, + agent_columns=["mistake_agents"], + step_columns=["mistake_steps"], + step_tolerance=step_tolerance, + verifier_mode=verifier_mode, + first_idx_mode=first_idx_mode, + ) + + result["run_config"] = { + "system": system, + "pred_glob": str(pred_glob), + "prediction_files_count": len(pred_paths), + "step_tolerance": step_tolerance, + "verifier_mode": verifier_mode, + "first_idx_mode": first_idx_mode, + } + result["annotation_source"] = { + "type": "trace_elephant", + "system": system, + "rows": len(examples), + } + + if output_path is not None: + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + Path(output_path).write_text( + json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8" + ) + if print_summary: + print(json.dumps(result["summary"], ensure_ascii=False, indent=2)) + return result + + +calculate_agent_step_accuracy = main + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="TraceElephant agent/step accuracy.") + parser.add_argument("--system", choices=("all", *SYSTEMS), default="all") + parser.add_argument("--pred-glob", default=None) + parser.add_argument("--data-dir", default=None) + parser.add_argument("--step-tolerance", type=int, default=1) + parser.add_argument("--verifier-mode", choices=("none", "strict", "soft"), default=None) + parser.add_argument( + "--first-idx-mode", + choices=("top_ranked", "min_index"), + default="top_ranked", + help="How the top-1 predicted span is chosen (default top_ranked).", + ) + parser.add_argument("--output-json", default=None) + args = parser.parse_args() + main( + system=args.system, + pred_glob=args.pred_glob, + data_dir=args.data_dir, + step_tolerance=args.step_tolerance, + verifier_mode=args.verifier_mode, + first_idx_mode=args.first_idx_mode, + output_path=args.output_json, + ) diff --git a/examples/trace_elephant/gen_validators_confirm.py b/examples/trace_elephant/gen_validators_confirm.py new file mode 100644 index 0000000..61e3768 --- /dev/null +++ b/examples/trace_elephant/gen_validators_confirm.py @@ -0,0 +1,109 @@ +"""Run the deterministic validators + LLM confirmer on TraceElephant traces and +attach them to the existing judge findings (the launcher deliberately skipped the +validators; see note below), writing to ``trace_elephant_{system}_valconfirm/``. + +Why the launcher skipped them: TraceElephant's judge/gold idx space is 1-based +(``format_trace`` uses ``enumerate(..., start=1)``) while ``who_and_when_to_spans`` +(the validators' span builder) is 0-based. That 1-vs-0 offset is the whole +"misaligned spans" reason — it is a narrow bookkeeping issue, not a deeper +incompatibility. We run the validators here and let the *scorer* shift validator +idxs into the 1-based space; the raw findings are stored in their native 0-based +form, exactly as ``run_on_trace`` emits them. + +Reuses the stored 11-judge findings untouched; only adds ``non_llm_validators`` +(regex, free) + ``llm_confirmation`` (~1 call per finding-bearing trace, full +reading view). Resumable: existing output files are skipped. + +Usage: + python gen_validators_confirm.py # all systems + python gen_validators_confirm.py --system swe +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from collections import defaultdict +from pathlib import Path + +from dotenv import load_dotenv + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +load_dotenv(ROOT / ".env") +load_dotenv(THIS_DIR / ".env") + +from maseval.validators import run_on_trace # noqa: E402 +from maseval.validators.llm_confirm import confirm_trace_async # noqa: E402 + +import trace_elephant_data as ted # noqa: E402 + +SYSTEMS = ("captain", "magentic", "swe") + + +def _n(nlv: dict) -> int: + return sum(len(m.get("findings", [])) for m in (nlv.get("metrics") or {}).values()) + + +async def main(system: str, model: str, data_dir: str | None = None, + confirm: bool = True) -> None: + if confirm and not os.getenv("OPENROUTER_API_KEY"): + raise RuntimeError("OPENROUTER_API_KEY is not set (source the root .env).") + + data_dir = data_dir or str(THIS_DIR / "data") + examples = ted.load_examples(data_dir) + bysys: dict[str, list] = defaultdict(list) + for e in examples: + bysys[e.system_category].append(e) + systems = SYSTEMS if system == "all" else (system,) + + for sysname in systems: + exs = bysys.get(sysname, []) + for i, e in enumerate(exs): # per-system reindex, matches the judge folder + e.row_index = i + src = THIS_DIR / f"trace_elephant_{sysname}_findings" + out = THIS_DIR / f"trace_elephant_{sysname}_valconfirm" + out.mkdir(parents=True, exist_ok=True) + totals = {"confirmed": 0, "benign": 0, "uncertain": 0, "appointed": 0, "bearing": 0} + print(f"\n=== {sysname}: {len(exs)} traces -> {out.name} ===") + for e in exs: + out_f = out / f"findings_{e.row_index}.json" + if out_f.exists(): + continue + src_f = src / f"findings_{e.row_index}.json" + data = json.loads(src_f.read_text(encoding="utf-8")) if src_f.exists() else {} + trace = {"history": e.history} + nlv = run_on_trace(trace) + if _n(nlv): + totals["bearing"] += 1 + if confirm: + try: + await confirm_trace_async(trace, nlv, model) # full reading view + except Exception as exc: # noqa: BLE001 + print(f" [{e.row_index}] confirm error: {type(exc).__name__}: {exc}") + s = nlv.get("llm_confirmation_summary", {}) + for k in ("confirmed", "benign", "uncertain", "appointed"): + totals[k] += int(s.get(k, 0) or 0) + print(f" [{e.row_index}] {_n(nlv)} det-findings -> {s}") + data["non_llm_validators"] = nlv + out_f.write_text(json.dumps(data, indent=2, default=str), encoding="utf-8") + print(f" {sysname} totals: {totals}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Validators + confirmer on TraceElephant.") + parser.add_argument("--system", choices=("all", *SYSTEMS), default="all") + parser.add_argument("--model", default="google/gemini-2.5-flash") + parser.add_argument("--data-dir", default=None) + parser.add_argument("--no-confirm", action="store_true", + help="Run the free deterministic validators only (no LLM calls).") + args = parser.parse_args() + asyncio.run(main(system=args.system, model=args.model, data_dir=args.data_dir, + confirm=not args.no_confirm)) diff --git a/examples/trace_elephant/launch_findings_judges.py b/examples/trace_elephant/launch_findings_judges.py new file mode 100644 index 0000000..0932a53 --- /dev/null +++ b/examples/trace_elephant/launch_findings_judges.py @@ -0,0 +1,272 @@ +"""Run the LLM findings-evaluators on TraceElephant failure traces. + +Mirrors the AgentRx findings runner, but for TraceElephant: + * spans are keyed by the 1-based step position (== gold ``mistake_step``); + * FinalAnswerVerifier runs in its no-ground-truth mode (the MAS Task + Completion judge) -- with ``gt=None`` it never touches a ``df`` extractor, so + ``df="trace_elephant"`` is fine and no new extractor is needed; + * no deterministic ``non_llm_validators`` (TraceElephant is not a validator + format; running them would emit misaligned spans). + +Each trace is evaluated by every LLM metric; raw findings + EvidenceVerifier +output + a compact ``report`` are written to one per-trace JSON file. + +Usage: + python launch_findings_judges.py --model google/gemini-2.5-flash + python launch_findings_judges.py --system captain --from-idx 0 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +import time +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv(".env") + +# Disable the default OTEL tracer provider BEFORE importing maseval. +os.environ["OTEL_TRACES_EXPORTER"] = "none" +from opentelemetry import trace # noqa: E402 + +trace.set_tracer_provider(None) + +from pydantic_ai.models.openai import OpenAIChatModel # noqa: E402 + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from maseval.evaluation_blocks.final_answer_verification import ( # noqa: E402 + FinalAnswerVerificationResult, + FinalAnswerVerifier, +) +from maseval.metrics import EvidenceVerifier, MetricType, create_metric # noqa: E402 +from maseval.models import RawTraceInput # noqa: E402 +from maseval.reporting import build_evaluation_report # noqa: E402 +from maseval.run_stats import aggregate_task_stats, extract_usage # noqa: E402 + +import trace_elephant_data as ted # noqa: E402 + +# Same LLM evaluator set as the Who&When / AgentRx findings runs. +LLM_METRICS_TO_TEST = [ + MetricType.OBSERVATION_ALIGNMENT, + MetricType.POLICY_ALIGNMENT, + MetricType.STATE_CONSISTENCY, + MetricType.TOOL_SELECTION, + MetricType.TOOL_PARAMETER_EXTRACTION, + MetricType.MAS_PLANNING, + MetricType.MAS_COMPLEXITY, + MetricType.MAS_TASK_TRANSFER, + MetricType.MAS_ROLES_DISTRIBUTION, + MetricType.TOOL_PERFORMANCE, + MetricType.PROMPT_QUALITY, +] + +METRIC_TIMEOUT = float(os.environ.get("METRIC_TIMEOUT", "600")) +MAX_OUTPUT_TOKENS = int(os.environ.get("MAX_OUTPUT_TOKENS", "4096")) +# Retries per evaluator. OpenRouter returns finish_reason='error' when the +# upstream (gemini) call fails transiently under load; pydantic_ai then raises +# (surfacing as UnexpectedModelBehavior / an IndexError in maseval's error path). +# These are retryable, so give each evaluator a few attempts with backoff. +METRIC_MAX_ATTEMPTS = int(os.environ.get("METRIC_MAX_ATTEMPTS", "4")) + +SYSTEMS = ("captain", "magentic", "swe") + + +async def _run_all_metrics(model, eval_input: RawTraceInput) -> tuple[dict, dict]: + """Run every LLM evaluator on one trace. + + Returns ``(findings_results, metric_status)`` where ``metric_status`` records + per-metric timing + token usage for the ``task_stats`` rollup. + """ + findings_results: dict = {} + metric_status: dict = {} + for metric_type in LLM_METRICS_TO_TEST: + name = metric_type.value + print(f" --- {name} ---") + metric = None + t0 = time.perf_counter() + for attempt in range(1, METRIC_MAX_ATTEMPTS + 1): + try: + metric = create_metric(metric_type, model) + result = await asyncio.wait_for( + metric.evaluate(eval_input), timeout=METRIC_TIMEOUT + ) + inp, out, tot = extract_usage(getattr(metric, "last_usage", None)) + findings_results[name] = result + metric_status[name] = { + "status": "ok", "duration_s": round(time.perf_counter() - t0, 3), + "input_tokens": inp, "output_tokens": out, "total_tokens": tot, + } + print(f" findings: {len(result.findings)} | in={inp} out={out}") + break + except Exception as exc: # noqa: BLE001 - isolate one metric's failure + if attempt < METRIC_MAX_ATTEMPTS: + await asyncio.sleep(2 * attempt) + print(f" retry {attempt}/{METRIC_MAX_ATTEMPTS - 1} after {type(exc).__name__}") + else: + inp, out, tot = extract_usage(getattr(metric, "last_usage", None)) + metric_status[name] = { + "status": "failed", "detail": f"{type(exc).__name__}: {exc}"[:300], + "duration_s": round(time.perf_counter() - t0, 3), + "input_tokens": inp, "output_tokens": out, "total_tokens": tot, + } + print(f" error: {type(exc).__name__}: {exc}") + return findings_results, metric_status + + +def _serialize_findings(findings_results: dict) -> dict: + return { + metric_name: { + "metric_name": result.metric_name, + "findings": [f.model_dump(mode="json") for f in result.findings], + } + for metric_name, result in findings_results.items() + } + + +def _serialize_evidence_verification(evidence_results: dict) -> dict: + return { + metric_name: verification_result.model_dump(mode="json") + for metric_name, verification_result in evidence_results.items() + } + + +async def _run_final_answer_verification( + final_answer_verifier: FinalAnswerVerifier, ex +) -> FinalAnswerVerificationResult | None: + """Run FinalAnswerVerifier in no-ground-truth mode for one trace. + + With ``gt=None`` the verifier routes to the MAS Task Completion judge, which + assesses whether the task was completed from the trace alone (no gold answer + needed). ``df`` only needs to be non-None here. The trace is passed + structured (instruction + steps) so the judge knows what "complete" means. + """ + # Cap the history the MTC judge sees so oversized SWE traces don't blow the + # model context (the judge only needs enough to assess task completion). + n = max(1, len(ex.history)) + cap = max(ted._MIN_STEP_CHARS, ted.TRACE_CHAR_BUDGET // n) + steps = [ + {"name": s["name"], + "content": s["content"] if len(s["content"]) <= cap else s["content"][:cap] + "…[truncated]"} + for s in ex.history + ] + try: + return await final_answer_verifier.verify_final_answer( + {"instruction": ex.question, "steps": steps}, + gt=None, + df="trace_elephant", + ) + except Exception as exc: # noqa: BLE001 - one judge failure must not abort a trace + print(f" final_answer_verification error: {type(exc).__name__}: {exc}") + return None + + +async def _evaluate_example( + model, ex, folder: Path, result_file_name: str, final_answer_verifier +) -> None: + """Evaluate one TraceElephant trace and write its per-trace JSON.""" + eval_input = RawTraceInput(trace=ted.format_trace(ex)) + _t0 = time.perf_counter() + findings_results, metric_status = await _run_all_metrics(model, eval_input) + task_stats = aggregate_task_stats(metric_status, time.perf_counter() - _t0) + evidence_results = EvidenceVerifier().verify_all(findings_results, eval_input) + final_answer_result = await _run_final_answer_verification(final_answer_verifier, ex) + + payload = _serialize_findings(findings_results) + payload["task_stats"] = task_stats + # Top-level id so the scorer can match by task_name as well as by filename index. + payload["task_name"] = ex.task_name + payload["evidence_verification"] = _serialize_evidence_verification(evidence_results) + # No-GT MAS task-completion verdict; feeds the report's answer_status. + if final_answer_result is not None: + payload["final_answer_verification"] = final_answer_result.model_dump(mode="json") + # TraceElephant gold (for inspection; scoring uses the gold table). + payload["trace_elephant_meta"] = { + "task_name": ex.task_name, + "system_category": ex.system_category, + "system_name": ex.system_name, + "num_steps": len(ex.history), + "mistake_agent": ex.mistake_agent, + "mistake_step": ex.mistake_step, + } + # TraceElephant has no gold final answer for the judge path -> answer_status + # comes from the no-GT task-completion verdict. + payload["report"] = build_evaluation_report(payload, reference_answer=None) + + folder.mkdir(parents=True, exist_ok=True) + out_file = folder / f"{result_file_name}{ex.row_index}.json" + out_file.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") + print(f" saved -> {out_file}") + + +async def main( + model_name: str, + system: str = "all", + data_dir: str | None = None, + folder_name: str | None = None, + result_file_name: str = "findings_", + from_idx: int = 0, + resume: bool = True, +) -> None: + api_key = os.getenv("OPENROUTER_API_KEY") + if not api_key: + raise RuntimeError("OPENROUTER_API_KEY is not set (check examples/trace_elephant/.env).") + os.environ["OPENROUTER_API_KEY"] = api_key + + model = OpenAIChatModel( + model_name, + provider="openrouter", + settings={"temperature": 0.0, "max_tokens": MAX_OUTPUT_TOKENS}, + ) + final_answer_verifier = FinalAnswerVerifier(model=model_name) + + data_dir = data_dir or str(THIS_DIR / "data") + examples = ted.load_examples(data_dir) + if system != "all": + examples = [e for e in examples if e.system_category == system] + # Re-index the filtered subset so row_index is contiguous per run. + for i, e in enumerate(examples): + e.row_index = i + folder = THIS_DIR / (folder_name or f"trace_elephant_{system}_findings") + print(f"TraceElephant/{system}: {len(examples)} traces -> {folder}") + + for ex in examples: + if ex.row_index < from_idx: + continue + out_file = folder / f"{result_file_name}{ex.row_index}.json" + if resume and out_file.exists(): + print(f"[{ex.row_index}] exists, skipping") + continue + print(f"\n=== [{ex.row_index}] {ex.task_name} ({len(ex.history)} steps) ===") + await _evaluate_example(model, ex, folder, result_file_name, final_answer_verifier) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run LLM findings judges on TraceElephant.") + parser.add_argument("--system", choices=("all", *SYSTEMS), default="all") + parser.add_argument("--model", default="google/gemini-2.5-flash") + parser.add_argument("--data-dir", default=None, help="Path to extracted data/ dir.") + parser.add_argument("--folder", default=None, help="Output subfolder name.") + parser.add_argument("--from-idx", type=int, default=0) + parser.add_argument("--no-resume", action="store_true", help="Re-run existing files.") + args = parser.parse_args() + + asyncio.run( + main( + model_name=args.model, + system=args.system, + data_dir=args.data_dir, + folder_name=args.folder, + from_idx=args.from_idx, + resume=not args.no_resume, + ) + ) diff --git a/examples/trace_elephant/score_none_plus_validators.py b/examples/trace_elephant/score_none_plus_validators.py new file mode 100644 index 0000000..d15f038 --- /dev/null +++ b/examples/trace_elephant/score_none_plus_validators.py @@ -0,0 +1,339 @@ +"""Score TraceElephant agent/step localization for verifier=``none``, folding in +the deterministic ``non_llm_validators`` (+ LLM confirmation) — the TraceElephant +analogue of Who&When's ``score_none_plus_validators.py``. + +Baseline is the verifier=``none`` prediction over the 11 LLM evaluators. On top we +OR-in the deterministic validator findings under three verdict filters: + +* ``all`` -- every regex-detected finding (no confirmer gating); +* ``conf+unc`` -- confirmer verdict confirmed OR uncertain; +* ``confirmed`` -- confirmed only (the honest column — captain fires on 100% of + traces, so ``all`` is a recall ceiling artifact, not a result). + +**Index spaces.** TraceElephant's judges *and* gold are 1-based +(``format_trace`` uses ``enumerate(..., start=1)``); only the validators +(``who_and_when_to_spans``) are 0-based. That 0-vs-1 offset was the entire reason +the launcher skipped these validators — a narrow bookkeeping issue, not a deeper +incompatibility. We shift every validator locus (surface evidence idx **and** +appointed ``corrected_idx``) by **+1** here, so baseline, gold, judges and +validators all live in one 1-based space. + +**Agent namespace.** The fold-in agent is always the step *label* at the (1-based) +validator locus — a sub-agent for captain/magentic, the invoked tool for swe. +For the baseline, captain/magentic use the judge-named ``problematic_agents``; +swe uses the tool at the judge locus (its gold ``mistake_agent`` is a tool, so +the judge names never match — same fix as ``score_swe_tool_attribution.py``). + +**Locus variants.** Reported twice: ``appointed`` (confirmer ``corrected_idx``, +fallback surface) and ``surface`` (raw evidence idx). On TraceElephant the +validators fire *before* the (later) gold step and appointing moves *earlier*, so +which locus wins is data-dependent — we don't assume appointed wins as it did on +Who&When. + +Usage: + python score_none_plus_validators.py # all systems + overall + python score_none_plus_validators.py --system swe +""" + +from __future__ import annotations + +import argparse +import glob +import json +import sys +from pathlib import Path + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from maseval.diagnostic_accuracy import ( # noqa: E402 + _idx_matches_any, + _normalize_agent, + read_prediction_file, +) + +import calculate_agent_step_accuracy as C # noqa: E402 + +SYSTEMS = ("captain", "magentic", "swe") +SWE_TOOL_SYSTEMS = ("swe",) # baseline agent derived from the tool at the locus + +FILTERS = ("none_only", "all", "conf_unc", "confirmed") +FILTER_LABELS = { + "none_only": "none (LLM only)", + "all": "none + val (all)", + "conf_unc": "none + val (conf+unc)", + "confirmed": "none + val (confirmed)", +} +METRICS = ("agent_top1", "agent_hit", "step_top1", "step_hit", "step_pm1") + + +def _label_map(example) -> dict[str, str]: + """1-based step idx -> step label (sub-agent for captain/magentic, tool for swe).""" + return {str(k + 1): s["name"] for k, s in enumerate(example.history)} + + +def _iter_val_findings(nlv: dict): + for m in (nlv.get("metrics") or {}).values(): + for f in m.get("findings", []): + yield f + + +def _shift(idx) -> str | None: + """Validator 0-based idx -> 1-based string (None if unparseable).""" + if idx is None: + return None + s = str(idx).strip() + if not s: + return None + try: + return str(int(s) + 1) + except ValueError: + return None + + +def _val_locus_idx(finding: dict, locus_mode: str) -> str | None: + """1-based locus. ``appointed`` = corrected_idx (fallback surface); ``surface`` = evidence idx.""" + conf = finding.get("llm_confirmation") or {} + if locus_mode == "appointed": + c = _shift(conf.get("corrected_idx")) + if c is not None: + return c + ev = finding.get("evidence") or [] + if ev and ev[0].get("idx") is not None: + return _shift(ev[0]["idx"]) + return None + + +def _passes(finding: dict, filt: str) -> bool: + if filt == "all": + return True + verdict = (finding.get("llm_confirmation") or {}).get("verdict") + if filt == "conf_unc": + return verdict in ("confirmed", "uncertain") + if filt == "confirmed": + return verdict == "confirmed" + return False + + +def _extend(agents: list[str], idxs: list[str], a: str | None, ix: str | None): + if ix is not None and ix not in idxs: + idxs.append(ix) + if a and a not in agents: + agents.append(a) + + +def score(system: str, locus_mode: str, data_dir: str | None = None, + pred_glob: str | None = None, step_tolerance: int = 1) -> dict: + data_dir = data_dir or str(THIS_DIR / "data") + examples = C._subset(data_dir, system) + by_row = {e.row_index: e for e in examples} + swe_tool = system in SWE_TOOL_SYSTEMS + + pred_glob = pred_glob or str(THIS_DIR / f"trace_elephant_{system}_valconfirm") + files = sorted( + glob.glob(str(Path(pred_glob) / "*.json")) if Path(pred_glob).is_dir() + else glob.glob(pred_glob), + key=lambda p: int(p.rsplit("_", 1)[1].split(".")[0]), + ) + if not files: + raise FileNotFoundError(f"No prediction files: {pred_glob!r}") + + acc = {f: {k: [] for k in METRICS} for f in FILTERS} + counts = {"bearing": 0, "confirmed": 0, "benign": 0, "uncertain": 0, "appointed": 0} + matched = 0 + + for fp in files: + i = int(fp.rsplit("_", 1)[1].split(".")[0]) + ex = by_row.get(i) + if ex is None: + continue + gold_agents = {_normalize_agent(ex.mistake_agent)} if ex.mistake_agent else set() + gold_idxs = [str(ex.mistake_step)] if ex.mistake_step else [] + if not gold_agents and not gold_idxs: + continue + matched += 1 + name_at = _label_map(ex) + + pred = read_prediction_file(fp, verifier_mode="none") + base_idxs = list(pred.idxs) # 1-based judge idxs + base_first = pred.first_idx + if swe_tool: + base_agents = [name_at[str(ix)] for ix in pred.idxs if name_at.get(str(ix))] + base_primary = name_at.get(str(pred.first_idx)) if pred.first_idx else None + else: + base_agents = list(pred.agents) + base_primary = pred.primary_agent + + data = json.loads(Path(fp).read_text(encoding="utf-8")) + nlv = data.get("non_llm_validators") or {} + val_hits = list(_iter_val_findings(nlv)) + s = nlv.get("llm_confirmation_summary") or {} + if val_hits: + counts["bearing"] += 1 + for k in ("confirmed", "benign", "uncertain", "appointed"): + counts[k] += int(s.get(k, 0) or 0) + + for filt in FILTERS: + agents = list(base_agents) + idxs = list(base_idxs) + primary = base_primary + first = base_first + if filt != "none_only": + for finding in val_hits: + if not _passes(finding, filt): + continue + ix = _val_locus_idx(finding, locus_mode) + a = name_at.get(ix) if ix is not None else None + _extend(agents, idxs, a, ix) + if primary is None and a: + primary = a + if first is None and ix is not None: + first = ix + + pred_agents_norm = [_normalize_agent(a) for a in agents] + primary_norm = _normalize_agent(primary) if primary else None + if gold_agents: + acc[filt]["agent_top1"].append(bool(primary_norm and primary_norm in gold_agents)) + acc[filt]["agent_hit"].append(any(a in gold_agents for a in pred_agents_norm)) + if gold_idxs: + acc[filt]["step_top1"].append( + bool(first and _idx_matches_any(first, gold_idxs, tolerance=0))) + acc[filt]["step_hit"].append( + any(_idx_matches_any(ix, gold_idxs, tolerance=0) for ix in idxs)) + acc[filt]["step_pm1"].append( + any(_idx_matches_any(ix, gold_idxs, tolerance=step_tolerance) for ix in idxs)) + + def mean(xs): + return sum(xs) / len(xs) if xs else None + + summary = {f: {k: mean(v) for k, v in acc[f].items()} for f in FILTERS} + return {"matched": matched, "summary": summary, "counts": counts, "raw": acc} + + +def _combine(per_system: dict[str, dict]) -> dict: + """True overall by concatenating per-trace booleans across systems.""" + acc = {f: {k: [] for k in METRICS} for f in FILTERS} + counts = {"bearing": 0, "confirmed": 0, "benign": 0, "uncertain": 0, "appointed": 0} + matched = 0 + for res in per_system.values(): + matched += res["matched"] + for k in counts: + counts[k] += res["counts"][k] + for f in FILTERS: + for k in METRICS: + acc[f][k].extend(res["raw"][f][k]) + + def mean(xs): + return sum(xs) / len(xs) if xs else None + + summary = {f: {k: mean(v) for k, v in acc[f].items()} for f in FILTERS} + return {"matched": matched, "summary": summary, "counts": counts} + + +def _pct(v): + return f"{v * 100:5.1f}%" if isinstance(v, (int, float)) else " —" + + +ROWS = [ + ("Agent Top-1", "agent_top1"), ("Agent Hit", "agent_hit"), + ("Step Top-1", "step_top1"), ("Step Hit", "step_hit"), ("Step Hit ±1", "step_pm1"), +] + + +def _render(name: str, res: dict, locus_mode: str) -> str: + s = res["summary"] + c = res["counts"] + header = f"{'Metric':<14}" + "".join(FILTER_LABELS[f].rjust(22) for f in FILTERS) + lines = [ + f"=== {name} — none + validators (llm-confirm), locus={locus_mode} (matched {res['matched']}) ===", + f"bearing={c['bearing']} confirmed={c['confirmed']} uncertain={c['uncertain']} " + f"benign={c['benign']} appointed={c['appointed']}", + "", + header, + "-" * len(header), + ] + for label, key in ROWS: + lines.append(f"{label:<14}" + "".join(_pct(s[f][key]).rjust(22) for f in FILTERS)) + return "\n".join(lines) + + +def _markdown(all_res: dict, locus_modes: list[str]) -> str: + md = [ + "# TraceElephant — none + non_llm_validators (LLM-confirm) fold-in", + "", + "- Baseline: verifier=`none` over the 11 LLM evaluators.", + "- Fold-in idx shifted **+1** (validators 0-based → judge/gold 1-based space).", + "- Fold-in agent = step label at the locus (sub-agent for captain/magentic, " + "tool for swe); swe baseline agent = tool at the judge locus.", + "- Filters: `all` = every regex finding (captain fires on 100% of traces → " + "recall ceiling, not a result); `conf+unc` = confirmed|uncertain; " + "`confirmed` = confirmed only (**the honest column**).", + "", + ] + for locus_mode in locus_modes: + md.append(f"## Locus = {locus_mode}") + md.append("") + for name in (*SYSTEMS, "OVERALL"): + res = all_res[locus_mode][name] + c = res["counts"] + md.append(f"### {name} (matched {res['matched']})") + md.append("") + md.append(f"_bearing={c['bearing']}, confirmed={c['confirmed']}, " + f"uncertain={c['uncertain']}, benign={c['benign']}, " + f"appointed={c['appointed']}_") + md.append("") + md.append("| Metric | " + " | ".join(FILTER_LABELS[f] for f in FILTERS) + " |") + md.append("|---|" + "---:|" * len(FILTERS)) + s = res["summary"] + for label, key in ROWS: + md.append("| " + label + " | " + + " | ".join(_pct(s[f][key]).strip() for f in FILTERS) + " |") + md.append("") + return "\n".join(md) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="TraceElephant none + validators fold-in.") + parser.add_argument("--system", choices=("all", *SYSTEMS), default="all") + parser.add_argument("--data-dir", default=None) + parser.add_argument("--step-tolerance", type=int, default=1) + parser.add_argument("--locus", choices=("appointed", "surface", "both"), default="both") + args = parser.parse_args() + + systems = SYSTEMS if args.system == "all" else (args.system,) + locus_modes = ["appointed", "surface"] if args.locus == "both" else [args.locus] + + all_res: dict[str, dict] = {} + for locus_mode in locus_modes: + per_system = { + name: score(name, locus_mode, data_dir=args.data_dir, + step_tolerance=args.step_tolerance) + for name in systems + } + blocks = dict(per_system) + if args.system == "all": + blocks["OVERALL"] = _combine(per_system) + all_res[locus_mode] = blocks + print() + for name, res in blocks.items(): + print(_render(name, res, locus_mode)) + print() + + reports_dir = THIS_DIR / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + # Strip raw boolean lists before serializing. + serializable = { + lm: {name: {k: v for k, v in res.items() if k != "raw"} for name, res in blocks.items()} + for lm, blocks in all_res.items() + } + json_path = reports_dir / "none_plus_validators.json" + json_path.write_text(json.dumps( + {"filter_labels": FILTER_LABELS, "results": serializable}, indent=2), encoding="utf-8") + if args.system == "all": + md_path = reports_dir / "none_plus_validators.md" + md_path.write_text(_markdown(all_res, locus_modes), encoding="utf-8") + print(f"Saved: {md_path}") + print(f"Saved: {json_path}") diff --git a/examples/trace_elephant/score_swe_tool_attribution.py b/examples/trace_elephant/score_swe_tool_attribution.py new file mode 100644 index 0000000..c9a2504 --- /dev/null +++ b/examples/trace_elephant/score_swe_tool_attribution.py @@ -0,0 +1,158 @@ +"""SWE-Agent culprit attribution scored in TraceElephant's own namespace. + +The default agent scorer compares the pipeline's named culprit *agent* +(``SWE-Agent`` / ``assistant``) against TraceElephant's SWE gold ``mistake_agent`` +— but for SWE that gold field is a **tool** name (``bash``, +``str_replace_editor``, ``str_replace_based_edit_tool``, ...), not an agent. The +namespaces never intersect, so agent accuracy comes out 0 even though step +localization works. (Captain/Magentic annotate real sub-agents, which the judges +name directly, so they are scored as-is and do not need this.) + +TraceElephant labels every SWE step by the tool it invoked, and the gold +``mistake_agent`` equals the label of the gold ``mistake_step`` (verified: 100%). +So the correct, comparable culprit prediction for SWE is **the tool at the step +we localize to**: + +* Agent Top-1 = tool at the top-1 predicted step (``first_problem_idx``) equals + the gold tool; +* Agent Hit = the gold tool is the label of any predicted problematic step. + +Step metrics are namespace-agnostic and identical to the default scorer; they are +recomputed here only so one table carries the full picture under each verifier +gating. + +Usage: + python score_swe_tool_attribution.py # none/strict/soft + python score_swe_tool_attribution.py --verifier-mode none +""" + +from __future__ import annotations + +import argparse +import glob +import json +import sys +from pathlib import Path + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from maseval.diagnostic_accuracy import ( # noqa: E402 + _idx_matches_any, + _normalize_agent, + read_prediction_file, +) + +import calculate_agent_step_accuracy as C # noqa: E402 + +MODES = ("none", "strict", "soft") + + +def _label_map(example) -> dict[str, str]: + """1-based step idx -> step label (the invoked tool for SWE).""" + return {str(k + 1): s["name"] for k, s in enumerate(example.history)} + + +def score( + data_dir: str | None = None, + pred_glob: str | None = None, + verifier_mode: str | None = None, + step_tolerance: int = 1, +) -> dict: + data_dir = data_dir or str(THIS_DIR / "data") + examples = C._subset(data_dir, "swe") + by_row = {e.row_index: e for e in examples} + pred_glob = pred_glob or str(THIS_DIR / "trace_elephant_swe_findings") + files = sorted( + glob.glob(str(Path(pred_glob) / "*.json")) if Path(pred_glob).is_dir() + else glob.glob(pred_glob), + key=lambda p: int(p.rsplit("_", 1)[1].split(".")[0]), + ) + if not files: + raise FileNotFoundError(f"No SWE prediction files: {pred_glob!r}") + + acc = {k: [] for k in ("agent_top1", "agent_hit", "step_top1", "step_hit", "step_pm1")} + for fp in files: + i = int(fp.rsplit("_", 1)[1].split(".")[0]) + ex = by_row.get(i) + if ex is None: + continue + name_at = _label_map(ex) + gold_tool = _normalize_agent(ex.mistake_agent) + gold_steps = [str(ex.mistake_step)] if ex.mistake_step else [] + + pred = read_prediction_file(fp, verifier_mode=verifier_mode) + + if gold_tool: + top1_tool = _normalize_agent(name_at.get(str(pred.first_idx))) if pred.first_idx else "" + pred_tools = {_normalize_agent(name_at.get(str(ix))) for ix in pred.idxs if name_at.get(str(ix))} + acc["agent_top1"].append(top1_tool == gold_tool) + acc["agent_hit"].append(gold_tool in pred_tools) + if gold_steps: + acc["step_top1"].append( + bool(pred.first_idx and _idx_matches_any(pred.first_idx, gold_steps, tolerance=0))) + acc["step_hit"].append( + any(_idx_matches_any(ix, gold_steps, tolerance=0) for ix in pred.idxs)) + acc["step_pm1"].append( + any(_idx_matches_any(ix, gold_steps, tolerance=step_tolerance) for ix in pred.idxs)) + + def mean(xs): + return sum(xs) / len(xs) if xs else None + + return {"matched": len(files), "summary": {k: mean(v) for k, v in acc.items()}} + + +def _pct(v): + return f"{v * 100:5.1f}%" if isinstance(v, (int, float)) else " —" + + +ROWS = [ + ("Agent Top-1 (tool)", "agent_top1"), + ("Agent Hit (tool)", "agent_hit"), + ("Step Top-1", "step_top1"), + ("Step Hit", "step_hit"), + ("Step Hit ±1", "step_pm1"), +] + + +def _render(results: dict[str, dict]) -> str: + matched = next(iter(results.values()))["matched"] + lines = [ + f"=== SWE tool-attribution (step-derived) — TraceElephant (matched {matched}) ===", + "Culprit = tool label at the localized step (SWE gold mistake_agent is a tool).", + "", + f"{'Metric':<20}" + "".join(m.rjust(9) for m in MODES), + "-" * (20 + 9 * len(MODES)), + ] + for label, key in ROWS: + lines.append(f"{label:<20}" + "".join(_pct(results[m]["summary"][key]).rjust(9) for m in MODES)) + return "\n".join(lines) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="SWE tool-namespace culprit attribution.") + parser.add_argument("--data-dir", default=None) + parser.add_argument("--pred-glob", default=None) + parser.add_argument("--verifier-mode", choices=MODES, default=None, + help="Score a single mode; omit to run all three.") + parser.add_argument("--step-tolerance", type=int, default=1) + args = parser.parse_args() + + modes = (args.verifier_mode,) if args.verifier_mode else MODES + results = { + m: score(args.data_dir, args.pred_glob, verifier_mode=m, step_tolerance=args.step_tolerance) + for m in modes + } + if len(modes) == len(MODES): + print(_render(results)) + else: + print(json.dumps(results[modes[0]]["summary"], indent=2)) + + reports_dir = THIS_DIR / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + (reports_dir / "swe_tool_attribution.json").write_text( + json.dumps(results, indent=2), encoding="utf-8") + print(f"\nSaved: {reports_dir / 'swe_tool_attribution.json'}") diff --git a/examples/trace_elephant/trace_elephant_data.py b/examples/trace_elephant/trace_elephant_data.py new file mode 100644 index 0000000..7a06947 --- /dev/null +++ b/examples/trace_elephant/trace_elephant_data.py @@ -0,0 +1,309 @@ +"""Shared data access for the TraceElephant findings pipeline. + +TraceElephant (https://huggingface.co/datasets/TraceElephant/TraceElephant, ACL +2026, "Seeing the Whole Elephant") ships 220 annotated *failure* traces (of 380 +total executions) from three multi-agent systems: **Captain-Agent**, +**Magentic-One**, and **SWE-Agent**. Each failed trace is annotated with the +responsible agent (``mistake_agent``) and the decisive step (``mistake_step``, +the earliest inevitable error, 1-based over the execution history). + +On disk (after unzipping ``data.zip``) a trace lives in a per-task directory +under a ``{captain,magentic,swe-agent}-runs-*`` parent, in one of two shapes: + +* **new** -- ``trace_metadata.json`` (task_instruction, ground_truth, mistake_agent, + mistake_step, agent_system_intro) + ``step_records.json`` (a list of + ``{agent_name, input, output}`` steps, where ``output`` is a ChatCompletion repr); +* **old** -- ``summary.json`` (question, ground_truth, mistake_agent, mistake_step) + + ``history.json`` (a list of ``{agent_name, request, response}`` steps). + +A flat ``{task}.json`` with ``mistake_agent``/``mistake_step``/``history`` is also +supported. We normalize all three into a ``history`` of ``{name, content}`` steps. + +The gold ``mistake_step`` is the **1-based position in the (failure-relevant) +history**, so spans MUST be keyed by that position (see :func:`format_trace` / +:func:`idxs`); evaluators cite the step number in ``evidence[i].idx``. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# runs-dir prefix -> coarse system category (matches TraceElephant's evaluate.py). +_RUNS_PREFIXES = { + "captain-runs-": "captain", + "magentic-runs-": "magentic", + "swe-agent-runs-": "swe", +} + + +def _extract_completion_content(output: Any) -> str | None: + """Pull the assistant message text out of a ChatCompletion repr string. + + ``step_records.json`` stores ``output`` as ``"ChatCompletion(... content='...' ...)"``. + Mirror TraceElephant's own regex so the rendered step shows the real content. + """ + if not isinstance(output, str) or "content=" not in output: + return None + m = re.search(r"content='(.*?)'(?:,|\))", output.replace("\\'", "'")) + if not m: + return None + return m.group(1).replace("\\n", "\n").replace("\\t", "\t") + + +def _step_content(name: str, request: Any, response: Any, raw_output: Any) -> str: + """A compact, attribution-relevant rendering of one agent turn. + + Prefer the assistant's response text (what the agent actually produced); + fall back to the raw request/response payloads so nothing is silently lost. + """ + content = _extract_completion_content(raw_output) + if content: + return content + if isinstance(response, dict) and response.get("content"): + return str(response["content"]) + if response: + return json.dumps(response, ensure_ascii=False, default=str) + if request: + return json.dumps(request, ensure_ascii=False, default=str) + return "" + + +@dataclass +class TraceElephantExample: + """One annotated failure trace: normalized history + gold attribution.""" + + row_index: int + task_name: str + system_category: str # captain | magentic | swe | other + system_name: str | None + question: str + ground_truth: str + agent_system_intro: str + # history[i] = {"name": agent, "content": text}; step number = i + 1. + history: list[dict[str, str]] = field(default_factory=list) + mistake_agent: str = "" + mistake_step: str = "" + + +def _normalize_task(task_dir: Path) -> dict[str, Any] | None: + """Load one task dir into the normalized shape, or None if unrecognized.""" + meta_f = task_dir / "trace_metadata.json" + steps_f = task_dir / "step_records.json" + summary_f = task_dir / "summary.json" + history_f = task_dir / "history.json" + + if meta_f.exists() and steps_f.exists(): + meta = json.loads(meta_f.read_text(encoding="utf-8")) + step_records = json.loads(steps_f.read_text(encoding="utf-8")) + history = [ + { + "name": s.get("agent_name", "Unknown"), + "content": _step_content( + s.get("agent_name", "Unknown"), s.get("input", {}), None, s.get("output", "") + ), + } + for s in step_records + ] + return { + "question": meta.get("task_instruction", ""), + "ground_truth": meta.get("ground_truth", ""), + "agent_system_intro": meta.get("agent_system_intro", ""), + "system_name": meta.get("system_name"), + "history": history, + "mistake_agent": str(meta.get("mistake_agent", "") or ""), + "mistake_step": str(meta.get("mistake_step", "") or ""), + } + + if summary_f.exists() and history_f.exists(): + summary = json.loads(summary_f.read_text(encoding="utf-8")) + history_steps = json.loads(history_f.read_text(encoding="utf-8")) + history = [ + { + "name": s.get("agent_name", "Unknown"), + "content": _step_content( + s.get("agent_name", "Unknown"), s.get("request", {}), s.get("response", {}), None + ), + } + for s in history_steps + ] + return { + "question": summary.get("question", ""), + "ground_truth": summary.get("ground_truth", ""), + "agent_system_intro": summary.get("agent_system_intro", ""), + "system_name": summary.get("system_name"), + "history": history, + "mistake_agent": str(summary.get("mistake_agent", "") or ""), + "mistake_step": str(summary.get("mistake_step", "") or ""), + } + + return None + + +def _flat_task(json_file: Path) -> dict[str, Any] | None: + """Load a flat ``{task}.json`` that already carries history + gold.""" + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + except Exception: + return None + if not isinstance(data, dict) or "mistake_agent" not in data: + return None + history = [] + for s in data.get("history") or []: + name = s.get("name") or s.get("agent_name", "Unknown") + content = s.get("content") + if content is None: + content = _step_content(name, s.get("request", {}), s.get("response", {}), s.get("output", "")) + history.append({"name": name, "content": str(content)}) + return { + "question": data.get("question", ""), + "ground_truth": data.get("ground_truth", ""), + "agent_system_intro": data.get("agent_system_intro", ""), + "system_name": data.get("system_name"), + "history": history, + "mistake_agent": str(data.get("mistake_agent", "") or ""), + "mistake_step": str(data.get("mistake_step", "") or ""), + } + + +def _iter_task_dirs(data_dir: Path): + """Yield (task_name, task_dir, category) for every task under data_dir. + + Handles the ``{system}-runs-*`` nested layout and a flat directory of task + subdirectories / task JSON files. + """ + runs_dirs = [ + d for d in data_dir.iterdir() + if d.is_dir() and any(d.name.startswith(p) for p in _RUNS_PREFIXES) + ] + if runs_dirs: + for runs_dir in sorted(runs_dirs): + category = next( + (c for p, c in _RUNS_PREFIXES.items() if runs_dir.name.startswith(p)), "other" + ) + for task_dir in sorted(p for p in runs_dir.iterdir() if p.is_dir()): + yield task_dir.name, task_dir, category + return + # Flat: task subdirectories, then loose task JSON files. + for task_dir in sorted(p for p in data_dir.iterdir() if p.is_dir()): + yield task_dir.name, task_dir, "other" + for jf in sorted(data_dir.glob("*.json")): + yield jf.stem, jf, "other" + + +def load_examples(data_dir: str | Path, *, failures_only: bool = True) -> list[TraceElephantExample]: + """Load TraceElephant tasks under ``data_dir`` into ordered examples. + + ``failures_only`` keeps the 220 annotated failure traces (those carrying both + a ``mistake_agent`` and a ``mistake_step``) -- the attribution benchmark. + """ + data_dir = Path(data_dir) + if not data_dir.exists(): + raise FileNotFoundError(f"TraceElephant data dir not found: {data_dir}") + + examples: list[TraceElephantExample] = [] + for task_name, path, category in _iter_task_dirs(data_dir): + norm = _normalize_task(path) if path.is_dir() else _flat_task(path) + if norm is None: + continue + if failures_only and not (norm["mistake_agent"] and norm["mistake_step"]): + continue + if not norm["history"]: + continue + examples.append( + TraceElephantExample( + row_index=len(examples), + task_name=task_name, + system_category=category, + system_name=norm.get("system_name"), + question=norm["question"], + ground_truth=norm["ground_truth"], + agent_system_intro=norm["agent_system_intro"], + history=norm["history"], + mistake_agent=norm["mistake_agent"], + mistake_step=norm["mistake_step"], + ) + ) + return examples + + +def idxs(history: list[dict[str, str]]) -> list[str]: + """The idx (span-id) space: each step's 1-based position (== gold mistake_step).""" + return [str(i + 1) for i in range(len(history))] + + +# Char budget for the rendered trace body. Some TraceElephant traces are enormous +# (SWE code dumps reach ~10M chars, ~2.6M tokens) and overflow gemini-2.5-flash's +# ~1M-token context -> the evaluator call 400s. We cap the *per-step* content so +# every step index stays visible (dropping steps would risk hiding the culprit +# step and break localization). Traces under budget render unchanged. +TRACE_CHAR_BUDGET = 500_000 +_MIN_STEP_CHARS = 600 + + +def _truncate(text: str, limit: int) -> str: + if len(text) <= limit: + return text + dropped = len(text) - limit + return f"{text[:limit]}\n …[truncated {dropped} chars]" + + +def format_trace(example: TraceElephantExample) -> str: + """Render a failure trace for an LLM evaluator. + + Each step is prefixed with its **1-based step number** (== gold + ``mistake_step``); evaluators cite that number in ``evidence[i].idx``. + Oversized traces are compressed by capping each step's content (all step + numbers are preserved) so the call fits the model context. + """ + lines = ["USER INSTRUCTION:", str(example.question), ""] + if example.agent_system_intro: + lines += ["AGENT SYSTEM:", str(example.agent_system_intro), ""] + lines.append( + "TRACE STEPS (each step is prefixed with its 1-based step number in " + "square brackets; cite that number in evidence[i].idx):" + ) + + n = len(example.history) or 1 + total = sum(len(s["content"]) for s in example.history) + # Only cap when the trace would exceed the budget; scale the per-step limit + # so the whole trace fits, but never below a floor that keeps steps legible. + step_limit = None if total <= TRACE_CHAR_BUDGET else max(_MIN_STEP_CHARS, TRACE_CHAR_BUDGET // n) + + for i, step in enumerate(example.history, start=1): + content = step["content"] if step_limit is None else _truncate(step["content"], step_limit) + lines.append(f"[{i}] {step['name']}: {content}") + return "\n".join(lines) + + +def gold_records(examples: list[TraceElephantExample]) -> list[dict[str, Any]]: + """Annotation rows for the scorer, one per failure trace. + + TraceElephant gold is a single decisive (agent, step) per trace; we expose it + as one-element lists so the shared set-based accuracy scorer applies directly. + """ + return [ + { + "row_index": ex.row_index, + "task_name": ex.task_name, + "system_category": ex.system_category, + "mistake_agents": [ex.mistake_agent] if ex.mistake_agent else [], + "mistake_steps": [ex.mistake_step] if ex.mistake_step else [], + "mistake_agent": ex.mistake_agent, + "mistake_step": ex.mistake_step, + } + for ex in examples + ] + + +def write_gold_jsonl(examples: list[TraceElephantExample], output_path: str) -> str: + """Write the gold table to JSONL (used by the accuracy scorer).""" + out = Path(output_path) + out.parent.mkdir(parents=True, exist_ok=True) + with out.open("w", encoding="utf-8") as f: + for rec in gold_records(examples): + f.write(json.dumps(rec, ensure_ascii=False, default=str) + "\n") + return str(out) diff --git a/examples/trace_elephant/verifier_ablation.py b/examples/trace_elephant/verifier_ablation.py new file mode 100644 index 0000000..e36a3e7 --- /dev/null +++ b/examples/trace_elephant/verifier_ablation.py @@ -0,0 +1,111 @@ +"""EvidenceVerifier ablation for TraceElephant agent/step localization. + +Scores the same TraceElephant findings under three verifier gating settings and +writes one JSON report per setting, plus a printed comparison table: + +* ``none`` -- no verifier: every LLM finding counts; +* ``strict`` -- only ``verified`` findings count (weak + invalid -> review); +* ``soft`` -- ``verified``/``weak`` count, ``invalid`` -> review (prior default). + +``non_llm_validators`` are NOT counted in any setting (LLM findings only). Reports +are rebuilt from the raw LLM findings under each gating via the shared +``maseval.diagnostic_accuracy`` scorer (``verifier_mode``). + +Run ``launch_findings_judges.py`` first to produce the findings this scores. + +Debugger-friendly: call ``run(...)`` directly, or run as a script. + + run(system="all") +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +import calculate_agent_step_accuracy as scorer # noqa: E402 + +MODES = ("none", "strict", "soft") + +# TraceElephant gold is one agent + one step, so both levels are meaningful. +METRIC_KEYS = [ + ("agent_top1_acc", "Agent Top-1"), + ("agent_hit_acc", "Agent Hit"), + ("step_top1_acc", "Step Top-1"), + ("step_top1_pm1_acc", "Step Top-1 ±1"), + ("step_hit_acc", "Step Hit"), +] + + +def run( + system: str = "all", + *, + pred_glob=None, + data_dir: str | None = None, + step_tolerance: int = 1, + reports_dir: str | Path = THIS_DIR / "reports", +) -> dict[str, dict]: + """Score the 3 verifier modes, write per-mode reports, return the summaries.""" + reports_dir = Path(reports_dir) + reports_dir.mkdir(parents=True, exist_ok=True) + + summaries: dict[str, dict] = {} + for mode in MODES: + result = scorer.main( + system=system, + pred_glob=pred_glob, + data_dir=data_dir, + step_tolerance=step_tolerance, + verifier_mode=mode, + first_idx_mode="top_ranked", + output_path=str(reports_dir / f"verifier_ablation_{system}_{mode}.json"), + print_summary=False, + ) + summaries[mode] = result["summary"] + + print(_render(system, summaries)) + return summaries + + +def _pct(v): + return f"{v * 100:5.1f}%" if isinstance(v, (int, float)) else " —" + + +def _render(system: str, summaries: dict[str, dict]) -> str: + matched = summaries["soft"].get("matched_annotations") + lines = [ + f"=== Verifier ablation — TraceElephant / {system} (matched {matched}) ===", + "LLM findings only; non_llm_validators not counted.", + "", + f"{'Metric':<16}{'none':>9}{'strict':>9}{'soft':>9}", + "-" * 43, + ] + for key, label in METRIC_KEYS: + row = "".join(_pct(summaries[m].get(key)).rjust(9) for m in MODES) + lines.append(f"{label:<16}{row}") + return "\n".join(lines) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="TraceElephant EvidenceVerifier ablation.") + parser.add_argument("--system", choices=("all", "captain", "magentic", "swe"), default="all") + parser.add_argument("--pred-glob", default=None, help="Override findings dir/glob.") + parser.add_argument("--data-dir", default=None) + parser.add_argument("--step-tolerance", type=int, default=1) + args = parser.parse_args() + + run( + system=args.system, + pred_glob=args.pred_glob, + data_dir=args.data_dir, + step_tolerance=args.step_tolerance, + ) diff --git a/examples/trail/calc_metrics_trail_findings.py b/examples/trail/calc_metrics_trail_findings.py new file mode 100755 index 0000000..a8fdf0c --- /dev/null +++ b/examples/trail/calc_metrics_trail_findings.py @@ -0,0 +1,320 @@ +"""Mapping-free TRAIL metrics for our MASeval pipeline run (minor-filter sweep). + +Scores the per-task findings produced by ``launch_findings_judges.py`` +(``examples/trail/trail_gemini_findings_v1/``) against the gold TRAIL +annotations embedded in each file under ``labels`` -- WITHOUT translating our +11 maseval metrics into TRAIL's 21-category taxonomy (the two taxonomies are +incommensurable, so category-level comparison is not attempted here). + +Metrics computed (all criterion-independent): + +1. LoCc (location accuracy) -- TRAIL's location metric, which by definition + uses ONLY span locations, not categories. We resolve each finding's cited + evidence ``idx`` back to the original TRAIL ``span_id`` and compare the set + of flagged spans to the set of gold error locations. Reported as exact + match plus within +/-1 / +/-3 span steps. + +2. Volume correlation -- Spearman correlation between the number of findings + our pipeline emits per trace and the number of gold errors, plus the + correlation of our finding count with the gold overall score. + +3. Descriptive summary -- distribution of our finding volume, gold error + volume, severity mix, and evidence-grounding status across the run. + +A constant ``IDX_SHIFT = +1`` corrects a version drift between the trace +formatter the LLM judge saw at evaluation time and the current +``trail_to_spans`` ordering (validated: exact gold-span match rises from 0.25% +to ~40% under the shift). + +The script sweeps ``MINOR_CONFIDENCE_CONFIGS`` -- how strictly "minor" findings +are filtered -- to show sensitivity of the metrics to that choice. +""" + +from __future__ import annotations + +import json +import os +import sys +from collections import Counter +from pathlib import Path + +import numpy as np + +# Make the repo root importable (src/ layout). +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from maseval.validators.base import trail_to_spans # noqa: E402 + +FINDINGS_DIR = Path(__file__).parent / "trail_gemini_findings_v1" +# GAIA raw-trace dir. Override per machine with $TRAIL_GAIA_DIR. +GAIA_DIR = Path(os.environ.get( + "TRAIL_GAIA_DIR", + "/mnt/c/Users/barak/Downloads/trail-benchmark-main/benchmarking/data/GAIA", +)) + +# Version-drift correction for the zero-based span index (see module docstring). +IDX_SHIFT = 1 + +# Tolerance windows (in span-index steps) reported alongside exact LoCc. +LOC_TOLERANCES = [0, 1, 3] + +# How "minor" findings are treated. Config name -> set of confidence levels at +# which a minor finding is still kept (None = keep all minors regardless). +MINOR_CONFIDENCE_CONFIGS = { + "minor=high": {"high"}, + "minor>=medium": {"high", "medium"}, + "all": None, +} + + +def _is_qualifying_finding(finding: dict, keep_minor) -> bool: + severity = str(finding.get("severity_estimate", "major")).lower() + confidence = str(finding.get("confidence_estimate", "medium")).lower() + if severity == "minor": + if keep_minor is None: + return True + return confidence in keep_minor + return True + + +def build_idx_to_span_id(trace_id: str) -> dict[int, str]: + """Map zero-based span index -> original TRAIL span_id for one trace.""" + trace_path = GAIA_DIR / f"{trace_id}.json" + if not trace_path.is_file(): + return {} + try: + raw = json.loads(trace_path.read_text(encoding="utf-8")) + except Exception: + return {} + spans = trail_to_spans(raw) + mapping: dict[int, str] = {} + for i, span in enumerate(spans): + sid = span.get("idx") or span.get("span_id") + if sid is not None: + mapping[i] = str(sid) + return mapping + + +def build_spanid_to_index(trace_id: str) -> dict[str, int]: + """Inverse of :func:`build_idx_to_span_id`: TRAIL span_id -> index.""" + return {sid: i for i, sid in build_idx_to_span_id(trace_id).items()} + + +def resolve_location(finding: dict, idx_to_span_id: dict[int, str]) -> str | None: + """Resolve a finding's evidence into a gold span_id (or None if unresolvable).""" + evidence = finding.get("evidence") or [] + if not evidence: + return None + for ev in evidence: + v = str(ev.get("idx", "")).strip() + if v in set(idx_to_span_id.values()): + return v + if v.lstrip("-").isdigit(): + base = int(v) + IDX_SHIFT + for cand in (base, int(v), base - 1, base + 1): + if cand in idx_to_span_id: + return idx_to_span_id[cand] + return None + + +def extract_predicted_locations(data: dict, trace_id: str, keep_minor) -> list[str]: + """Collect the gold span_ids our findings point at (mapping-free).""" + idx_to_span_id = build_idx_to_span_id(trace_id) + locations: list[str] = [] + for block in data.values(): + if not isinstance(block, dict) or "findings" not in block: + continue + for finding in block["findings"]: + if not _is_qualifying_finding(finding, keep_minor): + continue + loc = resolve_location(finding, idx_to_span_id) + if loc: + locations.append(loc) + return locations + + +def count_qualifying_findings(data: dict, keep_minor) -> int: + n = 0 + for block in data.values(): + if not isinstance(block, dict) or "findings" not in block: + continue + n += sum(1 for f in block["findings"] if _is_qualifying_finding(f, keep_minor)) + return n + + +def _ranks(a): + order = sorted(range(len(a)), key=lambda i: a[i]) + r = [0.0] * len(a) + i = 0 + while i < len(a): + j = i + while j + 1 < len(a) and a[order[j + 1]] == a[order[i]]: + j += 1 + avg = (i + j) / 2 + 1 + for k in range(i, j + 1): + r[order[k]] = avg + i = j + 1 + return r + + +def spearman(x, y): + rx, ry = _ranks(x), _ranks(y) + n = len(x) + mx, my = sum(rx) / n, sum(ry) / n + cov = sum((rx[i] - mx) * (ry[i] - my) for i in range(n)) + vx = (sum((v - mx) ** 2 for v in rx)) ** 0.5 + vy = (sum((v - my) ** 2 for v in ry)) ** 0.5 + return cov / (vx * vy) if vx and vy else 0.0 + + +def _median(values: list[float]) -> float: + if not values: + return 0.0 + s = sorted(values) + n = len(s) + mid = n // 2 + return s[mid] if n % 2 else (s[mid - 1] + s[mid]) / 2 + + +def run_config(keep_minor): + """Compute all mapping-free metrics for one minor-filter configuration.""" + locc_exact_sum = 0.0 + tol_acc_sum: dict[int, float] = {t: 0.0 for t in LOC_TOLERANCES} + tol_denom = 0 + + finding_counts: list[int] = [] + gold_error_counts: list[int] = [] + gold_overall: list[float] = [] + severity_counter: Counter = Counter() + evidence_status_counter: Counter = Counter() + + scored = 0 + traces_with_pred = 0 + + for fp in sorted(FINDINGS_DIR.glob("*.json")): + try: + data = json.loads(fp.read_text(encoding="utf-8")) + except Exception: + continue + + labels = data.get("labels") + if not labels or "errors" not in labels: + continue + scored += 1 + trace_id = data.get("trace_id") + + gold_errors = labels.get("errors", []) + pred_locs = extract_predicted_locations(data, trace_id, keep_minor) + if pred_locs: + traces_with_pred += 1 + + # --- LoCc (location only) --- + gold_locs = [e["location"] for e in gold_errors if e["location"]] + sid_to_idx = build_spanid_to_index(trace_id) + if gold_locs and sid_to_idx: + common = set(gold_locs).intersection(set(pred_locs)) + locc_exact_sum += len(common) / len(set(gold_locs)) + + gt_idxs = [sid_to_idx[l] for l in gold_locs if l in sid_to_idx] + pred_idxs = [sid_to_idx[l] for l in pred_locs if l in sid_to_idx] + if gt_idxs: + tol_denom += 1 + for tol in LOC_TOLERANCES: + hit = sum(1 for pi in pred_idxs if any(abs(pi - gi) <= tol for gi in gt_idxs)) + tol_acc_sum[tol] += hit / len(gt_idxs) + + # --- Summary accumulators --- + n_find = count_qualifying_findings(data, keep_minor) + finding_counts.append(n_find) + gold_error_counts.append(len(gold_errors)) + ov = labels.get("scores", [{}])[0].get("overall") + if ov is not None: + gold_overall.append(float(ov)) + + for block in data.values(): + if not isinstance(block, dict) or "findings" not in block: + continue + for finding in block["findings"]: + if not _is_qualifying_finding(finding, keep_minor): + continue + severity_counter[str(finding.get("severity_estimate", "major")).lower()] += 1 + + for ev_block in data.get("evidence_verification", {}).values(): + if not isinstance(ev_block, dict): + continue + for ver in ev_block.get("verifications", []): + evidence_status_counter[str(ver.get("evidence_status", "unknown")).lower()] += 1 + + if scored == 0: + return None + + result = { + "scored": scored, + "traces_with_pred": traces_with_pred, + "locc_exact": locc_exact_sum / scored, + "locc_tol": { + t: (tol_acc_sum[t] / tol_denom if tol_denom else 0.0) + for t in LOC_TOLERANCES + if t != 0 + }, + "rho_gold_err": spearman(finding_counts, gold_error_counts) if len(finding_counts) > 2 else 0.0, + "rho_overall": spearman(finding_counts, gold_overall) if len(finding_counts) > 2 else 0.0, + "find_mean": float(np.mean(finding_counts)), + "find_median": _median(finding_counts), + "sev": dict(severity_counter), + "ev": dict(evidence_status_counter), + } + return result + + +# Default minor-filter policy: keep ALL findings (no minor dropping). +DEFAULT_CONFIG = "all" + + +def main(): + r = run_config(MINOR_CONFIDENCE_CONFIGS[DEFAULT_CONFIG]) + if r is None: + print("No scorable traces found.") + return + + lines = [] + lines.append("=" * 80) + lines.append("Mapping-free TRAIL metrics for MASeval pipeline run") + lines.append(f"(examples/trail/trail_gemini_findings_v1 | minor policy: {DEFAULT_CONFIG})") + lines.append("=" * 80) + lines.append(f"Scored traces : {r['scored']}") + lines.append(f"Traces with >=1 finding : {r['traces_with_pred']}") + lines.append("") + lines.append("--- LoCc (location accuracy, category-free) ---") + lines.append(f"exact match : {r['locc_exact']:.4f}") + for tol in (1, 3): + lines.append(f"within +/-{tol} spans : {r['locc_tol'][tol]:.4f}") + lines.append("") + lines.append("--- Volume correlation (criterion-independent) ---") + lines.append(f"Spearman(#our findings, #gold errors) : {r['rho_gold_err']:.4f}") + lines.append(f"Spearman(#our findings, gold overall) : {r['rho_overall']:.4f}") + lines.append("") + lines.append("--- Descriptive summary ---") + lines.append( + f"our findings / trace : mean={r['find_mean']:.2f} " + f"median={r['find_median']:.1f}" + ) + lines.append( + f"severity mix (our findings): critical={r['sev'].get('critical',0)} " + f"major={r['sev'].get('major',0)} minor={r['sev'].get('minor',0)}" + ) + lines.append( + f"evidence grounding : " + ", ".join(f"{k}={v}" for k, v in r["ev"].items()) + ) + + out = "\n".join(lines) + print(out) + + out_file = FINDINGS_DIR.parent / "trail_gemini_findings_v1-metrics_mappingfree.txt" + out_file.write_text(out + "\n", encoding="utf-8") + print(f"\nWritten summary to: {out_file}") + + +if __name__ == "__main__": + main() diff --git a/examples/trail/calc_metrics_trail_foldin.py b/examples/trail/calc_metrics_trail_foldin.py new file mode 100644 index 0000000..7a8766b --- /dev/null +++ b/examples/trail/calc_metrics_trail_foldin.py @@ -0,0 +1,203 @@ +"""TRAIL LoCc fold-in: judges vs judges + non_llm_validators (LLM-confirmed). + +Extends ``calc_metrics_trail_findings.py`` (same LoCc definition, same +``resolve_location`` logic incl. ``IDX_SHIFT`` and the span-hash resolution) to +add the deterministic validator findings into the predicted-location set, so the +Who&When/AEGIS-style fold-in is measured on TRAIL's *canonical* location metric +rather than a bespoke step-hit. + +LoCc = mean over scored traces of ``|gold ∩ pred| / |gold|`` (per-trace location +recall); ±1/±3 are the neighbor-tolerant variants (span-index distance). + +Reads the CONFIRM folder (``trail_gemini_findings_v1_confirm``), which carries the +judges + validators + ``llm_confirmation``. Validator evidence idxs are already +span-hashes, so they resolve directly; the appointed locus uses the confirmer's +``corrected_idx``. + +Verdict filters: none (judges only, reproduces the 18.3% baseline) / all / +conf+unc / confirmed. Locus for folded validators: surface (evidence idx) or +appointed (corrected_idx, fallback surface). Reported overall + bearing-only. + +Usage: + python calc_metrics_trail_foldin.py +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +THIS = Path(__file__).resolve().parent +sys.path.insert(0, str(THIS)) +sys.path.insert(0, str(THIS.parents[1] / "src")) + +from calc_metrics_trail_findings import ( # noqa: E402 + GAIA_DIR, + IDX_SHIFT, + LOC_TOLERANCES, + MINOR_CONFIDENCE_CONFIGS, + _is_qualifying_finding, + resolve_location, +) +from maseval.validators.base import trail_to_spans # noqa: E402 + +CONFIRM_DIR = THIS / "trail_gemini_findings_v1_confirm" +FILTERS = ("none_only", "all", "conf_unc", "confirmed") +FILTER_LABELS = { + "none_only": "none (judges)", "all": "+val all", + "conf_unc": "+val conf+unc", "confirmed": "+val confirmed", +} +DEFAULT_MINOR = "all" + + +def _maps(trace_id): + p = GAIA_DIR / f"{trace_id}.json" + if not p.is_file(): + return {}, {} + spans = trail_to_spans(json.loads(p.read_text(encoding="utf-8"))) + idx_to_sid = {i: str(s.get("idx") or s.get("span_id")) + for i, s in enumerate(spans) if (s.get("idx") or s.get("span_id")) is not None} + return idx_to_sid, {sid: i for i, sid in idx_to_sid.items()} + + +def _resolve_idx(v, idx_to_sid): + """Single-idx version of resolve_location's inner logic (hash or +1-shifted numeric).""" + v = str(v).strip() + if not v: + return None + if v in set(idx_to_sid.values()): + return v + if v.lstrip("-").isdigit(): + base = int(v) + IDX_SHIFT + for cand in (base, int(v), base - 1, base + 1): + if cand in idx_to_sid: + return idx_to_sid[cand] + return None + + +def judge_locs(data, idx_to_sid, keep_minor): + out = [] + for block in data.values(): + if not isinstance(block, dict) or "findings" not in block: + continue # skips non_llm_validators (nested under 'metrics') + for f in block["findings"]: + if not _is_qualifying_finding(f, keep_minor): + continue + loc = resolve_location(f, idx_to_sid) + if loc: + out.append(loc) + return out + + +def _iter_val(data): + nlv = data.get("non_llm_validators") or {} + for m in (nlv.get("metrics") or {}).values(): + for f in m.get("findings", []): + yield f + + +def _passes(f, filt): + if filt == "all": + return True + verdict = (f.get("llm_confirmation") or {}).get("verdict") + if filt == "conf_unc": + return verdict in ("confirmed", "uncertain") + if filt == "confirmed": + return verdict == "confirmed" + return False + + +def val_locs(data, idx_to_sid, filt, locus): + out = [] + for f in _iter_val(data): + if not _passes(f, filt): + continue + chosen = None + if locus == "appointed": + c = (f.get("llm_confirmation") or {}).get("corrected_idx") + if c is not None and str(c).strip(): + chosen = c + if chosen is None: + ev = f.get("evidence") or [] + if ev: + chosen = ev[0].get("idx") + loc = _resolve_idx(chosen, idx_to_sid) if chosen is not None else None + if loc: + out.append(loc) + return out + + +def locc(gold_locs, pred_locs, sid_to_idx): + exact = len(set(gold_locs) & set(pred_locs)) / len(set(gold_locs)) + gt = [sid_to_idx[l] for l in gold_locs if l in sid_to_idx] + pr = [sid_to_idx[l] for l in pred_locs if l in sid_to_idx] + tol = {} + for t in LOC_TOLERANCES: + if t == 0: + continue + hit = sum(1 for pi in pr if any(abs(pi - gi) <= t for gi in gt)) + tol[t] = (hit / len(gt)) if gt else 0.0 + return exact, tol + + +def run(locus, keep_minor): + acc = {f: {"exact": 0.0, 1: 0.0, 3: 0.0, "n": 0} for f in FILTERS} + acc_b = {f: {"exact": 0.0, 1: 0.0, 3: 0.0, "n": 0} for f in FILTERS} + for fp in sorted(CONFIRM_DIR.glob("*.json")): + data = json.loads(fp.read_text(encoding="utf-8")) + labels = data.get("labels") + if not labels or "errors" not in labels: + continue + # Denominator matches calc_metrics_trail_findings: every trace with an + # 'errors' key is scored; traces with no gold location or no span map + # contribute exact=0 (they still count in n). + gold = [e["location"] for e in labels["errors"] if e.get("location")] + tid = data.get("trace_id") + idx_to_sid, sid_to_idx = _maps(tid) + computable = bool(gold and sid_to_idx) + bearing = any(True for _ in _iter_val(data)) + base = judge_locs(data, idx_to_sid, keep_minor) if computable else [] + for filt in FILTERS: + if computable: + preds = list(base) + if filt != "none_only": + preds += val_locs(data, idx_to_sid, filt, locus) + ex, tol = locc(gold, preds, sid_to_idx) + else: + ex, tol = 0.0, {} + for store, active in ((acc, True), (acc_b, bearing)): + if active: + store[filt]["exact"] += ex + store[filt][1] += tol.get(1, 0.0) + store[filt][3] += tol.get(3, 0.0) + store[filt]["n"] += 1 + return acc, acc_b + + +def _tab(title, acc): + n = acc["none_only"]["n"] + hdr = f"{'LoCc':<10}" + "".join(FILTER_LABELS[f].rjust(16) for f in FILTERS) + lines = [f"--- {title} (n={n}) ---", hdr, "-" * len(hdr)] + for key, lab in (("exact", "exact"), (1, "within ±1"), (3, "within ±3")): + row = f"{lab:<10}" + for f in FILTERS: + m = acc[f]["n"] + row += (f"{acc[f][key]/m*100:6.1f}%" if m else " —").rjust(16) + lines.append(row) + return "\n".join(lines) + + +if __name__ == "__main__": + keep = MINOR_CONFIDENCE_CONFIGS[DEFAULT_MINOR] + out = [f"TRAIL LoCc fold-in (minor policy: {DEFAULT_MINOR}; confirm dir)\n"] + for locus in ("surface", "appointed"): + acc, acc_b = run(locus, keep) + out.append(f"\n==================== LOCUS = {locus} ====================") + out.append(_tab("Overall", acc)) + out.append("") + out.append(_tab("Bearing-only", acc_b)) + text = "\n".join(out) + print(text) + (THIS / "trail_foldin_locc.txt").write_text(text + "\n", encoding="utf-8") + print(f"\nWritten: {THIS / 'trail_foldin_locc.txt'}") diff --git a/examples/trail/gen_validators_confirm.py b/examples/trail/gen_validators_confirm.py new file mode 100644 index 0000000..9d03d29 --- /dev/null +++ b/examples/trail/gen_validators_confirm.py @@ -0,0 +1,107 @@ +"""Add the LLM confirmation layer to TRAIL's already-stored deterministic +``non_llm_validators`` and write augmented result files to +``trail_gemini_findings_v1_confirm/``. + +Like AEGIS, TRAIL's launcher already ran the validators, so we do NOT re-run +``run_on_trace``; we only attach ``llm_confirmation`` (~1 call per finding-bearing +trace). The confirmer needs the raw trace, which the result files do not store, so +we reload it from the GAIA trace directory matched on ``trace_id``. We hand the +confirmer the raw trace dict so ``build_raw_spans`` (→ ``trail_to_spans``) re-derives +the native hex ``span_id`` idxs the findings cite. + +TRAIL localization gold is a **span-hash** (``labels.errors[].location``); the +validators already speak that namespace (evidence idx = span_id), while the LLM +judges cite numeric span *positions* — the scorer bridges that. The confirmer's +appointing (``corrected_idx``) is therefore meaningful here (unlike agent-only AEGIS). + +Resumable: existing output files are skipped. All 117 files are written through +(only finding-bearing ones incur an LLM call) so the scorer reads one folder. + +Usage: + python gen_validators_confirm.py + python gen_validators_confirm.py --limit 1 # smoke test +""" + +from __future__ import annotations + +import argparse +import asyncio +import glob +import json +import os +import sys +from pathlib import Path + +from dotenv import load_dotenv + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +load_dotenv(ROOT / ".env") +load_dotenv(THIS_DIR / ".env") + +from maseval.validators.llm_confirm import confirm_trace_async # noqa: E402 + +# GAIA raw-trace dir. Override per machine with $TRAIL_GAIA_DIR. +DEFAULT_GAIA = os.environ.get( + "TRAIL_GAIA_DIR", + "/mnt/c/Users/barak/Downloads/trail-benchmark-main/benchmarking/data/GAIA", +) + + +def _n(nlv: dict) -> int: + return sum(len(m.get("findings", [])) for m in (nlv.get("metrics") or {}).values()) + + +async def main(model: str, gaia_dir: str, limit: int | None) -> None: + if not os.getenv("OPENROUTER_API_KEY"): + raise RuntimeError("OPENROUTER_API_KEY is not set (source the root .env).") + + files = sorted(glob.glob(str(THIS_DIR / "trail_gemini_findings_v1" / "*.json"))) + out = THIS_DIR / "trail_gemini_findings_v1_confirm" + out.mkdir(parents=True, exist_ok=True) + totals = {"bearing": 0, "confirmed": 0, "benign": 0, "uncertain": 0, "appointed": 0, "no_gaia": 0} + done = 0 + for fp in files: + name = Path(fp).name + out_f = out / name + if out_f.exists(): + continue + data = json.loads(Path(fp).read_text(encoding="utf-8")) + nlv = data.get("non_llm_validators") or {} + if _n(nlv): + tid = data["trace_id"] + gpath = Path(gaia_dir) / f"{tid}.json" + if not gpath.exists(): + print(f" [{name}] MISSING GAIA trace {tid}") + totals["no_gaia"] += 1 + else: + raw = json.loads(gpath.read_text(encoding="utf-8")) + try: + await confirm_trace_async(raw, nlv, model) # full reading view + except Exception as exc: # noqa: BLE001 + print(f" [{name}] confirm error: {type(exc).__name__}: {exc}") + s = nlv.get("llm_confirmation_summary", {}) + totals["bearing"] += 1 + for k in ("confirmed", "benign", "uncertain", "appointed"): + totals[k] += int(s.get(k, 0) or 0) + print(f" [{name}] tid={tid[:8]} {_n(nlv)} det-findings -> {s}") + done += 1 + data["non_llm_validators"] = nlv + out_f.write_text(json.dumps(data, indent=2, default=str), encoding="utf-8") + if limit is not None and done >= limit: + print(f"(stopped after {done} confirmed traces per --limit)") + break + print(f"totals: {totals}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Add LLM confirmation to TRAIL validators.") + parser.add_argument("--model", default="google/gemini-2.5-flash") + parser.add_argument("--gaia-dir", default=DEFAULT_GAIA, help="Dir of GAIA raw trace JSONs.") + parser.add_argument("--limit", type=int, default=None, help="Confirm at most N bearing traces.") + args = parser.parse_args() + asyncio.run(main(model=args.model, gaia_dir=args.gaia_dir, limit=args.limit)) diff --git a/examples/trail/launch_findings_judges.py b/examples/trail/launch_findings_judges.py index e77dd83..6c2cfd8 100644 --- a/examples/trail/launch_findings_judges.py +++ b/examples/trail/launch_findings_judges.py @@ -16,6 +16,7 @@ import json import os +import time from pathlib import Path from typing import Any @@ -42,6 +43,7 @@ from maseval.metrics import EvidenceVerifier, MetricType, create_metric # noqa: E402 from maseval.models import RawTraceInput # noqa: E402 from maseval.reporting import build_evaluation_report # noqa: E402 +from maseval.run_stats import aggregate_task_stats, extract_usage # noqa: E402 from maseval.validators import run_on_trace # noqa: E402 from maseval.validators.base import trail_to_spans # noqa: E402 @@ -61,12 +63,15 @@ ] -TRACES_DIR = Path( - "/Users/alina/trail-benchmark/benchmarking/data/GAIA" -) -ANNOTATIONS_DIR = Path( - "/Users/alina/trail-benchmark/benchmarking/processed_annotations_gaia" -) +# Override per machine with $TRAIL_GAIA_DIR / $TRAIL_ANNO_DIR. +TRACES_DIR = Path(os.environ.get( + "TRAIL_GAIA_DIR", + "/Users/alina/trail-benchmark/benchmarking/data/GAIA", +)) +ANNOTATIONS_DIR = Path(os.environ.get( + "TRAIL_ANNO_DIR", + "/Users/alina/trail-benchmark/benchmarking/processed_annotations_gaia", +)) OUTPUT_DIR = Path(__file__).parent / "trail_gemini_findings_v1" # Binarization threshold for the gold annotation ``scores[0].overall``: @@ -185,19 +190,33 @@ def _serialize_evidence_verification(evidence_results: dict) -> dict: } -async def _run_all_metrics(model, eval_input: RawTraceInput) -> dict: - """Run every LLM evaluator on one task and return ``{metric_name: MetricResult}``.""" +async def _run_all_metrics(model, eval_input: RawTraceInput) -> tuple[dict, dict]: + """Run every LLM evaluator on one task. + + Returns ``(findings_results, metric_status)`` — the latter carries per-metric + timing + token usage for the ``task_stats`` rollup. + """ findings_results: dict = {} + metric_status: dict = {} for metric_type in LLM_METRICS_TO_TEST: - print(f"\n--- Evaluating LLM metric: {metric_type.value} ---") + name = metric_type.value + print(f"\n--- Evaluating LLM metric: {name} ---") + metric = None + t0 = time.perf_counter() try: metric = create_metric(metric_type, model) result = await metric.evaluate(eval_input) - findings_results[metric_type.value] = result - - print(f"Metric: {result.metric_name} | findings: {len(result.findings)}") + inp, out, tot = extract_usage(getattr(metric, "last_usage", None)) + findings_results[name] = result + metric_status[name] = { + "status": "ok", "duration_s": round(time.perf_counter() - t0, 3), + "input_tokens": inp, "output_tokens": out, "total_tokens": tot, + } + + print(f"Metric: {result.metric_name} | findings: {len(result.findings)} " + f"| in={inp} out={out}") for finding in result.findings: print( f" - [{finding.severity_estimate.value}/" @@ -205,10 +224,16 @@ async def _run_all_metrics(model, eval_input: RawTraceInput) -> dict: f"{finding.problem_description[:120]}" ) except Exception as exc: # noqa: BLE001 - print(f"Error evaluating {metric_type.value}: {exc}") + inp, out, tot = extract_usage(getattr(metric, "last_usage", None)) + metric_status[name] = { + "status": "failed", "detail": f"{type(exc).__name__}: {exc}"[:300], + "duration_s": round(time.perf_counter() - t0, 3), + "input_tokens": inp, "output_tokens": out, "total_tokens": tot, + } + print(f"Error evaluating {name}: {exc}") continue - return findings_results + return findings_results, metric_status async def _run_final_answer_verification_no_gt( @@ -291,7 +316,9 @@ async def _evaluate_task( eval_input = RawTraceInput(trace=_format_indexed_trail_trace(raw_trace)) - findings_results = await _run_all_metrics(model, eval_input) + _t0 = time.perf_counter() + findings_results, metric_status = await _run_all_metrics(model, eval_input) + task_stats = aggregate_task_stats(metric_status, time.perf_counter() - _t0) evidence_results = EvidenceVerifier().verify_all(findings_results, eval_input) final_answer_result = await _run_final_answer_verification_no_gt( final_answer_verifier, @@ -303,6 +330,7 @@ async def _evaluate_task( print("=" * 80) serializable_results = _serialize_findings(findings_results) + serializable_results["task_stats"] = task_stats serializable_results["non_llm_validators"] = run_on_trace(raw_trace) serializable_results["evidence_verification"] = _serialize_evidence_verification( evidence_results @@ -357,7 +385,9 @@ async def _evaluate_task_traced( eval_input = RawTraceInput(trace=_format_indexed_trail_trace(raw_trace)) - findings_results = await _run_all_metrics(model, eval_input) + _t0 = time.perf_counter() + findings_results, metric_status = await _run_all_metrics(model, eval_input) + task_stats = aggregate_task_stats(metric_status, time.perf_counter() - _t0) evidence_results = EvidenceVerifier().verify_all(findings_results, eval_input) final_answer_result = await _run_final_answer_verification_no_gt( final_answer_verifier, @@ -365,6 +395,7 @@ async def _evaluate_task_traced( ) serializable_results = _serialize_findings(findings_results) + serializable_results["task_stats"] = task_stats serializable_results["non_llm_validators"] = run_on_trace(raw_trace) serializable_results["evidence_verification"] = _serialize_evidence_verification( evidence_results diff --git a/examples/trail/score_none_plus_validators.py b/examples/trail/score_none_plus_validators.py new file mode 100644 index 0000000..cf6f134 --- /dev/null +++ b/examples/trail/score_none_plus_validators.py @@ -0,0 +1,243 @@ +"""Score TRAIL error-location localization for verifier=``none``, folding in the +deterministic ``non_llm_validators`` (+ LLM confirmation) — the TRAIL analogue of +the Who&When / TraceElephant / AEGIS ``score_none_plus_validators``. + +TRAIL gold is a **span-hash location** (``labels.errors[].location``), with no +agent field (GAIA is effectively single-agent; the "agent" on a finding is a step +label). So this scores **location** localization only — the mirror image of AEGIS +(agent-only). + +**The namespace bridge (the crux).** ``trail_to_spans`` gives every span a native +hex ``span_id``; gold locations and the *validators*' evidence idx both live in that +hash space. But the LLM *judges* cite numeric span **positions** (``[0]``, ``[1]``, +… from ``_format_indexed_trail_trace``, which enumerates the same span list). So we +build ``pos2hash = {str(i): span["idx"]}`` from the trace and map every judge idx +(and ``first_idx``) through it; validator idxs need no mapping. Everything is then +compared in the one hash space. The GAIA raw trace is reloaded by ``trace_id`` for +this map (and it is the same trace the confirmer read). + +Validator locus, two variants: + +* ``surface`` -- the finding's evidence idx (a span_id); +* ``appointed`` -- the confirmer's ``corrected_idx`` (a span_id), fallback surface. + +Filters: ``all`` / ``conf+unc`` / ``confirmed`` (confirmed = the honest column). +Reports overall (all traces with gold) + bearing-only (traces with ≥1 validator +finding), where the signal lives. + +Usage: + python score_none_plus_validators.py + python score_none_plus_validators.py --locus surface +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +import sys +from pathlib import Path + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from maseval.diagnostic_accuracy import read_prediction_file # noqa: E402 +from maseval.validators.base import trail_to_spans # noqa: E402 + +# GAIA raw-trace dir. Override per machine with $TRAIL_GAIA_DIR. +DEFAULT_GAIA = os.environ.get( + "TRAIL_GAIA_DIR", + "/mnt/c/Users/barak/Downloads/trail-benchmark-main/benchmarking/data/GAIA", +) + +FILTERS = ("none_only", "all", "conf_unc", "confirmed") +FILTER_LABELS = { + "none_only": "none (LLM only)", + "all": "none + val (all)", + "conf_unc": "none + val (conf+unc)", + "confirmed": "none + val (confirmed)", +} +METRICS = ("step_top1", "step_hit") + + +def _gold_locations(data: dict) -> set[str]: + return {e["location"] for e in (data.get("labels") or {}).get("errors", []) if e.get("location")} + + +def _iter_val_findings(nlv: dict): + for m in (nlv.get("metrics") or {}).values(): + for f in m.get("findings", []): + yield f + + +def _val_locus_hash(finding: dict, locus_mode: str) -> str | None: + """Span-hash locus. ``appointed`` = corrected_idx (fallback surface); ``surface`` = evidence idx.""" + if locus_mode == "appointed": + c = (finding.get("llm_confirmation") or {}).get("corrected_idx") + if c is not None and str(c).strip(): + return str(c) + ev = finding.get("evidence") or [] + if ev and ev[0].get("idx") is not None: + return str(ev[0]["idx"]) + return None + + +def _passes(finding: dict, filt: str) -> bool: + if filt == "all": + return True + verdict = (finding.get("llm_confirmation") or {}).get("verdict") + if filt == "conf_unc": + return verdict in ("confirmed", "uncertain") + if filt == "confirmed": + return verdict == "confirmed" + return False + + +def score(pred_dir: str, gaia_dir: str, locus_mode: str) -> dict: + pred_dir_p = THIS_DIR / pred_dir if not Path(pred_dir).is_absolute() else Path(pred_dir) + files = sorted(glob.glob(str(pred_dir_p / "*.json"))) + if not files: + raise FileNotFoundError(f"No prediction files in {pred_dir_p}") + gaia = Path(gaia_dir) + + acc = {f: {k: [] for k in METRICS} for f in FILTERS} + acc_bearing = {f: {k: [] for k in METRICS} for f in FILTERS} + matched = bearing = 0 + counts = {"confirmed": 0, "benign": 0, "uncertain": 0, "appointed": 0} + + for fp in files: + data = json.loads(Path(fp).read_text(encoding="utf-8")) + gold = _gold_locations(data) + if not gold: + continue + gpath = gaia / f"{data['trace_id']}.json" + if not gpath.exists(): + continue + matched += 1 + spans = trail_to_spans(json.loads(gpath.read_text(encoding="utf-8"))) + pos2hash = {str(i): s["idx"] for i, s in enumerate(spans)} + + pred = read_prediction_file(fp, verifier_mode="none") + base_idxs = [pos2hash[i] for i in pred.idxs if pos2hash.get(i)] # judge pos -> hash + base_first = pos2hash.get(str(pred.first_idx)) if pred.first_idx else None + + nlv = data.get("non_llm_validators") or {} + val_hits = list(_iter_val_findings(nlv)) + is_bearing = bool(val_hits) + if is_bearing: + bearing += 1 + s = nlv.get("llm_confirmation_summary") or {} + for k in counts: + counts[k] += int(s.get(k, 0) or 0) + + for filt in FILTERS: + idxs = list(base_idxs) + first = base_first + if filt != "none_only": + for finding in val_hits: + if not _passes(finding, filt): + continue + h = _val_locus_hash(finding, locus_mode) + if h is not None and h not in idxs: + idxs.append(h) + if first is None and h is not None: + first = h + + top1 = bool(first and first in gold) + hit = any(h in gold for h in idxs) + for store, active in ((acc, True), (acc_bearing, is_bearing)): + if active: + store[filt]["step_top1"].append(top1) + store[filt]["step_hit"].append(hit) + + def summarize(store): + return {f: {k: (sum(v) / len(v) if v else None) for k, v in store[f].items()} for f in FILTERS} + + return { + "matched": matched, "bearing": bearing, "counts": counts, + "overall": summarize(acc), "bearing_only": summarize(acc_bearing), + } + + +def _pct(v): + return f"{v * 100:5.1f}%" if isinstance(v, (int, float)) else " —" + + +ROWS = [("Step Top-1", "step_top1"), ("Step Hit", "step_hit")] + + +def _render(title: str, summ: dict, n: int, locus: str) -> str: + header = f"{'Metric':<12}" + "".join(FILTER_LABELS[f].rjust(22) for f in FILTERS) + lines = [f"=== {title} — locus={locus} (n={n}) ===", "", header, "-" * len(header)] + for label, key in ROWS: + lines.append(f"{label:<12}" + "".join(_pct(summ[f][key]).rjust(22) for f in FILTERS)) + return "\n".join(lines) + + +def _markdown(all_res: dict, locus_modes: list[str], pred_dir: str) -> str: + any_res = all_res[locus_modes[0]] + md = [ + "# TRAIL — none + non_llm_validators (LLM-confirm) fold-in", + "", + f"- Predictions: `{pred_dir}`", + f"- Matched (gold + GAIA trace): {any_res['matched']}; finding-bearing: {any_res['bearing']}", + f"- Confirmer: confirmed={any_res['counts']['confirmed']}, " + f"uncertain={any_res['counts']['uncertain']}, benign={any_res['counts']['benign']}, " + f"appointed={any_res['counts']['appointed']}", + "- Gold = span-hash `location`; no agent gold (location-only, mirror of AEGIS).", + "- Judge idxs are numeric span positions, mapped to span-hashes via " + "`trail_to_spans`; validator/gold idxs are span-hashes directly.", + "- Filters: `all` = every regex finding; `conf+unc` = confirmed|uncertain; " + "`confirmed` = confirmed only (**the honest column**).", + "", + ] + for locus in locus_modes: + res = all_res[locus] + md.append(f"## Locus = {locus}") + md.append("") + for title, key in (("Overall (traces with gold)", "overall"), + ("Bearing-only (≥1 validator finding)", "bearing_only")): + n = res["matched"] if key == "overall" else res["bearing"] + md.append(f"### {title} (n={n})") + md.append("") + md.append("| Metric | " + " | ".join(FILTER_LABELS[f] for f in FILTERS) + " |") + md.append("|---|" + "---:|" * len(FILTERS)) + s = res[key] + for label, mk in ROWS: + md.append("| " + label + " | " + " | ".join(_pct(s[f][mk]).strip() for f in FILTERS) + " |") + md.append("") + return "\n".join(md) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="TRAIL none + validators (llm-confirm) fold-in.") + parser.add_argument("--pred-dir", default="trail_gemini_findings_v1_confirm") + parser.add_argument("--gaia-dir", default=DEFAULT_GAIA) + parser.add_argument("--locus", choices=("appointed", "surface", "both"), default="both") + args = parser.parse_args() + + locus_modes = ["appointed", "surface"] if args.locus == "both" else [args.locus] + all_res = {lm: score(args.pred_dir, args.gaia_dir, lm) for lm in locus_modes} + + for lm in locus_modes: + res = all_res[lm] + print(_render("TRAIL — Overall", res["overall"], res["matched"], lm)) + print() + print(_render("TRAIL — Bearing-only", res["bearing_only"], res["bearing"], lm)) + print(f" confirmer: confirmed={res['counts']['confirmed']} " + f"uncertain={res['counts']['uncertain']} benign={res['counts']['benign']} " + f"appointed={res['counts']['appointed']}") + print() + + reports_dir = THIS_DIR / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + (reports_dir / "none_plus_validators.json").write_text( + json.dumps({"pred_dir": args.pred_dir, "filter_labels": FILTER_LABELS, "results": all_res}, + indent=2), encoding="utf-8") + (reports_dir / "none_plus_validators.md").write_text( + _markdown(all_res, locus_modes, args.pred_dir), encoding="utf-8") + print(f"Saved: {reports_dir / 'none_plus_validators.md'}") diff --git a/examples/trail/verifier_ablation.py b/examples/trail/verifier_ablation.py new file mode 100644 index 0000000..29ed10b --- /dev/null +++ b/examples/trail/verifier_ablation.py @@ -0,0 +1,104 @@ +"""EvidenceVerifier ablation for TRAIL error-location localization. + +Scores the LLM judge predictions under three verifier gating settings and writes +one JSON report per setting plus a printed table: + +* ``none`` -- no verifier: every LLM finding counts; +* ``strict`` -- only ``verified`` findings count; +* ``soft`` -- ``verified``/``weak`` count, ``invalid`` -> review. + +``non_llm_validators`` are NOT counted in any setting (LLM findings only), matching +the AEGIS/TraceElephant ablations. TRAIL gold is a span-hash ``location`` (no agent), +so only location Step Top-1 / Step Hit are populated. Judge idxs are numeric span +positions, mapped to span-hashes via ``trail_to_spans`` on the reloaded GAIA trace +(see ``score_none_plus_validators`` for the namespace bridge). + +Usage: + python verifier_ablation.py +""" + +from __future__ import annotations + +import argparse +import glob +import json +import sys +from pathlib import Path + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from maseval.diagnostic_accuracy import read_prediction_file # noqa: E402 +from maseval.validators.base import trail_to_spans # noqa: E402 + +from score_none_plus_validators import DEFAULT_GAIA, _gold_locations # noqa: E402 + +MODES = ("none", "strict", "soft") + + +def score(pred_dir: str, gaia_dir: str, mode: str) -> dict: + pred_dir_p = THIS_DIR / pred_dir if not Path(pred_dir).is_absolute() else Path(pred_dir) + files = sorted(glob.glob(str(pred_dir_p / "*.json"))) + if not files: + raise FileNotFoundError(f"No prediction files in {pred_dir_p}") + gaia = Path(gaia_dir) + + top1, hit = [], [] + for fp in files: + data = json.loads(Path(fp).read_text(encoding="utf-8")) + gold = _gold_locations(data) + if not gold: + continue + gpath = gaia / f"{data['trace_id']}.json" + if not gpath.exists(): + continue + spans = trail_to_spans(json.loads(gpath.read_text(encoding="utf-8"))) + pos2hash = {str(i): s["idx"] for i, s in enumerate(spans)} + pred = read_prediction_file(fp, verifier_mode=mode) + idxs = [pos2hash[i] for i in pred.idxs if pos2hash.get(i)] + first = pos2hash.get(str(pred.first_idx)) if pred.first_idx else None + top1.append(bool(first and first in gold)) + hit.append(any(h in gold for h in idxs)) + + def mean(xs): + return sum(xs) / len(xs) if xs else None + + return {"matched": len(top1), "step_top1": mean(top1), "step_hit": mean(hit)} + + +def _pct(v): + return f"{v * 100:5.1f}%" if isinstance(v, (int, float)) else " —" + + +def _render(summaries: dict) -> str: + matched = next(iter(summaries.values()))["matched"] + lines = [ + f"=== Verifier ablation — TRAIL (matched {matched}) ===", + "LLM findings only; non_llm_validators not counted. Location-level (TRAIL gold has no agent).", + "", + f"{'Metric':<12}{'none':>9}{'strict':>9}{'soft':>9}", + "-" * 39, + ] + for label, key in (("Step Top-1", "step_top1"), ("Step Hit", "step_hit")): + lines.append(f"{label:<12}" + "".join(_pct(summaries[m][key]).rjust(9) for m in MODES)) + return "\n".join(lines) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="TRAIL EvidenceVerifier ablation (location).") + parser.add_argument("--pred-dir", default="trail_gemini_findings_v1") + parser.add_argument("--gaia-dir", default=DEFAULT_GAIA) + args = parser.parse_args() + + summaries = {m: score(args.pred_dir, args.gaia_dir, m) for m in MODES} + print(_render(summaries)) + + reports_dir = THIS_DIR / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + for m in MODES: + (reports_dir / f"verifier_ablation_trail_{m}.json").write_text( + json.dumps(summaries[m], indent=2), encoding="utf-8") + print(f"\nSaved: {reports_dir / 'verifier_ablation_trail_{none,strict,soft}.json'}") diff --git a/examples/who_and_when/apply_llm_confirm.py b/examples/who_and_when/apply_llm_confirm.py new file mode 100644 index 0000000..e2d474a --- /dev/null +++ b/examples/who_and_when/apply_llm_confirm.py @@ -0,0 +1,157 @@ +"""Apply the opt-in LLM confirmation layer to existing Who&When findings. + +``maseval.validators.llm_confirm`` is a thin, opt-in pass over the DETERMINISTIC +``non_llm_validators`` findings (it confirms genuine-vs-benign and appoints the +causal agent turn). It does NOT touch the 11 LLM evaluators, so there is no need +to re-run them: this script loads the already-generated per-trace findings, runs +``confirm_trace_async`` on their ``non_llm_validators`` block, and writes the +augmented findings to a sibling ``*_llmconfirm`` folder (non-destructive). + +Only traces that carry deterministic findings incur an LLM call (confirm is a +no-op otherwise), so this is cheap (~1 call per finding-bearing trace). + +The launcher's ``--llm-confirm`` flag does the same thing inline during a fresh +findings run; this script is the way to apply it to findings you already have. + +Usage: + python apply_llm_confirm.py --run hc # Hand-Crafted split + python apply_llm_confirm.py --run algo --model google/gemini-2.5-flash +""" + +from __future__ import annotations + +import argparse +import ast +import asyncio +import glob +import json +import os +import sys +from pathlib import Path + +from dotenv import load_dotenv + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +load_dotenv(THIS_DIR / ".env") + +from maseval.validators.llm_confirm import confirm_trace_async # noqa: E402 + +SPLITS = { + "hc": ("who&when_hand_gemini_idx_msg_v2", "Hand-Crafted.parquet"), + "algo": ("who&when_algo_gemini_idx_msg_v2", "Algorithm-Generated.parquet"), +} + + +def _load_parquet(filename: str): + """Robustly fetch a Who&When split (cached hf_hub_download, not flaky streaming).""" + import pandas as pd + from huggingface_hub import hf_hub_download + + last = None + for _ in range(5): + try: + path = hf_hub_download("Kevin355/Who_and_When", filename=filename, repo_type="dataset") + return pd.read_parquet(path) + except Exception as exc: # noqa: BLE001 - transient network + last = exc + raise RuntimeError(f"Could not download {filename}: {last}") + + +def _coerce_history_to_steps(history) -> list: + """Best-effort conversion of dataset history into ordered trace steps + (mirrors launch_findings_judges._coerce_history_to_steps).""" + if isinstance(history, list): + return history + if isinstance(history, tuple): + return list(history) + if hasattr(history, "tolist"): # numpy ndarray / pandas array from read_parquet + # Expand the per-message list; otherwise the whole array collapses into a + # single step (idx 0, no agent), breaking the validators + appointing. + return list(history.tolist()) + if isinstance(history, str): + text = history.strip() + for parser in (ast.literal_eval, json.loads): + try: + parsed = parser(text) + if isinstance(parsed, list): + return parsed + if isinstance(parsed, dict): + for key in ("history", "messages", "trace", "steps"): + value = parsed.get(key) + if isinstance(value, list): + return value + return [parsed] + except Exception: + pass + return [history] + return [history] + + +def _n_findings(nlv: dict) -> int: + return sum(len(m.get("findings", [])) for m in (nlv.get("metrics") or {}).values()) + + +async def main(run: str, model: str, from_idx: int = 0) -> None: + import pandas as pd + + if not os.getenv("OPENROUTER_API_KEY"): + raise RuntimeError("OPENROUTER_API_KEY is not set (source a .env with the key).") + + folder_name, parquet = SPLITS[run] + src_dir = THIS_DIR / folder_name + out_dir = THIS_DIR / f"{folder_name}_llmconfirm" + out_dir.mkdir(parents=True, exist_ok=True) + + df = _load_parquet(parquet) + files = sorted(glob.glob(str(src_dir / "gemini_findings_*.json")), + key=lambda p: int(p.rsplit("_", 1)[1].split(".")[0])) + print(f"{run}: {len(files)} findings files, {len(df)} dataset rows -> {out_dir.name}") + + totals = {"confirmed": 0, "benign": 0, "uncertain": 0, "appointed": 0, "traces_confirmed": 0} + for fp in files: + i = int(fp.rsplit("_", 1)[1].split(".")[0]) + if i < from_idx: + continue + findings = json.loads(Path(fp).read_text(encoding="utf-8")) + nlv = findings.get("non_llm_validators") or {} + n = _n_findings(nlv) + if n == 0: + # confirm is a no-op; copy through unchanged. + (out_dir / Path(fp).name).write_text(json.dumps(findings, indent=2, default=str), encoding="utf-8") + continue + + history = df.iloc[i]["history"] + trace = {"history": _coerce_history_to_steps(history)} + try: + await confirm_trace_async(trace, nlv, model) + except Exception as exc: # noqa: BLE001 - one trace must not abort the run + print(f" [{i}] confirm error: {type(exc).__name__}: {exc}") + (out_dir / Path(fp).name).write_text(json.dumps(findings, indent=2, default=str), encoding="utf-8") + continue + + summary = nlv.get("llm_confirmation_summary", {}) + totals["traces_confirmed"] += 1 + for k in ("confirmed", "benign", "uncertain", "appointed"): + totals[k] += int(summary.get(k, 0) or 0) + findings["non_llm_validators"] = nlv + (out_dir / Path(fp).name).write_text(json.dumps(findings, indent=2, default=str), encoding="utf-8") + print(f" [{i}] {n} det-findings -> {summary}") + + print("\n=== LLM-confirm summary (" + run + ") ===") + print(f"finding-bearing traces confirmed: {totals['traces_confirmed']}") + print(f"per-finding verdicts: confirmed={totals['confirmed']} " + f"benign={totals['benign']} uncertain={totals['uncertain']} | appointed={totals['appointed']}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Apply LLM confirmation to existing Who&When findings.") + parser.add_argument("--run", choices=("hc", "algo"), default="hc") + parser.add_argument("--model", default="google/gemini-2.5-flash") + parser.add_argument("--from-idx", type=int, default=0) + args = parser.parse_args() + asyncio.run(main(run=args.run, model=args.model, from_idx=args.from_idx)) diff --git a/examples/who_and_when/launch_findings_judges.py b/examples/who_and_when/launch_findings_judges.py index f049dcf..01650f0 100644 --- a/examples/who_and_when/launch_findings_judges.py +++ b/examples/who_and_when/launch_findings_judges.py @@ -13,6 +13,7 @@ import asyncio import json import os +import time from pathlib import Path import pandas as pd @@ -38,7 +39,9 @@ from maseval.metrics import EvidenceVerifier, MetricType, create_metric from maseval.models import RawTraceInput from maseval.reporting import build_evaluation_report +from maseval.run_stats import aggregate_task_stats, extract_usage from maseval.validators import run_on_trace +from maseval.validators.llm_confirm import confirm_trace_async # All active LLM evaluators (the regex-based mas_api_issues / mas_environment_setup_errors # were removed; non-LLM metrics are intentionally excluded). @@ -64,8 +67,8 @@ def _coerce_history_to_steps(history) -> list: return history if isinstance(history, tuple): return list(history) - if hasattr(history, "tolist"): # numpy ndarray / pandas array from read_parquet - return list(history.tolist()) + # if hasattr(history, "tolist"): # numpy ndarray / pandas array from read_parquet + # return list(history.tolist()) if isinstance(history, str): text = history.strip() for parser in (ast.literal_eval, json.loads): @@ -110,14 +113,18 @@ def _format_trace_step(step) -> str: return str(step) -def _format_indexed_raw_trace(history, question) -> str: - """Format raw traces with stable zero-based message indices. +def _format_indexed_from_steps(steps, question) -> str: + """Format already-coerced steps with stable zero-based message indices. LLM evaluators can cite these indices in `evidence[i].idx` when no explicit state_id/response_id is available. EvidenceVerifier then maps `[0]`, `[1]`, ... blocks back to the quoted text. + + Takes the coerced ``steps`` directly so the SAME step list feeds the judges, + the deterministic validators, and the confirmer -- they cannot disagree about + the trace's span space (which is what silently diverged when each path + re-coerced ``history`` separately). """ - steps = _coerce_history_to_steps(history) lines = [ "USER QUESTION:", str(question), @@ -129,6 +136,11 @@ def _format_indexed_raw_trace(history, question) -> str: return "\n".join(lines) +def _format_indexed_raw_trace(history, question) -> str: + """Backward-compatible wrapper: coerce ``history`` then format.""" + return _format_indexed_from_steps(_coerce_history_to_steps(history), question) + + async def main( model_name: str, enable_tracing: bool, @@ -136,6 +148,7 @@ async def main( result_file_name: str = "findings_", folder_name: str = "who&when_hand_findings", from_idx: int = 0, + llm_confirm: bool = False, ): """Run all LLM findings-evaluators on each row of the dataset. @@ -146,6 +159,8 @@ async def main( result_file_name: Prefix for per-task JSON result files. folder_name: Subfolder (next to this script) where JSON files are written. from_idx: Skip rows before this index (resume support). + llm_confirm: If True, run the opt-in LLM confirmation + appointing layer + over the deterministic ``non_llm_validators`` findings. """ api_key = os.getenv("OPENROUTER_API_KEY") @@ -175,8 +190,14 @@ async def main( continue row = df.iloc[task] + # Coerce the trace ONCE, here, and thread the same `steps` object through + # the judges, the deterministic validators, and the confirmer. No + # tolist/to_pylist: in an environment where `history` already loads as a + # list of message dicts, this expands to the per-message spans; the point + # is that all three consumers share the identical span space. + steps = _coerce_history_to_steps(row["history"]) eval_input = RawTraceInput( - trace=_format_indexed_raw_trace(row["history"], row["question"]) + trace=_format_indexed_from_steps(steps, row["question"]) ) print(f"\n=== Evaluating Task {task} ===") @@ -203,49 +224,81 @@ async def main( judge_client, model, eval_input, - row["history"], + steps, task, trace_metadata, ground_truth_value, final_answer_verifier, result_file_name, folder_name, + llm_confirm, ) else: await _evaluate_task( model, eval_input, - row["history"], + steps, task, trace_metadata, ground_truth_value, final_answer_verifier, result_file_name, folder_name, + llm_confirm, ) -async def _run_all_metrics(model, eval_input: RawTraceInput) -> dict: - """Run every LLM evaluator on one task and return {metric_name: MetricResult}.""" +# Retries per evaluator. OpenRouter returns finish_reason='error' when the +# upstream (gemini) call fails transiently under load; those are retryable. +METRIC_MAX_ATTEMPTS = int(os.environ.get("METRIC_MAX_ATTEMPTS", "4")) + + +async def _run_all_metrics(model, eval_input: RawTraceInput) -> tuple[dict, dict]: + """Run every LLM evaluator on one task. + + Returns ``(findings_results, metric_status)`` — the latter carries per-metric + timing + token usage for the ``task_stats`` rollup. + """ findings_results: dict = {} + metric_status: dict = {} for metric_type in LLM_METRICS_TO_TEST: - print(f"\n--- Evaluating LLM metric: {metric_type.value} ---") - try: - metric = create_metric(metric_type, model) - result = await metric.evaluate(eval_input) - findings_results[metric_type.value] = result - print( - f"Metric: {result.metric_name} | findings: {len(result.findings)}" - ) - for finding in result.findings: + name = metric_type.value + print(f"\n--- Evaluating LLM metric: {name} ---") + metric = None + t0 = time.perf_counter() + for attempt in range(1, METRIC_MAX_ATTEMPTS + 1): + try: + metric = create_metric(metric_type, model) + result = await metric.evaluate(eval_input) + inp, out, tot = extract_usage(getattr(metric, "last_usage", None)) + findings_results[name] = result + metric_status[name] = { + "status": "ok", "duration_s": round(time.perf_counter() - t0, 3), + "input_tokens": inp, "output_tokens": out, "total_tokens": tot, + } print( - f" - [{finding.severity_estimate.value}/{finding.confidence_estimate.value}] " - f"{finding.problem_description[:120]}" + f"Metric: {result.metric_name} | findings: {len(result.findings)} " + f"| in={inp} out={out}" ) - except Exception as e: - print(f"Error evaluating {metric_type.value}: {e}") - continue - return findings_results + for finding in result.findings: + print( + f" - [{finding.severity_estimate.value}/{finding.confidence_estimate.value}] " + f"{finding.problem_description[:120]}" + ) + break + except Exception as e: + if attempt < METRIC_MAX_ATTEMPTS: + await asyncio.sleep(2 * attempt) + print(f" retry {attempt}/{METRIC_MAX_ATTEMPTS - 1} after {type(e).__name__}") + else: + inp, out, tot = extract_usage(getattr(metric, "last_usage", None)) + metric_status[name] = { + "status": "failed", "detail": f"{type(e).__name__}: {e}"[:300], + "duration_s": round(time.perf_counter() - t0, 3), + "input_tokens": inp, "output_tokens": out, "total_tokens": tot, + } + print(f"Error evaluating {name}: {e}") + return findings_results, metric_status def _serialize_findings(findings_results: dict) -> dict: @@ -329,24 +382,40 @@ def _save_results( async def _evaluate_task( model, eval_input: RawTraceInput, - history, + steps, task: int, trace_metadata: dict, ground_truth_value, final_answer_verifier: FinalAnswerVerifier, result_file_name: str, folder_name: str, + llm_confirm: bool = False, ): - """Run evaluators without Langfuse tracing.""" - findings_results = await _run_all_metrics(model, eval_input) + """Run evaluators without Langfuse tracing. + + ``steps`` is the trace already coerced by the caller; the validators and the + confirmer use the SAME object as the judges (no re-coercion) so their spans + are identical. + """ + _t0 = time.perf_counter() + findings_results, metric_status = await _run_all_metrics(model, eval_input) + task_stats = aggregate_task_stats(metric_status, time.perf_counter() - _t0) evidence_results = EvidenceVerifier().verify_all(findings_results, eval_input) final_answer_result = await _run_final_answer_verification( final_answer_verifier, - history, + steps, trace_metadata["ground_truth"], ) serializable_results = _serialize_findings(findings_results) - serializable_results["non_llm_validators"] = run_on_trace({"history": _coerce_history_to_steps(history)}) + serializable_results["task_stats"] = task_stats + # Same span space as the judges: run_on_trace + the confirmer read `steps`. + trace = {"history": steps} + serializable_results["non_llm_validators"] = run_on_trace(trace) + if llm_confirm: + # Confirmer runs inline, as another judge: it reads the WHOLE trace + # (read_window=None) to settle confirmed-vs-benign, and appoints the + # causal turn within the tight default window. + await confirm_trace_async(trace, serializable_results["non_llm_validators"], model) serializable_results["evidence_verification"] = _serialize_evidence_verification(evidence_results) if final_answer_result is not None: serializable_results["final_answer_verification"] = final_answer_result.model_dump(mode="json") @@ -364,15 +433,20 @@ async def _evaluate_task_traced( judge_client, model, eval_input: RawTraceInput, - history, + steps, task: int, trace_metadata: dict, ground_truth_value, final_answer_verifier: FinalAnswerVerifier, result_file_name: str, folder_name: str, + llm_confirm: bool = False, ): - """Run evaluators inside a Langfuse parent span; full findings go to span.output.""" + """Run evaluators inside a Langfuse parent span; full findings go to span.output. + + ``steps`` is the already-coerced trace shared with the judges (see + :func:`_evaluate_task`). + """ with judge_client.start_as_current_span( name=f"evaluate_task_{task}", input={"task_id": task, "trace_id": trace_metadata["task_id"]}, @@ -382,15 +456,22 @@ async def _evaluate_task_traced( tags=["maseval", "findings", f"task_id:{task}"] ) - findings_results = await _run_all_metrics(model, eval_input) + _t0 = time.perf_counter() + findings_results, metric_status = await _run_all_metrics(model, eval_input) + task_stats = aggregate_task_stats(metric_status, time.perf_counter() - _t0) evidence_results = EvidenceVerifier().verify_all(findings_results, eval_input) final_answer_result = await _run_final_answer_verification( final_answer_verifier, - history, + steps, trace_metadata["ground_truth"], ) serializable_results = _serialize_findings(findings_results) - serializable_results["non_llm_validators"] = run_on_trace({"history": _coerce_history_to_steps(history)}) + serializable_results["task_stats"] = task_stats + # Same span space as the judges (see _evaluate_task). + trace = {"history": steps} + serializable_results["non_llm_validators"] = run_on_trace(trace) + if llm_confirm: + await confirm_trace_async(trace, serializable_results["non_llm_validators"], model) serializable_results["evidence_verification"] = _serialize_evidence_verification(evidence_results) if final_answer_result is not None: serializable_results["final_answer_verification"] = final_answer_result.model_dump(mode="json") @@ -423,12 +504,24 @@ async def _evaluate_task_traced( if __name__ == "__main__": import argparse - df_hc = pd.read_parquet( - "hf://datasets/Kevin355/Who_and_When/Hand-Crafted.parquet" - ) - df_algo = pd.read_parquet( - "hf://datasets/Kevin355/Who_and_When/Algorithm-Generated.parquet" - ) + def _load_split(filename: str): + """Robustly load a Who&When split via cached hf_hub_download (the flaky + hf:// streaming path times out on this network).""" + from huggingface_hub import hf_hub_download + + last = None + for _ in range(5): + try: + path = hf_hub_download( + "Kevin355/Who_and_When", filename=filename, repo_type="dataset" + ) + return pd.read_parquet(path) + except Exception as exc: # noqa: BLE001 - transient network + last = exc + raise RuntimeError(f"Could not download {filename}: {last}") + + df_hc = _load_split("Hand-Crafted.parquet") + df_algo = _load_split("Algorithm-Generated.parquet") # Resume support: read FROM_IDX from env (default 0). Useful when the # script is interrupted and you want to continue from a specific task. @@ -444,26 +537,45 @@ async def _evaluate_task_traced( default="both", help="Which dataset(s) to evaluate.", ) + parser.add_argument( + "--llm-confirm", + action="store_true", + help="Run the opt-in LLM confirmation + appointing layer over the " + "deterministic non_llm_validators findings.", + ) + parser.add_argument( + "--folder", + default=None, + help="Override the output folder name (applied to the selected split).", + ) + parser.add_argument( + "--no-trace", + action="store_true", + help="Disable Langfuse tracing (use when LANGFUSE_* env vars are unset). " + "Findings output is identical either way.", + ) args = parser.parse_args() async def _run_selected(selected_run: str): if selected_run in ("algo", "both"): await main( model_name="google/gemini-2.5-flash", - enable_tracing=True, + enable_tracing=not args.no_trace, df=df_algo, result_file_name="gemini_findings_", - folder_name="who&when_algo_gemini_idx_msg_v2", + folder_name=args.folder or "who&when_algo_gemini_idx_msg_v2", from_idx=from_idx, + llm_confirm=args.llm_confirm, ) if selected_run in ("hc", "both"): await main( model_name="google/gemini-2.5-flash", - enable_tracing=True, + enable_tracing=not args.no_trace, df=df_hc, result_file_name="gemini_findings_", - folder_name="who&when_hand_gemini_idx_msg_v2", + folder_name=args.folder or "who&when_hand_gemini_idx_msg_v2", from_idx=from_idx, + llm_confirm=args.llm_confirm, ) asyncio.run(_run_selected(args.run)) diff --git a/examples/who_and_when/regen_validators_confirm.py b/examples/who_and_when/regen_validators_confirm.py new file mode 100644 index 0000000..ce74673 --- /dev/null +++ b/examples/who_and_when/regen_validators_confirm.py @@ -0,0 +1,110 @@ +"""Regenerate the deterministic ``non_llm_validators`` (+ LLM confirmation) on +properly-expanded Who&When traces, reusing the already-good LLM findings. + +Why this exists: HF parquet returns the ``history`` column as a numpy ndarray of +message dicts. The launcher's ``_coerce_history_to_steps`` only expands it after +the ndarray fix; runs made before that collapsed every message into a single +step (idx 0, agent None), so the stored ``non_llm_validators`` are degenerate and +appointing is dead. The 11 LLM evaluators in those same files were produced on a +24-step trace and are fine, so we do NOT re-run them (expensive). We only: + + 1. re-expand the trace correctly, + 2. re-run ``run_on_trace`` (regex, zero API) to get real per-step findings, + 3. re-confirm + appoint with the LLM (~1 call per finding-bearing trace), + +and write the augmented files to a sibling ``*_fixed`` folder (non-destructive). + +Usage: + python regen_validators_confirm.py \ + --src who&when_hand_gemini_llmconfirm_scratch \ + --parquet Hand-Crafted.parquet +""" + +from __future__ import annotations + +import argparse +import asyncio +import glob +import json +import os +import sys +from pathlib import Path + +from dotenv import load_dotenv + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +# Load the root .env (canonical creds) then a local one if present. +load_dotenv(ROOT / ".env") +load_dotenv(THIS_DIR / ".env") + +from maseval.validators import run_on_trace # noqa: E402 +from maseval.validators.llm_confirm import confirm_trace_async # noqa: E402 + +from apply_llm_confirm import _coerce_history_to_steps, _load_parquet # noqa: E402 + + +def _n_findings(nlv: dict) -> int: + return sum(len(m.get("findings", [])) for m in (nlv.get("metrics") or {}).values()) + + +async def main(src: str, parquet: str, model: str, from_idx: int = 0) -> None: + if not os.getenv("OPENROUTER_API_KEY"): + raise RuntimeError("OPENROUTER_API_KEY is not set (source the root .env).") + + src_dir = THIS_DIR / src + out_dir = THIS_DIR / f"{src}_fixed" + out_dir.mkdir(parents=True, exist_ok=True) + + df = _load_parquet(parquet) + files = sorted( + glob.glob(str(src_dir / "gemini_findings_*.json")), + key=lambda p: int(p.rsplit("_", 1)[1].split(".")[0]), + ) + print(f"{src}: {len(files)} findings files, {len(df)} rows -> {out_dir.name}") + + totals = {"confirmed": 0, "benign": 0, "uncertain": 0, "appointed": 0, "traces_with_findings": 0} + for fp in files: + i = int(fp.rsplit("_", 1)[1].split(".")[0]) + if i < from_idx: + continue + data = json.loads(Path(fp).read_text(encoding="utf-8")) + + # Re-expand the trace correctly and regenerate the deterministic layer. + trace = {"history": _coerce_history_to_steps(df.iloc[i]["history"])} + nlv = run_on_trace(trace) + n = _n_findings(nlv) + if n: + totals["traces_with_findings"] += 1 + try: + await confirm_trace_async(trace, nlv, model) + except Exception as exc: # noqa: BLE001 - one trace must not abort the run + print(f" [{i}] confirm error: {type(exc).__name__}: {exc}") + summary = nlv.get("llm_confirmation_summary", {}) + for k in ("confirmed", "benign", "uncertain", "appointed"): + totals[k] += int(summary.get(k, 0) or 0) + print(f" [{i}] {n} det-findings -> {summary}") + + data["non_llm_validators"] = nlv + (out_dir / Path(fp).name).write_text( + json.dumps(data, indent=2, default=str), encoding="utf-8" + ) + + print("\n=== regenerated validators + confirm ===") + print(f"finding-bearing traces: {totals['traces_with_findings']}") + print(f"per-finding verdicts: confirmed={totals['confirmed']} benign={totals['benign']} " + f"uncertain={totals['uncertain']} | appointed={totals['appointed']}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Regenerate validators+confirm on expanded traces.") + parser.add_argument("--src", default="who&when_hand_gemini_llmconfirm_scratch") + parser.add_argument("--parquet", default="Hand-Crafted.parquet") + parser.add_argument("--model", default="google/gemini-2.5-flash") + parser.add_argument("--from-idx", type=int, default=0) + args = parser.parse_args() + asyncio.run(main(src=args.src, parquet=args.parquet, model=args.model, from_idx=args.from_idx)) diff --git a/examples/who_and_when/score_none_plus_validators.py b/examples/who_and_when/score_none_plus_validators.py new file mode 100644 index 0000000..52b9344 --- /dev/null +++ b/examples/who_and_when/score_none_plus_validators.py @@ -0,0 +1,257 @@ +"""Score Who&When agent/step localization for verifier=none, optionally folding +in the deterministic ``non_llm_validators`` (with the LLM confirmation layer). + +Baseline is the verifier=``none`` prediction over the 11 LLM evaluators +(``diagnostic_accuracy.read_prediction_file(verifier_mode="none")``). On top of +that we OR-in the deterministic validator findings, under three verdict filters: + +* ``all`` -- every regex-detected finding counts (no confirmer gating); +* ``conf+unc`` -- findings the confirmer marked confirmed OR uncertain; +* ``confirmed`` -- only findings the confirmer marked confirmed. + +For each counted validator finding the predicted locus is the **appointed causal +turn** (``llm_confirmation.corrected_idx``), falling back to the finding's +evidence/surface idx when appointing returned null. The predicted AGENT is the +agent of the span at that idx, resolved from the trace history via +``who_and_when_to_spans`` (the finding itself carries no agent name). The extra +agent/idx are appended to the LLM ``none`` prediction (dedup, order preserved), +so this can only ADD recall, never remove an LLM prediction. + +Prints a comparison table: none (LLM only) vs none+validators under each filter. + +Usage: + python score_none_plus_validators.py \ + --pred-dir who&when_hand_gemini_llmconfirm_scratch_fixed \ + --parquet Hand-Crafted.parquet +""" + +from __future__ import annotations + +import argparse +import glob +import json +import sys +from pathlib import Path + +THIS_DIR = Path(__file__).resolve().parent +ROOT = THIS_DIR.parents[1] +for p in (ROOT / "src", THIS_DIR): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from maseval.diagnostic_accuracy import ( # noqa: E402 + _idx_matches_any, + _normalize_agent, + read_prediction_file, +) +from maseval.validators.base import who_and_when_to_spans # noqa: E402 + +from apply_llm_confirm import _coerce_history_to_steps, _load_parquet # noqa: E402 + +FILTERS = ("none_only", "all", "conf_unc", "confirmed") +FILTER_LABELS = { + "none_only": "none (LLM only)", + "all": "none + val (all)", + "conf_unc": "none + val (conf+unc)", + "confirmed": "none + val (confirmed)", +} + + +def _iter_val_findings(nlv: dict): + for m in (nlv.get("metrics") or {}).values(): + for f in m.get("findings", []): + yield f + + +def _val_locus_idx(finding: dict) -> str | None: + """Appointed causal turn if present, else the evidence/surface idx.""" + conf = finding.get("llm_confirmation") or {} + corrected = conf.get("corrected_idx") + if corrected is not None and str(corrected).strip(): + return str(corrected) + ev = finding.get("evidence") or [] + if ev and ev[0].get("idx") is not None: + return str(ev[0]["idx"]) + return None + + +def _passes(finding: dict, filt: str) -> bool: + if filt == "all": + return True + verdict = (finding.get("llm_confirmation") or {}).get("verdict") + if filt == "conf_unc": + return verdict in ("confirmed", "uncertain") + if filt == "confirmed": + return verdict == "confirmed" + return False + + +def _extend(agents: list[str], idxs: list[str], a: str | None, ix: str | None): + if ix is not None and ix not in idxs: + idxs.append(ix) + if a and a not in agents: + agents.append(a) + + +def score(pred_dir: str, parquet: str, step_tolerance: int = 1) -> dict: + pred_dir_p = THIS_DIR / pred_dir if not Path(pred_dir).is_absolute() else Path(pred_dir) + files = sorted( + glob.glob(str(pred_dir_p / "gemini_findings_*.json")), + key=lambda p: int(p.rsplit("_", 1)[1].split(".")[0]), + ) + if not files: + raise FileNotFoundError(f"No prediction files in {pred_dir_p}") + df = _load_parquet(parquet) + + # Accumulators per filter. + acc = {f: {"agent_top1": [], "agent_hit": [], "step_top1": [], "step_hit": [], "step_pm1": []} + for f in FILTERS} + matched = 0 + + for fp in files: + i = int(fp.rsplit("_", 1)[1].split(".")[0]) + row = df.iloc[i] + gold_agents = {_normalize_agent(a) for a in _as_list(row.get("mistake_agent"))} + gold_idxs = [str(x) for x in _as_list(row.get("mistake_step"))] + if not gold_agents and not gold_idxs: + continue + matched += 1 + + # Baseline: verifier=none over the LLM evaluators. + pred = read_prediction_file(fp, verifier_mode="none") + base_agents = list(pred.agents) + base_idxs = list(pred.idxs) + base_primary = pred.primary_agent + base_first = pred.first_idx + + # idx -> agent map from the (properly expanded) trace history. + steps = _coerce_history_to_steps(row["history"]) + spans = who_and_when_to_spans({"history": steps}) + idx_to_agent = {s["idx"]: s.get("agent") for s in spans} + + data = json.loads(Path(fp).read_text(encoding="utf-8")) + nlv = data.get("non_llm_validators") or {} + val_hits = list(_iter_val_findings(nlv)) + + for filt in FILTERS: + agents = list(base_agents) + idxs = list(base_idxs) + primary = base_primary + first = base_first + if filt != "none_only": + for finding in val_hits: + if not _passes(finding, filt): + continue + ix = _val_locus_idx(finding) + a = idx_to_agent.get(ix) if ix is not None else None + _extend(agents, idxs, a, ix) + if primary is None and a: + primary = a + if first is None and ix is not None: + first = ix + + pred_agents_norm = [_normalize_agent(a) for a in agents] + primary_norm = _normalize_agent(primary) if primary else None + if gold_agents: + acc[filt]["agent_top1"].append(bool(primary_norm and primary_norm in gold_agents)) + acc[filt]["agent_hit"].append(any(a in gold_agents for a in pred_agents_norm)) + if gold_idxs: + acc[filt]["step_top1"].append( + bool(first and _idx_matches_any(first, gold_idxs, tolerance=0))) + acc[filt]["step_hit"].append( + any(_idx_matches_any(ix, gold_idxs, tolerance=0) for ix in idxs)) + acc[filt]["step_pm1"].append( + any(_idx_matches_any(ix, gold_idxs, tolerance=step_tolerance) for ix in idxs)) + + def mean(xs): + return sum(xs) / len(xs) if xs else None + + summary = {f: {k: mean(v) for k, v in acc[f].items()} for f in FILTERS} + return {"matched": matched, "summary": summary} + + +def _as_list(value): + if value is None: + return [] + if isinstance(value, (list, tuple, set)): + return [x for x in value if x is not None and str(x).strip()] + if hasattr(value, "tolist"): + return [x for x in value.tolist() if x is not None and str(x).strip()] + text = str(value).strip() + return [text] if text and text.lower() not in {"none", "nan", "null", ""} else [] + + +def _pct(v): + return f"{v * 100:5.1f}%" if isinstance(v, (int, float)) else " —" + + +def _render(res: dict, split: str = "") -> str: + s = res["summary"] + rows = [ + ("Agent Top-1", "agent_top1"), + ("Agent Hit", "agent_hit"), + ("Step Top-1", "step_top1"), + ("Step Hit", "step_hit"), + ("Step Hit ±1", "step_pm1"), + ] + header = f"{'Metric':<14}" + "".join(FILTER_LABELS[f].rjust(22) for f in FILTERS) + lines = [ + f"=== none + non_llm_validators (llm-confirm), {split.upper() or '?'} (matched {res['matched']}) ===", + "Locus = appointed causal turn (corrected_idx), fallback surface idx; " + "agent = agent at that idx.", + "", + header, + "-" * len(header), + ] + for label, key in rows: + lines.append(f"{label:<14}" + "".join(_pct(s[f][key]).rjust(22) for f in FILTERS)) + return "\n".join(lines) + + +def _markdown(res: dict, pred_dir: str, parquet: str) -> str: + s = res["summary"] + rows = [ + ("Agent Top-1", "agent_top1"), ("Agent Hit", "agent_hit"), + ("Step Top-1", "step_top1"), ("Step Hit", "step_hit"), ("Step Hit ±1", "step_pm1"), + ] + md = [ + "# Who&When — none + non_llm_validators (LLM-confirm) fold-in", + "", + f"- Split / annotations: `{parquet}`", + f"- Predictions: `{pred_dir}`", + f"- Matched traces: {res['matched']}", + "- Baseline: verifier=`none` over the 11 LLM evaluators.", + "- Fold-in locus: appointed causal turn (`corrected_idx`), fallback surface idx; " + "agent = agent at that idx (via `who_and_when_to_spans`).", + "- Filters: `all` = every regex finding; `conf+unc` = confirmed|uncertain; " + "`confirmed` = confirmed only.", + "", + "| Metric | " + " | ".join(FILTER_LABELS[f] for f in FILTERS) + " |", + "|---|" + "---:|" * len(FILTERS), + ] + for label, key in rows: + md.append("| " + label + " | " + " | ".join(_pct(s[f][key]).strip() for f in FILTERS) + " |") + md.append("") + return "\n".join(md) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Score none + validators (llm-confirm) fold-in.") + parser.add_argument("--pred-dir", default="who&when_hand_gemini_llmconfirm_scratch_fixed") + parser.add_argument("--parquet", default="Hand-Crafted.parquet") + parser.add_argument("--step-tolerance", type=int, default=1) + parser.add_argument("--split", default="hc", help="Label for the output report filename.") + args = parser.parse_args() + res = score(args.pred_dir, args.parquet, step_tolerance=args.step_tolerance) + print(_render(res, args.split)) + + reports_dir = THIS_DIR / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + json_path = reports_dir / f"none_plus_validators_{args.split}.json" + md_path = reports_dir / f"none_plus_validators_{args.split}.md" + json_path.write_text(json.dumps( + {"pred_dir": args.pred_dir, "parquet": args.parquet, + "filter_labels": FILTER_LABELS, **res}, indent=2), encoding="utf-8") + md_path.write_text(_markdown(res, args.pred_dir, args.parquet), encoding="utf-8") + print(f"\nSaved: {md_path}") + print(f"Saved: {json_path}") diff --git a/pyproject.toml b/pyproject.toml index dda0411..66f40aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "pydantic>=2.11.9", "pydantic-ai>=2.0.0", "python-dotenv>=1.1.1", - "langfuse>=3.6.1", + "langfuse==3.14.4", "deepeval>=3.6.7", "ftfy>=6.3.1", "langchain_core>=0.3.76", diff --git a/src/maseval/metrics/base.py b/src/maseval/metrics/base.py index 697d702..ce83204 100644 --- a/src/maseval/metrics/base.py +++ b/src/maseval/metrics/base.py @@ -35,6 +35,9 @@ def __init__( self.prompt_template = prompt_template self.result_type = result_type self.deps_type = deps_type + # Token usage from the most recent evaluate() run (pydantic_ai usage object, + # or None). Lets launchers meter per-metric input/output tokens. + self.last_usage = None # Create the pydantic AI agent with dependency injection self.agent = Agent( @@ -92,6 +95,7 @@ async def evaluate(self, eval_input: EvaluationInput | RawTraceInput) -> MetricR "Please evaluate the provided data according to the metric criteria.", deps=eval_input, ) + self._record_usage(result) return self._convert_result(result.output) except Exception as e: if "finish_reason" in str(e) and self.result_type: @@ -99,10 +103,22 @@ async def evaluate(self, eval_input: EvaluationInput | RawTraceInput) -> MetricR agent_text.system_prompt = self.agent._system_prompts[0] response = await agent_text.run("... + Return JSON", deps=eval_input) + self._record_usage(response) parsed = json.loads(response.output) return self._convert_result(self.result_type(**parsed)) raise + def _record_usage(self, run_result) -> None: + """Stash the run's token usage on ``self.last_usage`` (best-effort). + + ``usage`` is a method on some pydantic_ai versions and a property on others. + """ + try: + usage = getattr(run_result, "usage", None) + self.last_usage = usage() if callable(usage) else usage + except Exception: # noqa: BLE001 - usage metering must never break evaluation + self.last_usage = None + def _convert_result(self, result: FindingsResult) -> MetricResult: """Convert findings result to MetricResult. diff --git a/src/maseval/run_stats.py b/src/maseval/run_stats.py new file mode 100644 index 0000000..1de93ca --- /dev/null +++ b/src/maseval/run_stats.py @@ -0,0 +1,52 @@ +"""Per-task token/latency metering helpers for the example launchers. + +A launcher times each LLM metric (``time.perf_counter``) and reads the metric's +``last_usage`` (set by ``metrics.base.LLMMetric.evaluate``) into a per-metric +record, then rolls those up with :func:`aggregate_task_stats` into a ``task_stats`` +block it stores on each result file — so runs are cost/latency-auditable after the +fact (see ``who&when_*_v9`` runs). + +Per-metric record shape: + {"status": "ok"|"failed", "reason"?, "detail"?, + "duration_s", "input_tokens", "output_tokens", "total_tokens"} +""" + +from __future__ import annotations + +from typing import Any + + +def extract_usage(usage: Any) -> tuple[int, int, int]: + """Pull ``(input, output, total)`` tokens from a pydantic_ai usage object, + tolerating naming differences across versions. Returns zeros when unavailable.""" + if usage is None: + return 0, 0, 0 + + def pick(*names: str) -> int: + for n in names: + v = getattr(usage, n, None) + if v is not None: + try: + return int(v) + except (TypeError, ValueError): + continue + return 0 + + inp = pick("input_tokens", "request_tokens", "prompt_tokens") + out = pick("output_tokens", "response_tokens", "completion_tokens") + tot = pick("total_tokens") or (inp + out) + return inp, out, tot + + +def aggregate_task_stats(metric_status: dict, wall_s: float) -> dict: + """Roll up per-metric timing/tokens for one task into a ``task_stats`` block.""" + vals = list(metric_status.values()) + return { + "wall_s": round(wall_s, 3), + "sum_metric_s": round(sum(v.get("duration_s") or 0 for v in vals), 3), + "input_tokens": sum(v.get("input_tokens") or 0 for v in vals), + "output_tokens": sum(v.get("output_tokens") or 0 for v in vals), + "total_tokens": sum(v.get("total_tokens") or 0 for v in vals), + "metrics_ok": sum(1 for v in vals if v.get("status") == "ok"), + "metrics_failed": sum(1 for v in vals if v.get("status") == "failed"), + } diff --git a/src/maseval/validators/llm_confirm.py b/src/maseval/validators/llm_confirm.py index ed4c1c7..3dd48c3 100644 --- a/src/maseval/validators/llm_confirm.py +++ b/src/maseval/validators/llm_confirm.py @@ -261,12 +261,22 @@ def build_llm_input( raw_spans: list[dict[str, Any]], window: int = _WINDOW, max_span_chars: int = _MAX_SPAN_CHARS, + read_window: int | None = None, ) -> tuple[dict[str, Any], dict[str, tuple[str, dict[str, Any]]]]: """Assemble the per-trace LLM payload and an id->(metric_name, finding) map. - Candidate spans for appointing are a ``±window`` neighborhood (flattened - pre-order) around each finding's current span. The ``spans`` dict carries the - full raw text of every referenced span, deduped across findings. + Two independent knobs control span exposure (kept separate deliberately): + + * **appoint candidates** (``window``): ``corrected_idx`` may only be a span in + the ``±window`` neighborhood of a finding's current span. On Who&When every + real appointment lands within ±4 of the surface span, so this stays tight — + it bounds where the causal turn can be, and ``_apply`` enforces it. + * **reading view** (``read_window``): which spans' full text the model may + *read* to judge confirmed-vs-benign. Confirmation must trace FORWARD ("did a + later span actually deliver the needed data?"), which a ±window cannot reach. + ``read_window=None`` exposes the WHOLE trace (judge-like view); an int + exposes a forward-biased neighborhood (``-window`` back, ``+read_window`` + forward) to bound context on very large traces. """ order = {s["idx"]: i for i, s in enumerate(raw_spans)} by_id = {s["idx"]: s for s in raw_spans} @@ -274,7 +284,7 @@ def build_llm_input( findings_payload: list[dict[str, Any]] = [] id_map: dict[str, tuple[str, dict[str, Any]]] = {} - referenced: set[str] = set() + read_ids: set[str] = set() for fid, metric_name, finding in _iter_findings(result): id_map[fid] = (metric_name, finding) @@ -282,13 +292,18 @@ def build_llm_input( candidates: list[str] = [] if cur is not None and cur in order: i = order[cur] + # Appoint candidates: tight symmetric window. lo, hi = max(0, i - window), min(n, i + window + 1) candidates = [raw_spans[j]["idx"] for j in range(lo, hi)] + # Reading view: forward-biased (or whole trace when read_window is None). + if read_window is None: + r_lo, r_hi = 0, n + else: + r_lo, r_hi = max(0, i - window), min(n, i + read_window + 1) + read_ids.update(raw_spans[j]["idx"] for j in range(r_lo, r_hi)) elif cur is not None: candidates = [cur] - referenced.update(candidates) - if cur is not None: - referenced.add(cur) + read_ids.add(cur) ev = finding.get("evidence") or [] findings_payload.append( { @@ -302,8 +317,12 @@ def build_llm_input( } ) + # When read_window is None, expose every span (deduped) regardless of findings. + if read_window is None: + read_ids.update(by_id.keys()) + spans_payload: dict[str, Any] = {} - for sid in referenced: + for sid in read_ids: s = by_id.get(sid) if s is None: continue @@ -404,17 +423,23 @@ async def confirm_trace_async( result: dict[str, Any], model: Any, agent: Any | None = None, + read_window: int | None = None, ) -> dict[str, Any]: """Confirm + appoint the deterministic ``result`` for one trace (in place). No-op (returns ``result`` unchanged) when there are no deterministic findings. Attaches ``finding["llm_confirmation"]`` to every finding and records a ``result["llm_confirmation_summary"]`` roll-up. + + ``read_window`` sets the confirmer's reading view (see :func:`build_llm_input`): + ``None`` gives it the WHOLE trace like a judge (right for small traces such as + Who&When); an int forward-bounds context for very large traces. Appointing + stays on the tight ``_WINDOW`` regardless. """ if not any(True for _ in _iter_findings(result)): return result _fmt, raw = build_raw_spans(trace) - llm_input, id_map = build_llm_input(result, raw) + llm_input, id_map = build_llm_input(result, raw, read_window=read_window) agent = agent or build_agent(model) run = await agent.run(json.dumps(llm_input, ensure_ascii=False, default=str)) trace_conf: TraceConfirmation = run.output @@ -427,6 +452,9 @@ def confirm_trace( result: dict[str, Any], model: Any, agent: Any | None = None, + read_window: int | None = None, ) -> dict[str, Any]: """Synchronous convenience wrapper around :func:`confirm_trace_async`.""" - return asyncio.run(confirm_trace_async(trace, result, model, agent=agent)) + return asyncio.run( + confirm_trace_async(trace, result, model, agent=agent, read_window=read_window) + )