feat: Weave (W&B) integration for LLM/agent tracing (#228)

Weave auto-patches OpenAI and Anthropic clients after weave.init(),
giving automatic tracing of every VLM call with prompts, responses,
costs, and latency in hierarchical trace trees.

Integration points:
- vlm_call() — @weave_op: all planner/grounder/evaluator calls traced
- vlm_judge() — @weave_op: milestone evaluation traced
- DemoExecutor.run() + _execute_step() — @weave_op: episode trace tree
- PlannerGrounderAgent.act() + _call_planner() — @weave_op: agent decisions
- WandbLogger.init() — calls weave_init() alongside wandb.init()

When weave is not installed, all decorators are zero-cost passthrough.
weave>=0.50.0 added to [wandb] optional extra.

76/76 tests pass.

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: abrichr

openadapt_evals/agents/demo_executor.py

@@ -30,6 +30,11 @@
 from openadapt_evals.adapters.base import BenchmarkAction, BenchmarkObservation
 from openadapt_evals.demo_library import Demo, DemoStep
 
+try:
+    from openadapt_evals.integrations.weave_integration import weave_op
+except ImportError:
+    weave_op = lambda fn: fn  # noqa: E731
+
 logger = logging.getLogger(__name__)
 
 
@@ -72,6 +77,7 @@ def __init__(
         self._step_delay = step_delay
         self._recovery_budget = recovery_budget
 
+    @weave_op
     def run(
         self,
         env,  # RLEnvironment
@@ -176,6 +182,7 @@ def run(
 
         return score, screenshots
 
+    @weave_op
     def _execute_step(
         self,
         step: DemoStep,

openadapt_evals/agents/planner_grounder_agent.py

@@ -67,6 +67,11 @@
     format_accessibility_tree,
 )
 
+try:
+    from openadapt_evals.integrations.weave_integration import weave_op
+except ImportError:
+    weave_op = lambda fn: fn  # noqa: E731
+
 if TYPE_CHECKING:
     from openadapt_evals.training.planner_cache import PlannerCache
     from openadapt_evals.training.trajectory_logger import PlannerTrajectoryLogger
@@ -257,6 +262,7 @@ def __init__(
             grounder_provider,
         )
 
+    @weave_op
     def act(
         self,
         observation: BenchmarkObservation,
@@ -660,6 +666,7 @@ def _check_action_loop(self) -> str:
 
         return ""
 
+    @weave_op
     def _call_planner(
         self,
         observation: BenchmarkObservation,

openadapt_evals/integrations/__init__.py

@@ -3,6 +3,7 @@
 This module provides integrations with:
 - Weights & Biases (wandb) for experiment tracking and report generation
 - W&B callback functions for the standalone GRPO trainer
+- Weave (W&B) for LLM/agent execution tracing
 """
 
 from openadapt_evals.integrations.wandb_logger import WandbLogger
@@ -33,9 +34,18 @@
     generate_demo_report = None
     _REPORTS_AVAILABLE = False
 
+# Weave tracing (optional — no-op when weave not installed)
+try:
+    from openadapt_evals.integrations.weave_integration import weave_init, weave_op
+except ImportError:
+    weave_init = None
+    weave_op = None
+
 __all__ = [
     "WandbLogger",
     "WandbReportGenerator",
+    "weave_init",
+    "weave_op",
     "wandb_model_loaded",
     "wandb_rollout_logger",
     "wandb_step_logger",

openadapt_evals/integrations/wandb_logger.py

@@ -135,6 +135,14 @@ def init(self) -> wandb.Run:
             benchmark = self.config.get("benchmark", "eval")
             self.name = f"{benchmark}_{model_id}_{timestamp}"
 
+        # Initialize Weave tracing alongside W&B — auto-patches
+        # OpenAI/Anthropic clients for VLM call tracing.
+        try:
+            from openadapt_evals.integrations.weave_integration import weave_init
+            weave_init(self.project)
+        except Exception:
+            pass
+
         self._run = wandb.init(
             project=self.project,
             entity=self.entity,

openadapt_evals/integrations/weave_integration.py

@@ -0,0 +1,79 @@
+"""Weave (W&B) integration for LLM/agent tracing.
+
+Weave auto-patches OpenAI and Anthropic clients after ``weave.init()``,
+giving automatic tracing of every VLM call (planner, grounder, evaluator)
+with prompts, responses, costs, and latency in hierarchical trace trees.
+
+Usage:
+    from openadapt_evals.integrations.weave_integration import weave_init, weave_op
+
+    # Initialize once at startup (alongside wandb.init if used)
+    weave_init("openadapt-evals")
+
+    # Decorate functions for explicit trace tree structure
+    @weave_op
+    def my_agent_step(screenshot, instruction):
+        ...
+
+When weave is not installed, ``weave_init`` is a no-op and ``weave_op``
+passes through the function unchanged. No runtime cost.
+"""
+
+from __future__ import annotations
+
+import functools
+import logging
+from typing import Any, Callable, TypeVar
+
+logger = logging.getLogger(__name__)
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+_weave_initialized = False
+
+
+def weave_init(project: str = "openadapt-evals") -> bool:
+    """Initialize Weave tracing.
+
+    Call once at startup. Auto-patches OpenAI and Anthropic clients
+    so all subsequent VLM calls are traced automatically.
+
+    Args:
+        project: Weave project name (appears in W&B UI).
+
+    Returns:
+        True if Weave initialized successfully, False otherwise.
+    """
+    global _weave_initialized
+    if _weave_initialized:
+        return True
+
+    try:
+        import weave
+        weave.init(project)
+        _weave_initialized = True
+        logger.info("Weave tracing initialized (project=%s)", project)
+        return True
+    except ImportError:
+        logger.debug("weave not installed — tracing disabled")
+        return False
+    except Exception as exc:
+        logger.warning("Weave init failed: %s", exc)
+        return False
+
+
+def weave_op(fn: F) -> F:
+    """Decorator that wraps a function with ``@weave.op`` if available.
+
+    When weave is installed, the decorated function appears in trace
+    trees with its arguments, return value, and execution time.
+
+    When weave is not installed, this is a zero-cost passthrough.
+    """
+    try:
+        import weave
+        return weave.op(fn)  # type: ignore[return-value]
+    except ImportError:
+        return fn
+    except Exception:
+        return fn

openadapt_evals/vlm.py

@@ -16,6 +16,13 @@
 _DEFAULT_TIMEOUT = 120  # seconds — prevents indefinite hangs on API calls
 
 
+try:
+    from openadapt_evals.integrations.weave_integration import weave_op
+except ImportError:
+    weave_op = lambda fn: fn  # noqa: E731
+
+
+@weave_op
 def vlm_call(
     prompt: str,
     *,

openadapt_evals/vlm_evaluator.py

@@ -42,6 +42,13 @@
 Respond with ONLY the JSON object."""
 
 
+try:
+    from openadapt_evals.integrations.weave_integration import weave_op
+except ImportError:
+    weave_op = lambda fn: fn  # noqa: E731
+
+
+@weave_op
 def vlm_judge(
     screenshot: bytes,
     description: str,

pyproject.toml

@@ -89,8 +89,10 @@ viewer = [
     "flask-cors>=4.0.0",
 ]
 wandb = [
-    # Weights & Biases for experiment tracking
+    # Weights & Biases for experiment tracking + Weave agent tracing
     "wandb>=0.16.0",
+    # Weave: LLM/agent tracing with auto-patched OpenAI/Anthropic clients
+    "weave>=0.50.0",
 ]
 training = [
     # Perceptual hashing for planner cache (falls back to MD5 if not installed)