diff --git a/.gitignore b/.gitignore index 0142b230e3..f8c8f762db 100644 --- a/.gitignore +++ b/.gitignore @@ -11,12 +11,14 @@ **/*.pem **/*dont_commit_me* web/packages/agenta-api-client/dist/ +web/tsconfig.tsbuildinfo __pycache__/ **/__pycache__/ **/error-*.log **/*.logs +**/*.log /.nox/ /docs/_build/ diff --git a/AGENTS.md b/AGENTS.md index 2201a3091c..60f5b19ff8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,6 +176,26 @@ Concrete examples: - Legacy app storage marker (`WORKFLOW_MARKER_KEY`): `api/oss/src/core/applications/service.py` - Legacy dedup key normalization (`__dedup_id__` <-> `testcase_dedup_id`): `api/oss/src/apis/fastapi/testsets/router.py` +### Alembic migration chains (OSS + EE) + +Migrations live in two separate, parallel chains that must each resolve to a +single head: +- OSS: `api/oss/databases/postgres/migrations/core/versions/` +- EE: `api/ee/databases/postgres/migrations/core/versions/` + +Rules: +- After adding/editing/renaming any migration, verify each chain has exactly ONE + head with the bundled tool: from each `.../migrations/` directory run + `python3 find_head.py core` and confirm the `Heads:` list has a single entry. + Run it for BOTH OSS and EE. +- New migrations chain linearly after the existing head — never fork off an older + node (a fork produces two heads; alembic then can't resolve a linear upgrade). +- Revision ids must be globally unique within a chain. A duplicate id makes + alembic silently skip one file (the migration never runs). +- The EE chain and the OSS chain are parallel, so an OSS migration chains after + the OSS head while its EE counterpart chains after the EE head — same revision id, + possibly different `down_revision`. + ### Router and function style conventions Router style: diff --git a/api/ee/databases/postgres/migrations/core/data_migrations/applications_workflow.py b/api/ee/databases/postgres/migrations/core/data_migrations/applications_workflow.py index 0c078ec15c..2f8216603f 100644 --- a/api/ee/databases/postgres/migrations/core/data_migrations/applications_workflow.py +++ b/api/ee/databases/postgres/migrations/core/data_migrations/applications_workflow.py @@ -254,9 +254,11 @@ def check_url_safety(cls, v: Any) -> Any: # noqa: N805 return v from oss.src.dbs.postgres.git.mappings import map_dto_to_dbe - from oss.src.dbs.postgres.shared.engine import engine as db_engine + from oss.src.dbs.postgres.shared.engine import get_transactions_engine from datetime import datetime, timezone + db_engine = get_transactions_engine() + workflow_create = WorkflowCreate( **application_create.model_dump(mode="json"), ) @@ -267,7 +269,7 @@ def check_url_safety(cls, v: Any) -> Any: # noqa: N805 # Avoid slug collision with existing workflow artifacts (e.g. evaluators) artifact_slug = git_artifact_create.slug - async with db_engine.core_session() as session: + async with db_engine.session() as session: existing = ( await session.execute( select(WorkflowArtifactDBE).filter( @@ -298,7 +300,7 @@ def check_url_safety(cls, v: Any) -> Any: # noqa: N805 dto=artifact_dto, ) - async with db_engine.core_session() as session: + async with db_engine.session() as session: session.add(artifact_dbe) await session.commit() @@ -364,7 +366,7 @@ def check_url_safety(cls, v: Any) -> Any: # noqa: N805 dto=variant_dto, ) - async with db_engine.core_session() as session: + async with db_engine.session() as session: session.add(variant_dbe) await session.commit() @@ -415,7 +417,7 @@ def check_url_safety(cls, v: Any) -> Any: # noqa: N805 dto=revision_dto, ) - async with db_engine.core_session() as session: + async with db_engine.session() as session: session.add(revision_dbe) await session.commit() diff --git a/api/ee/databases/postgres/migrations/core/env.py b/api/ee/databases/postgres/migrations/core/env.py index 492b4f9a69..1faed28fe5 100644 --- a/api/ee/databases/postgres/migrations/core/env.py +++ b/api/ee/databases/postgres/migrations/core/env.py @@ -6,7 +6,7 @@ from alembic import context -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.utils.env import env from oss.src.dbs.postgres.shared.base import Base # Side-effect imports: register SQLAlchemy models with Base.metadata @@ -29,7 +29,7 @@ # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config -config.set_main_option("sqlalchemy.url", engine.postgres_uri_core) # type: ignore +config.set_main_option("sqlalchemy.url", env.postgres.uri_core) # Interpret the config file for Python logging. diff --git a/api/ee/databases/postgres/migrations/core/versions/a1d2e3f4a5b6_add_default_evaluation_queues.py b/api/ee/databases/postgres/migrations/core/versions/a1d2e3f4a5b6_add_default_evaluation_queues.py new file mode 100644 index 0000000000..ca5c922224 --- /dev/null +++ b/api/ee/databases/postgres/migrations/core/versions/a1d2e3f4a5b6_add_default_evaluation_queues.py @@ -0,0 +1,41 @@ +"""add default evaluation queues + +Revision ID: a1d2e3f4a5b6 +Revises: e6f7a8b9c0d2 +Create Date: 2026-05-15 00:00:00 + +Previously shared revision id `a1b2c3d4e5f6` with +`drop_corrupted_metrics_for_some_runs`, so alembic skipped it and the index +below never ran. Renamed to `a1d2e3f4a5b6`. The EE chain extends past the shared +OSS head `e6f7a8b9c0d1` with EE-only migrations +(`9d3e8f0a1b2c -> a1b2c3d4e5f7 -> b2c3d4e5f7a8`), so this EE copy chains after +`b2c3d4e5f7a8` while the OSS copy chains after `e6f7a8b9c0d1`. + +The partial unique index covers ALL default queues (active or archived), so +there is at most ONE default queue row per (project_id, run_id) for the lifetime +of the run. Archiving a default does NOT free the slot — the single row is +archived/unarchived in place by reconcile, and user-facing archive of a default +is forbidden in the service layer. +""" + +from typing import Sequence, Union + +from alembic import op + +revision: str = "a1d2e3f4a5b6" +down_revision: Union[str, None] = "e6f7a8b9c0d2" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("DROP INDEX IF EXISTS ux_evaluation_queues_default_per_run") + op.execute(""" + CREATE UNIQUE INDEX ux_evaluation_queues_default_per_run + ON evaluation_queues (project_id, run_id) + WHERE (flags ->> 'is_default')::boolean = true + """) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ux_evaluation_queues_default_per_run") diff --git a/api/ee/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py b/api/ee/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py new file mode 100644 index 0000000000..30da5d2a86 --- /dev/null +++ b/api/ee/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py @@ -0,0 +1,294 @@ +"""backfill default evaluation queues + +Revision ID: a2b3c4d5e6f8 +Revises: a1d2e3f4a5b6 +Create Date: 2026-05-15 00:10:00 + +Backfills source-family flags (`has_traces` / `has_testcases` / `has_queries` / +`has_testsets`) to match the runtime derivation rule, then mass-creates default +queues per the runtime policy and recomputes `is_queue`. The query/testset +recompute keys on exact reference-key presence, not a substring match. + +Performance shape: runs are processed in keyset-paginated chunks (by +`evaluation_runs.id`). For each chunk a single COMPUTE select does the heavy +`jsonb_array_elements` step scan and the per-run default-queue existence check +ONCE (no correlated `EXISTS`/`JOIN` re-evaluated per written row). Python then +decides per run what to do, and the writes are cheap set-based `unnest` updates +keyed by id — no subqueries in the mutation path. Every step is idempotent (the +flag rebuild is deterministic; runs that already have a default queue are not +re-created), so a re-run is safe. +""" + +import json +from typing import List, Sequence, Union + +import click +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision: str = "a2b3c4d5e6f8" +down_revision: Union[str, None] = "a1d2e3f4a5b6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +BATCH_SIZE = 100 + + +# Keyset pagination over run ids. id > cursor with ORDER BY id is index-friendly +# (id is part of the PK) and stable; LIMIT bounds each chunk. Compared as text so +# the cursor is a plain string seeded as "" below every UUID. +_NEXT_RUN_IDS = sa.text(""" + SELECT id::text + FROM evaluation_runs + WHERE id::text > :cursor + ORDER BY id::text + LIMIT :batch +""") + +# One heavy pass per chunk. Derives the source-family flags from the run's steps +# (the `jsonb_array_elements` scan that mirrors dbs/postgres/evaluations/utils.py) +# and, via a single LEFT JOIN over the chunk's runs, reports whether each run +# already has a default queue (any state) and an ACTIVE default queue. Direct +# sources (`has_traces`/`has_testcases`) come from the exact step key on a +# reference-less input; reference-backed sources (`has_queries`/`has_testsets`) +# from exact-key presence (JSONB `?`), not a substring match that would misfire +# on `query_anchor` / `testset_metadata`. +_COMPUTE_CHUNK = sa.text(""" + SELECT + r.id::text AS run_id, + r.project_id::text AS project_id, + r.created_by_id::text AS created_by_id, + COALESCE(r.status, 'running') AS status, + COALESCE((r.flags ->> 'has_human')::boolean, false) AS has_human, + EXISTS ( + SELECT 1 + FROM jsonb_array_elements(COALESCE(r.data::jsonb -> 'steps', '[]'::jsonb)) AS step + WHERE step ->> 'type' = 'input' + AND COALESCE(step -> 'references', '{}'::jsonb) = '{}'::jsonb + AND lower(COALESCE(step ->> 'key', '')) IN ('traces', 'query-direct') + ) AS has_traces, + EXISTS ( + SELECT 1 + FROM jsonb_array_elements(COALESCE(r.data::jsonb -> 'steps', '[]'::jsonb)) AS step + WHERE step ->> 'type' = 'input' + AND COALESCE(step -> 'references', '{}'::jsonb) = '{}'::jsonb + AND lower(COALESCE(step ->> 'key', '')) IN ('testcases', 'testset-direct') + ) AS has_testcases, + EXISTS ( + SELECT 1 + FROM jsonb_array_elements(COALESCE(r.data::jsonb -> 'steps', '[]'::jsonb)) AS step + WHERE step ->> 'type' = 'input' + AND COALESCE(step -> 'references', '{}'::jsonb) ? 'query_revision' + ) AS has_queries, + EXISTS ( + SELECT 1 + FROM jsonb_array_elements(COALESCE(r.data::jsonb -> 'steps', '[]'::jsonb)) AS step + WHERE step ->> 'type' = 'input' + AND COALESCE(step -> 'references', '{}'::jsonb) ? 'testset_revision' + ) AS has_testsets, + bool_or((q.flags ->> 'is_default')::boolean) AS has_any_default, + bool_or( + (q.flags ->> 'is_default')::boolean AND q.deleted_at IS NULL + ) AS has_active_default + FROM evaluation_runs r + LEFT JOIN evaluation_queues q + ON q.project_id = r.project_id + AND q.run_id = r.id + WHERE r.id = ANY(CAST(:ids AS uuid[])) + GROUP BY r.id, r.project_id, r.created_by_id, r.status, r.flags +""") + +# Cheap set-based flag write: merge the precomputed flag object onto each run by +# id. No subqueries; `unnest` pairs each id with its jsonb patch. +_APPLY_FLAGS = sa.text(""" + UPDATE evaluation_runs r + SET flags = COALESCE(r.flags, '{}'::jsonb) || patch.flags::jsonb + FROM unnest( + CAST(:ids AS uuid[]), + CAST(:flags AS jsonb[]) + ) AS patch(id, flags) + WHERE r.id = patch.id +""") + +# Cheap create: one row per run id that needs a default queue (computed in +# Python — has_human and no existing default). No NOT EXISTS in the write path. +_CREATE_DEFAULT_QUEUES = sa.text(""" + INSERT INTO evaluation_queues ( + project_id, + id, + created_at, + created_by_id, + flags, + data, + status, + run_id + ) + SELECT + CAST(src.project_id AS uuid), + gen_random_uuid(), + CURRENT_TIMESTAMP, + CAST(src.created_by_id AS uuid), + jsonb_build_object('is_default', true, 'is_sequential', false), + '{}'::json, + src.status, + CAST(src.run_id AS uuid) + FROM unnest( + CAST(:project_ids AS text[]), + CAST(:run_ids AS text[]), + CAST(:created_by_ids AS text[]), + CAST(:statuses AS text[]) + ) AS src(project_id, run_id, created_by_id, status) +""") + +# Cheap archive: flip deleted_at on the active default queue of each run id that +# should no longer have one (computed in Python). Keyed by (project_id, run_id). +_ARCHIVE_STALE_DEFAULT_QUEUES = sa.text(""" + UPDATE evaluation_queues q + SET deleted_at = CURRENT_TIMESTAMP, + deleted_by_id = CAST(src.created_by_id AS uuid) + FROM unnest( + CAST(:project_ids AS text[]), + CAST(:run_ids AS text[]), + CAST(:created_by_ids AS text[]) + ) AS src(project_id, run_id, created_by_id) + WHERE q.project_id = CAST(src.project_id AS uuid) + AND q.run_id = CAST(src.run_id AS uuid) + AND (q.flags ->> 'is_default')::boolean = true + AND q.deleted_at IS NULL +""") + + +def _next_run_ids(connection: Connection, *, cursor: str) -> List[str]: + result = connection.execute( + _NEXT_RUN_IDS, + {"cursor": cursor, "batch": BATCH_SIZE}, + ) + return [row[0] for row in result.fetchall()] + + +def _flags_patch(row) -> str: + # Build the jsonb patch for a run: the four source-family flags plus the + # recomputed is_queue, matching the queue state this chunk produces. + # + # is_queue is true iff the run has a human step AND ends this chunk with an + # ACTIVE default queue. Mirroring _reconcile_default_queue and the create + # guard below: + # - human + already-active default -> active (is_queue=True) + # - human + no default at all -> created (is_queue=True) + # - human + only an ARCHIVED default -> not unarchived here -> False + # (same as the prior migration) + # - non-human + active default -> archived (is_queue=False) + # - non-human -> no active default (False) + if row.has_human: + ends_with_active_default = row.has_active_default or not row.has_any_default + else: + ends_with_active_default = False + + return json.dumps( + { + "has_traces": bool(row.has_traces), + "has_testcases": bool(row.has_testcases), + "has_queries": bool(row.has_queries), + "has_testsets": bool(row.has_testsets), + "is_queue": bool(ends_with_active_default), + } + ) + + +def upgrade() -> None: + connection = op.get_bind() + + cursor = "" + processed = 0 + created = 0 + archived = 0 + + while True: + ids = _next_run_ids(connection, cursor=cursor) + if not ids: + break + + rows = connection.execute(_COMPUTE_CHUNK, {"ids": ids}).fetchall() + + flag_ids: List[str] = [] + flag_patches: List[str] = [] + + create_projects: List[str] = [] + create_runs: List[str] = [] + create_creators: List[str] = [] + create_statuses: List[str] = [] + + archive_projects: List[str] = [] + archive_runs: List[str] = [] + archive_creators: List[str] = [] + + for row in rows: + flag_ids.append(row.run_id) + flag_patches.append(_flags_patch(row)) + + # Mirror _reconcile_default_queue: has_human runs should have a + # default; create one only when none exists yet. + if row.has_human and not row.has_any_default: + create_projects.append(row.project_id) + create_runs.append(row.run_id) + create_creators.append(row.created_by_id) + create_statuses.append(row.status) + + # Non-human runs with a stale active default get it archived. + if not row.has_human and row.has_active_default: + archive_projects.append(row.project_id) + archive_runs.append(row.run_id) + archive_creators.append(row.created_by_id) + + # Create/archive first so the flag write's is_queue (computed in Python + # from the same decisions) matches the resulting queue state. + if create_runs: + connection.execute( + _CREATE_DEFAULT_QUEUES, + { + "project_ids": create_projects, + "run_ids": create_runs, + "created_by_ids": create_creators, + "statuses": create_statuses, + }, + ) + created += len(create_runs) + + if archive_runs: + connection.execute( + _ARCHIVE_STALE_DEFAULT_QUEUES, + { + "project_ids": archive_projects, + "run_ids": archive_runs, + "created_by_ids": archive_creators, + }, + ) + archived += len(archive_runs) + + connection.execute( + _APPLY_FLAGS, + {"ids": flag_ids, "flags": flag_patches}, + ) + + processed += len(ids) + cursor = ids[-1] + + click.echo( + f"[default-queue backfill] processed_runs={processed} " + f"batch={len(ids)} created={created} archived={archived}" + ) + + click.echo( + f"[default-queue backfill] done processed_runs={processed} " + f"created={created} archived={archived}" + ) + + +def downgrade() -> None: + # Keep generated queues/results intact on downgrade. Remove only the newly + # inferred flags; old is_queue semantics cannot be reconstructed safely. + op.execute(""" + UPDATE evaluation_runs + SET flags = COALESCE(flags, '{}'::jsonb) - 'has_traces' - 'has_testcases' + """) diff --git a/api/ee/databases/postgres/migrations/tracing/env.py b/api/ee/databases/postgres/migrations/tracing/env.py index 2dadd6892e..83ae21ab6b 100644 --- a/api/ee/databases/postgres/migrations/tracing/env.py +++ b/api/ee/databases/postgres/migrations/tracing/env.py @@ -7,7 +7,7 @@ from alembic import context -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.utils.env import env from oss.src.dbs.postgres.shared.base import Base # Side-effect import: register SQLAlchemy model with Base.metadata @@ -19,7 +19,7 @@ # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config -config.set_main_option("sqlalchemy.url", engine.postgres_uri_tracing) # type: ignore +config.set_main_option("sqlalchemy.url", env.postgres.uri_tracing) # Interpret the config file for Python logging. diff --git a/api/ee/src/apis/fastapi/access/router.py b/api/ee/src/apis/fastapi/access/router.py index 0eefe50d8a..1ce73e467f 100644 --- a/api/ee/src/apis/fastapi/access/router.py +++ b/api/ee/src/apis/fastapi/access/router.py @@ -4,8 +4,8 @@ from oss.src.utils.exceptions import intercept_exceptions -from ee.src.core.entitlements.types import SCOPES, Tracker -from ee.src.core.entitlements.controls import ( +from ee.src.core.access.entitlements.types import SCOPES, Tracker +from ee.src.core.access.controls import ( get_plans, get_plan_description, get_roles, @@ -70,6 +70,6 @@ async def fetch_roles(self) -> Dict[str, List[Dict[str, Any]]]: verbatim from access-controls, including the `"*"` wildcard for `owner` — callers that need to render the full permission list should expand the wildcard themselves (see - `ee.src.services.converters._expand_permissions`). + `ee.src.services.db_manager_ee._expand_permissions`). """ return {scope: list(get_roles(scope)) for scope in SCOPES} diff --git a/api/ee/src/apis/fastapi/billing/router.py b/api/ee/src/apis/fastapi/billing/router.py index ee6380f0ff..e19dfcd1a8 100644 --- a/api/ee/src/apis/fastapi/billing/router.py +++ b/api/ee/src/apis/fastapi/billing/router.py @@ -6,15 +6,14 @@ from fastapi import APIRouter, Request, status, HTTPException, Query from fastapi.responses import JSONResponse -import stripe - from oss.src.utils.common import is_ee from oss.src.utils.logging import get_module_logger from oss.src.utils.exceptions import intercept_exceptions -from oss.src.utils.caching import acquire_lock, release_lock, renew_lock +from oss.src.utils.locking import acquire_lock, release_lock, renew_lock from oss.src.utils.env import env +from oss.src.utils.lazy import _load_stripe -from ee.src.utils.entitlements import period_from, scope_from +from ee.src.core.access.entitlements.service import period_from, scope_from from ee.src.core.meters.types import Meters, MeterPeriod from oss.src.utils.context import get_auth_scope @@ -24,10 +23,10 @@ ) from ee.src.services import db_manager_ee -from ee.src.utils.permissions import check_action_access -from ee.src.models.shared_models import Permission -from ee.src.core.entitlements.types import Tracker, Quota, Period, Scope -from ee.src.core.entitlements.controls import get_plan_entitlements, get_plans +from ee.src.core.access.permissions.service import check_action_access +from ee.src.core.access.permissions.types import Permission +from ee.src.core.access.entitlements.types import Tracker, Quota, Period, Scope +from ee.src.core.access.controls import get_plan_entitlements, get_plans from ee.src.core.subscriptions.settings import ( get_catalog, get_pricing_plan, @@ -47,12 +46,6 @@ log = get_module_logger(__name__) -# Initialize Stripe only if enabled -if env.stripe.enabled: - stripe.api_key = env.stripe.api_key - log.info("✓ Stripe enabled:", target=env.stripe.webhook_target) -else: - log.info("✗ Stripe disabled") FORBIDDEN_RESPONSE = JSONResponse( status_code=403, @@ -249,8 +242,9 @@ async def handle_events( self, request: Request, ): - # No-op if Stripe is disabled - if not env.stripe.enabled: + # No-op if Stripe is unavailable (disabled or failed to load) + stripe = _load_stripe() + if stripe is None: return JSONResponse( status_code=status.HTTP_200_OK, content={"status": "ok", "message": "Stripe not configured"}, @@ -476,8 +470,9 @@ async def create_portal( self, organization_id: str, ): - # No-op if Stripe is disabled - if not env.stripe.enabled: + # No-op if Stripe is unavailable (disabled or failed to load) + stripe = _load_stripe() + if stripe is None: return JSONResponse( status_code=status.HTTP_200_OK, content={"status": "ok", "message": "Stripe not configured"}, @@ -514,8 +509,9 @@ async def create_checkout( plan: str, success_url: str, ): - # No-op if Stripe is disabled - if not env.stripe.enabled: + # No-op if Stripe is unavailable (disabled or failed to load) + stripe = _load_stripe() + if stripe is None: return JSONResponse( status_code=status.HTTP_200_OK, content={"status": "ok", "message": "Stripe not configured"}, @@ -727,7 +723,9 @@ async def fetch_subscription( _subscription = None - if env.stripe.enabled: + stripe = _load_stripe() + + if stripe is not None: if not subscription.subscription_id: return JSONResponse( status_code=status.HTTP_404_NOT_FOUND, @@ -887,6 +885,13 @@ async def cancel_subscription( detail="Subscription (Stripe) not found", ) + stripe = _load_stripe() + if stripe is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Could not load Stripe. Please try again or contact support.", + ) + try: stripe_subscription = stripe.Subscription.retrieve( subscription.subscription_id diff --git a/api/ee/src/apis/fastapi/events/router.py b/api/ee/src/apis/fastapi/events/router.py index 3a8e2b9c98..b14d6848b7 100644 --- a/api/ee/src/apis/fastapi/events/router.py +++ b/api/ee/src/apis/fastapi/events/router.py @@ -13,14 +13,17 @@ from oss.src.utils.logging import get_module_logger from oss.src.utils.exceptions import intercept_exceptions -from oss.src.utils.caching import acquire_lock, release_lock +from oss.src.utils.locking import acquire_lock, release_lock from oss.src.core.events.service import EventsService from ee.src.apis.fastapi.events.models import EventQueryRequest, EventsQueryResponse from ee.src.core.events.service import EventsRetentionService -from ee.src.models.shared_models import Permission -from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION -from ee.src.utils.entitlements import ( +from ee.src.core.access.permissions.types import Permission +from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, +) +from ee.src.core.access.entitlements.service import ( check_entitlements, NOT_ENTITLED_RESPONSE, Flag, diff --git a/api/ee/src/apis/fastapi/organizations/router.py b/api/ee/src/apis/fastapi/organizations/router.py index 0315f83cf0..773a99d989 100644 --- a/api/ee/src/apis/fastapi/organizations/router.py +++ b/api/ee/src/apis/fastapi/organizations/router.py @@ -4,8 +4,8 @@ from fastapi import APIRouter, Request, HTTPException from fastapi.responses import JSONResponse, Response -from ee.src.utils.permissions import check_user_org_access -from ee.src.utils.entitlements import ( +from ee.src.core.access.permissions.service import check_user_org_access +from ee.src.core.access.entitlements.service import ( check_entitlements, NOT_ENTITLED_RESPONSE, Tracker, @@ -13,7 +13,7 @@ ) from ee.src.services import db_manager_ee -from ee.src.services.selectors import get_user_org_and_workspace_id +from ee.src.services.db_manager_ee import get_user_org_and_workspace_id from ee.src.services.organization_service import ( OrganizationDomainsService, OrganizationProvidersService, diff --git a/api/ee/src/apis/fastapi/spans/router.py b/api/ee/src/apis/fastapi/spans/router.py index db443c600b..ecc361395c 100644 --- a/api/ee/src/apis/fastapi/spans/router.py +++ b/api/ee/src/apis/fastapi/spans/router.py @@ -10,7 +10,7 @@ from oss.src.utils.logging import get_module_logger from oss.src.utils.exceptions import intercept_exceptions -from oss.src.utils.caching import acquire_lock, release_lock +from oss.src.utils.locking import acquire_lock, release_lock from ee.src.core.tracing.service import TracingRetentionService @@ -23,7 +23,7 @@ def __init__( self, tracing_retention_service: TracingRetentionService, ): - self.tracing_retention_service = tracing_retention_service + self.tracing_service = tracing_retention_service self.admin_router = APIRouter() @@ -60,7 +60,7 @@ async def flush(self): try: log.info("[flush-spans] [endpoint] Retention started") - await self.tracing_retention_service.flush_spans() + await self.tracing_service.flush_spans() log.info("[flush-spans] [endpoint] Retention completed") return JSONResponse( diff --git a/api/ee/src/core/entitlements/__init__.py b/api/ee/src/core/access/__init__.py similarity index 100% rename from api/ee/src/core/entitlements/__init__.py rename to api/ee/src/core/access/__init__.py diff --git a/api/ee/src/core/access/controls.py b/api/ee/src/core/access/controls.py new file mode 100644 index 0000000000..00ca0e41e4 --- /dev/null +++ b/api/ee/src/core/access/controls.py @@ -0,0 +1,144 @@ +"""Access controls: the combined effective plan + role surface. + +This is the single runtime source of truth and the composition root for access +controls. It builds — once, at import time — the effective state from the two +domain builders: + +- plans/entitlements: `ee.src.core.access.entitlements.controls.build_plan_controls` +- roles/permissions: `ee.src.core.access.permissions.controls.build_role_controls` + +and exposes the public `get_plan*` / `get_role*` accessors plus a stable +`controls_hash`. The domain `controls.py` modules are pure builders/parsers with +no module-level state; this module owns the singleton. + +Code defaults live in the domain `types.py` modules; env overrides come from +`AGENTA_ACCESS_PLANS`, `AGENTA_ACCESS_ROLES`, `AGENTA_ACCESS_ROLES_OVERLAY`, and +`AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY` (raw JSON exposed via `env.agenta.access`). +""" + +import hashlib +from json import dumps +from typing import Any, Dict, List, Optional + +from oss.src.utils.logging import get_module_logger + +from ee.src.core.access.entitlements.types import SCOPES, Tracker +from ee.src.core.access.entitlements.controls import build_plan_controls +from ee.src.core.access.permissions.controls import build_role_controls + + +log = get_module_logger(__name__) + + +# --------------------------------------------------------------------------- +# Effective controls (built once at import time) +# --------------------------------------------------------------------------- + + +def _build_controls() -> tuple[ + Dict[str, Dict[Tracker, Any]], + Dict[str, str], + Dict[str, List[Dict[str, Any]]], + str, +]: + plans, descriptions, plan_source = build_plan_controls() + roles, role_source = build_role_controls() + + payload = dumps( + { + "plans": sorted(plans.keys()), + "descriptions": descriptions, + "roles": {scope: [r["role"] for r in roles[scope]] for scope in SCOPES}, + }, + sort_keys=True, + default=str, + ) + controls_hash = hashlib.sha256(payload.encode()).hexdigest()[:12] + + log.info( + "[access-controls] %s %s hash=%s", + plan_source, + role_source, + controls_hash, + ) + + return plans, descriptions, roles, controls_hash + + +_PLANS, _PLAN_DESCRIPTIONS, _ROLES, _CONTROLS_HASH = _build_controls() + + +# --------------------------------------------------------------------------- +# Public accessors — plans +# --------------------------------------------------------------------------- + + +def get_plans() -> Dict[str, Dict[Tracker, Any]]: + """Return the effective plan map (slug -> entitlement controls).""" + return _PLANS + + +def get_plan(slug: Optional[str]) -> Optional[Dict[Tracker, Any]]: + """Return the entitlement controls for a plan slug, or None if missing.""" + if not slug: + return None + return _PLANS.get(slug) + + +def get_plan_entitlements(slug: Optional[str]) -> Optional[Dict[Tracker, Any]]: + """Alias for `get_plan`. Kept distinct for readability at call sites.""" + if not slug: + return None + return _PLANS.get(slug) + + +def get_plan_description(slug: Optional[str]) -> Optional[str]: + """Return the operator-facing description for a plan, if any.""" + if not slug: + return None + return _PLAN_DESCRIPTIONS.get(slug) + + +# --------------------------------------------------------------------------- +# Public accessors — roles +# --------------------------------------------------------------------------- + + +def get_roles(scope: str) -> List[Dict[str, Any]]: + """Return the effective role catalog for a scope.""" + if scope not in SCOPES: + return [] + return _ROLES[scope] + + +def get_role(scope: str, slug: str) -> Optional[Dict[str, Any]]: + """Return a single role entry within a scope.""" + for entry in get_roles(scope): + if entry["role"] == slug: + return entry + return None + + +def get_role_permissions(scope: str, slug: str) -> List[str]: + """Return the permission slugs for a role within a scope.""" + role = get_role(scope, slug) + if not role: + return [] + return list(role["permissions"]) + + +def get_role_description(scope: str, slug: str) -> Optional[str]: + role = get_role(scope, slug) + if not role: + return None + return role.get("description") + + +# --------------------------------------------------------------------------- +# Utility +# --------------------------------------------------------------------------- + + +def get_controls_hash() -> str: + """Stable short hash of the effective controls; useful in logs.""" + return _CONTROLS_HASH diff --git a/api/oss/tests/pytest/acceptance/events/__init__.py b/api/ee/src/core/access/entitlements/__init__.py similarity index 100% rename from api/oss/tests/pytest/acceptance/events/__init__.py rename to api/ee/src/core/access/entitlements/__init__.py diff --git a/api/ee/src/core/access/entitlements/controls.py b/api/ee/src/core/access/entitlements/controls.py new file mode 100644 index 0000000000..e315660f49 --- /dev/null +++ b/api/ee/src/core/access/entitlements/controls.py @@ -0,0 +1,355 @@ +"""Plan/entitlement controls: pure builders + parsers (no singleton). + +Builds the effective plan map (slug -> entitlement controls: flags, counters, +gauges, throttles) and plan descriptions from code defaults or env overrides +(`AGENTA_ACCESS_PLANS`, `AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY`). + +This module holds no module-level state and no public accessors. The shared +singleton + `get_plan*` accessors live in `ee.src.core.access.controls`, which +calls `build_plan_controls()` once at import time. +""" + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, ValidationError + +from oss.src.utils.env import env + +from ee.src.core.access.entitlements.types import ( + Category, + Counter, + DEFAULT_ENTITLEMENTS, + DefaultPlan, + Flag, + Gauge, + Quota, + Throttle, + Tracker, +) + + +# --------------------------------------------------------------------------- +# Override schemas +# --------------------------------------------------------------------------- + + +class _PlanOverride(BaseModel): + description: Optional[str] = None + flags: Optional[Dict[str, bool]] = None + counters: Optional[Dict[str, Quota]] = None + gauges: Optional[Dict[str, Quota]] = None + throttles: Optional[List[Throttle]] = None + + model_config = ConfigDict(extra="forbid") + + +# --------------------------------------------------------------------------- +# Plan entitlements + descriptions +# --------------------------------------------------------------------------- + +# Effective state is a dict[plan_slug, plan_entry] where plan_entry has the same +# shape as DEFAULT_ENTITLEMENTS values (Tracker -> mapping) plus an optional +# "description" key in the top-level plan entry's own description map. + +_DEFAULT_PLAN_DESCRIPTIONS: Dict[str, str] = {} + + +def _default_plans() -> Dict[str, Dict[Tracker, Any]]: + # Keys in DEFAULT_ENTITLEMENTS are `DefaultPlan` (str, Enum) members. + # Coerce to plain strings so the runtime plan map is uniformly keyed by slug. + return {str(plan.value): entry for plan, entry in DEFAULT_ENTITLEMENTS.items()} + + +def _validate_flag_key(key: str) -> Flag: + try: + return Flag(key) + except ValueError as e: + raise ValueError(f"Unknown flag '{key}'") from e + + +def _validate_counter_key(key: str) -> Counter: + try: + return Counter(key) + except ValueError as e: + raise ValueError(f"Unknown counter '{key}'") from e + + +def _validate_gauge_key(key: str) -> Gauge: + try: + return Gauge(key) + except ValueError as e: + raise ValueError(f"Unknown gauge '{key}'") from e + + +def _parse_plans_override( + decoded: Any, +) -> tuple[Dict[str, Dict[Tracker, Any]], Dict[str, str]]: + if not isinstance(decoded, dict) or not decoded: + raise ValueError("AGENTA_ACCESS_PLANS must be a non-empty JSON object") + + plans: Dict[str, Dict[Tracker, Any]] = {} + descriptions: Dict[str, str] = {} + + for slug, payload in decoded.items(): + if not slug or not isinstance(slug, str): + raise ValueError(f"Invalid plan slug '{slug}' in AGENTA_ACCESS_PLANS") + + try: + override = _PlanOverride.model_validate(payload) + except ValidationError as e: + raise ValueError( + f"Invalid plan override for '{slug}' in AGENTA_ACCESS_PLANS: {e}" + ) from e + + plan_entry: Dict[Tracker, Any] = {} + + if override.flags is not None: + plan_entry[Tracker.FLAGS] = { + _validate_flag_key(k): bool(v) for k, v in override.flags.items() + } + + if override.counters is not None: + plan_entry[Tracker.COUNTERS] = { + _validate_counter_key(k): v for k, v in override.counters.items() + } + + if override.gauges is not None: + plan_entry[Tracker.GAUGES] = { + _validate_gauge_key(k): v for k, v in override.gauges.items() + } + + if override.throttles is not None: + plan_entry[Tracker.THROTTLES] = list(override.throttles) + + # Plans with only a description are allowed — they represent custom / + # display-only plans (e.g. self-hosted Enterprise) that don't enforce + # quotas server-side. Downstream consumers must handle the empty + # entitlement map gracefully (e.g. `fetch_usage` returns no rows). + plans[slug] = plan_entry + + if override.description: + descriptions[slug] = override.description + + return plans, descriptions + + +# --------------------------------------------------------------------------- +# Default-plan overlay +# --------------------------------------------------------------------------- +# +# `AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY` lets self-hosted operators tweak individual +# entitlement values on the default plan without restating the entire plan in +# `AGENTA_ACCESS_PLANS`. Common cases: bumping trace retention, raising the +# standard-throttle rate, flipping a flag. +# +# Shape mirrors a plan entry (same keys, same units) with one divergence: +# `throttles` is a map keyed by category slug instead of a list, so per- +# category patches don't require restating the whole list. Throttles that +# combine multiple categories or use `endpoints` cannot be addressed via the +# overlay — operators who need that should use `AGENTA_ACCESS_PLANS`. + + +class _ThrottleOverlay(BaseModel): + """Partial throttle update keyed by a single category. + + Every field is optional; only fields explicitly set on the overlay + replace the matching field on the existing throttle entry. + """ + + bucket: Optional[Dict[str, Any]] = None + mode: Optional[str] = None + + model_config = ConfigDict(extra="forbid") + + +class _DefaultPlanOverlay(BaseModel): + """Partial overlay for the default plan. Same shape as `_PlanOverride` + except `throttles` is a map keyed by category slug.""" + + description: Optional[str] = None + flags: Optional[Dict[str, bool]] = None + counters: Optional[Dict[str, Dict[str, Any]]] = None + gauges: Optional[Dict[str, Dict[str, Any]]] = None + throttles: Optional[Dict[str, _ThrottleOverlay]] = None + + model_config = ConfigDict(extra="forbid") + + +def _merge_quota(existing: Optional[Quota], patch: Dict[str, Any]) -> Quota: + """Patch a Quota field-by-field, preserving fields the overlay didn't set.""" + if existing is None: + # Patching an entitlement key that isn't defined on the base plan: + # treat the overlay as the full definition. + return Quota.model_validate(patch) + merged = existing.model_dump() + merged.update(patch) + return Quota.model_validate(merged) + + +def _merge_throttle(existing: Throttle, patch: _ThrottleOverlay) -> Throttle: + """Patch one throttle entry. `bucket` is field-merged; `mode` replaces.""" + base = existing.model_dump() + if patch.bucket is not None: + bucket = dict(base.get("bucket") or {}) + bucket.update(patch.bucket) + base["bucket"] = bucket + if patch.mode is not None: + base["mode"] = patch.mode + return Throttle.model_validate(base) + + +def _parse_default_plan_overlay(decoded: Any) -> _DefaultPlanOverlay: + if not isinstance(decoded, dict) or not decoded: + raise ValueError( + "AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY must be a non-empty JSON object" + ) + try: + overlay = _DefaultPlanOverlay.model_validate(decoded) + except ValidationError as e: + raise ValueError(f"Invalid AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY: {e}") from e + + # Validate slugs upfront so the error message points at the bad key + # rather than failing inside the merge. + if overlay.flags is not None: + for key in overlay.flags: + _validate_flag_key(key) + if overlay.counters is not None: + for key in overlay.counters: + _validate_counter_key(key) + if overlay.gauges is not None: + for key in overlay.gauges: + _validate_gauge_key(key) + if overlay.throttles is not None: + valid_categories = {c.value for c in Category} + for key in overlay.throttles: + if key not in valid_categories: + raise ValueError( + f"AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY.throttles['{key}'] is not a " + f"valid throttle category. Allowed: {sorted(valid_categories)}." + ) + return overlay + + +def _apply_default_plan_overlay( + plans: Dict[str, Dict[Tracker, Any]], + descriptions: Dict[str, str], + overlay: _DefaultPlanOverlay, + default_plan_slug: str, +) -> tuple[Dict[str, Dict[Tracker, Any]], Dict[str, str]]: + """Apply the overlay to the resolved default plan in-place (returning new dicts).""" + if default_plan_slug not in plans: + raise ValueError( + f"AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY targets the default plan " + f"'{default_plan_slug}', which is not in the effective plan set " + f"({sorted(plans.keys())}). Add the slug to AGENTA_ACCESS_PLANS or " + "unset AGENTA_DEFAULT_PLAN." + ) + + plans = {slug: dict(entry) for slug, entry in plans.items()} + descriptions = dict(descriptions) + entry = plans[default_plan_slug] + + if overlay.description is not None: + descriptions[default_plan_slug] = overlay.description + + if overlay.flags is not None: + flags = dict(entry.get(Tracker.FLAGS) or {}) + for key, value in overlay.flags.items(): + flags[Flag(key)] = bool(value) + entry[Tracker.FLAGS] = flags + + if overlay.counters is not None: + counters = dict(entry.get(Tracker.COUNTERS) or {}) + for key, patch in overlay.counters.items(): + counter = Counter(key) + counters[counter] = _merge_quota(counters.get(counter), patch) + entry[Tracker.COUNTERS] = counters + + if overlay.gauges is not None: + gauges = dict(entry.get(Tracker.GAUGES) or {}) + for key, patch in overlay.gauges.items(): + gauge = Gauge(key) + gauges[gauge] = _merge_quota(gauges.get(gauge), patch) + entry[Tracker.GAUGES] = gauges + + if overlay.throttles is not None: + existing_throttles = list(entry.get(Tracker.THROTTLES) or []) + + for category_key, patch in overlay.throttles.items(): + category = Category(category_key) + # Find the single-category throttle entry that matches. + target_idx = next( + ( + idx + for idx, t in enumerate(existing_throttles) + if t.categories + and len(t.categories) == 1 + and t.categories[0] == category + ), + None, + ) + if target_idx is None: + raise ValueError( + f"AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY.throttles['{category_key}']: " + f"the default plan '{default_plan_slug}' has no single-" + f"category throttle entry for '{category_key}'. Use " + "AGENTA_ACCESS_PLANS to define multi-category or endpoint-" + "keyed throttles." + ) + existing_throttles[target_idx] = _merge_throttle( + existing_throttles[target_idx], patch + ) + entry[Tracker.THROTTLES] = existing_throttles + + plans[default_plan_slug] = entry + return plans, descriptions + + +def _resolve_default_plan_slug(plans: Dict[str, Dict[Tracker, Any]]) -> str: + """Resolve the default plan slug for overlay targeting. + + Mirrors `subscriptions.types.get_default_plan()` without importing it (to + avoid pulling subscription/Stripe code into the access-controls layer). + """ + raw = env.agenta.access.default_plan + if raw: + return raw + if env.stripe.enabled: + return DefaultPlan.CLOUD_V0_HOBBY.value + return DefaultPlan.SELF_HOSTED_ENTERPRISE.value + + +# --------------------------------------------------------------------------- +# Build (called once by ee.src.core.access.controls) +# --------------------------------------------------------------------------- + + +def build_plan_controls() -> tuple[Dict[str, Dict[Tracker, Any]], Dict[str, str], str]: + """Build the effective plan map + descriptions from defaults or env overrides. + + Returns ``(plans, descriptions, source_label)`` where ``source_label`` is a + short string for startup logging (e.g. ``"defaults"`` or + ``"env plan_overlay=env→"``). + """ + plans_payload = env.agenta.access.plans + plan_overlay_payload = env.agenta.access.default_plan_overlay + + if plans_payload is not None: + plans, descriptions = _parse_plans_override(plans_payload) + plans_source = "env" + else: + plans = _default_plans() + descriptions = dict(_DEFAULT_PLAN_DESCRIPTIONS) + plans_source = "defaults" + + plan_overlay_source = "none" + if plan_overlay_payload is not None: + plan_overlay = _parse_default_plan_overlay(plan_overlay_payload) + default_plan_slug = _resolve_default_plan_slug(plans) + plans, descriptions = _apply_default_plan_overlay( + plans, descriptions, plan_overlay, default_plan_slug + ) + plan_overlay_source = f"env→{default_plan_slug}" + + source = f"plans={plans_source} plan_overlay={plan_overlay_source}" + return plans, descriptions, source diff --git a/api/ee/src/utils/entitlements.py b/api/ee/src/core/access/entitlements/service.py similarity index 95% rename from api/ee/src/utils/entitlements.py rename to api/ee/src/core/access/entitlements/service.py index 9b290e4ac6..202f098c6e 100644 --- a/api/ee/src/utils/entitlements.py +++ b/api/ee/src/core/access/entitlements/service.py @@ -8,7 +8,7 @@ from fastapi.responses import JSONResponse from ee.src.core.subscriptions.service import SubscriptionsService -from ee.src.core.entitlements.types import ( +from ee.src.core.access.entitlements.types import ( Tracker, Flag, Counter, @@ -16,7 +16,7 @@ Period, Scope, ) -from ee.src.core.entitlements.controls import get_plan_entitlements +from ee.src.core.access.controls import get_plan_entitlements from ee.src.core.meters.service import MetersService from ee.src.core.meters.types import MeterDTO, MeterScope, MeterPeriod, Meters @@ -86,7 +86,6 @@ def bootstrap_entitlements_services( if subscriptions_service is None: subscriptions_service = SubscriptionsService( subscriptions_dao=SubscriptionsDAO(), - meters_service=meters_service, ) register_entitlements_services( @@ -408,12 +407,6 @@ async def _check_entitlements( check = flags[flag] - if flag.name != "RBAC": - # TODO: remove this line - log.info( - f"[METERS] adjusting: {organization_id} | | {'allow' if check else 'deny '} | {flag.name}" - ) - return check is True, None, None # -------------------------------------------------------------- # @@ -562,17 +555,4 @@ async def _check_entitlements( key=cache_key, ) - # TODO: remove this line - log.info( - f"[METERS] adjusting: {_scope.organization_id} | " - f"{_scope.workspace_id} | " - f"{_scope.project_id} | " - f"{_scope.user_id} | " - f"{(meter.year if meter.year else ' ')}-" - f"{(meter.month if meter.month else ' ')}-" - f"{(meter.day if meter.day else ' ')} | " - f"{'allow' if check else 'deny '} | " - f"{meter.key}: {(meter.value or 0) - (meter.synced or 0)} [{meter.value}]" - ) - return check is True, meter, _ diff --git a/api/ee/src/core/entitlements/types.py b/api/ee/src/core/access/entitlements/types.py similarity index 95% rename from api/ee/src/core/entitlements/types.py rename to api/ee/src/core/access/entitlements/types.py index 39c1eb644b..4547ef40c8 100644 --- a/api/ee/src/core/entitlements/types.py +++ b/api/ee/src/core/access/entitlements/types.py @@ -6,7 +6,7 @@ class DefaultPlan(str, Enum): """Code-default plan slugs. - Runtime plan slugs come from `ee.src.core.entitlements.controls.get_plans()` + Runtime plan slugs come from `ee.src.core.access.controls.get_plans()` (env-overridable). This enum is only the default-fallback identifier set, used as keys for `DEFAULT_ENTITLEMENTS` / `DEFAULT_CATALOG` and as fallback in `get_default_plan()` / `get_free_plan()` / `get_trial_plan()`. @@ -21,24 +21,9 @@ class DefaultPlan(str, Enum): SELF_HOSTED_ENTERPRISE = "self_hosted_enterprise" -class DefaultRole(str, Enum): - """Required role slugs per scope. - - `owner` and `viewer` must exist in every scope (organization, workspace, - project) — they are merged in by the access-controls builder regardless of - `AGENTA_ACCESS_ROLES` content, so application code can depend on these - two slugs being valid in any scope. - - Env overrides may customize the permissions of these roles or add - additional roles, but cannot remove them. - """ - - OWNER = "owner" - VIEWER = "viewer" - - # Permission slugs that the OWNER role always implies. `"*"` is the wildcard -# permission recognized by `permissions.py`. +# permission recognized by `permissions.py`. The `RequiredRole` enum itself now +# lives in `ee.src.core.access.permissions.types`. OWNER_PERMISSIONS: list[str] = ["*"] @@ -186,7 +171,7 @@ class Throttle(BaseModel): (Method.POST, "/tracing/spans/analytics"), # LEGACY ], Category.SERVICES_FAST: [ - (Method.ANY, "/permissions/verify"), + (Method.ANY, "/access/permissions/check"), ], Category.SERVICES_SLOW: [ # None defined yet @@ -685,12 +670,11 @@ class Throttle(BaseModel): } -REPORTS = [ - Counter.TRACES_INGESTED.value, - Gauge.USERS.value, -] - -STRIPE_METER_NAMES: dict[str, str] = { +# Internal Counter/Gauge slug -> Stripe-side meter slot name. Membership in +# this map doubles as the "reportable to Stripe" set: a meter is reported iff +# its key is present here (`key in REPORTS`), and the value is the Stripe-side +# name to report under (`REPORTS[key]`). +REPORTS: dict[str, str] = { Counter.TRACES_INGESTED.value: "traces", Gauge.USERS.value: "users", } diff --git a/sdks/python/oss/tests/pytest/integration/.gitkeep b/api/ee/src/core/access/permissions/__init__.py similarity index 100% rename from sdks/python/oss/tests/pytest/integration/.gitkeep rename to api/ee/src/core/access/permissions/__init__.py diff --git a/api/ee/src/core/access/permissions/controls.py b/api/ee/src/core/access/permissions/controls.py new file mode 100644 index 0000000000..054fc65402 --- /dev/null +++ b/api/ee/src/core/access/permissions/controls.py @@ -0,0 +1,438 @@ +"""Role/permission controls: pure builders + parsers (no singleton). + +Builds the per-scope role catalogs (organization, workspace, project) and their +role->permission mappings from code defaults or env overrides +(`AGENTA_ACCESS_ROLES`, `AGENTA_ACCESS_ROLES_OVERLAY`). + +This module holds no module-level state and no public accessors. The shared +singleton + `get_role*` accessors live in `ee.src.core.access.controls`, which +calls `build_role_controls()` once at import time. + +Minima contract: every scope MUST expose `owner`, `admin`, and `viewer` with +code-defined permissions. Env overrides may add roles or customize permissions +of non-minima roles, but the minima are always present and their slugs cannot be +re-bound to a different permission set. +""" + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, ValidationError + +from oss.src.utils.env import env + +from ee.src.core.access.entitlements.types import OWNER_PERMISSIONS, SCOPES +from ee.src.core.access.permissions.types import ( + Permission, + DefaultRole, + RequiredRole, +) + + +# --------------------------------------------------------------------------- +# Override schemas +# --------------------------------------------------------------------------- + + +class _RoleOverride(BaseModel): + role: str + description: Optional[str] = None + permissions: List[str] + + model_config = ConfigDict(extra="forbid") + + +class _RoleOverlayEntry(BaseModel): + """Partial role update. ``permissions`` and ``description`` are both + optional; whatever is set replaces the matching field on the existing + role entry. To add a new role both fields must be supplied — that + constraint is enforced in :func:`_apply_roles_overlay`. + """ + + description: Optional[str] = None + permissions: Optional[List[str]] = None + + model_config = ConfigDict(extra="forbid") + + +# --------------------------------------------------------------------------- +# Role catalogs (scoped) +# --------------------------------------------------------------------------- + + +def _read_only_permissions() -> List[str]: + """Read-only permission set sourced from the code-default `DefaultRole.VIEWER`. + + Used as the `viewer` minima permissions for the `workspace` and `project` + scopes where permissions are actually enforced. Organization-scope `viewer` + has no permissions (it's just a membership marker today). + """ + return [p.value for p in Permission.default_permissions(DefaultRole.VIEWER)] + + +def _viewer_permissions_for_scope(scope: str) -> List[str]: + """Per-scope code-default permissions for the `viewer` minima role. + + - `organization`: empty — orgs have no permission concept today. + - `workspace` and `project`: the code-default `DefaultRole.VIEWER` read-only set. + """ + if scope == "organization": + return [] + return _read_only_permissions() + + +def _admin_permissions_for_scope(scope: str) -> List[str]: + """Per-scope code-default permissions for the `admin` minima role. + + - `organization`: empty — orgs have no permission concept today. + - `workspace` and `project`: the code-default `DefaultRole.ADMIN` set. + """ + if scope == "organization": + return [] + return [p.value for p in Permission.default_permissions(DefaultRole.ADMIN)] + + +def _minima_for(scope: str) -> List[Dict[str, Any]]: + """Return the required role entries for a scope (owner + admin + viewer). + + Application code relies on these slugs being present in every scope; the + builder synthesizes them up front and re-applies them after any env + overrides so they can never be dropped or relabeled. + + `owner` is always wildcard. The permission sets for `admin` and `viewer` + vary per scope (see `_admin_permissions_for_scope` / + `_viewer_permissions_for_scope`): both are empty at organization scope + (orgs have no permission concept today) and code-default elsewhere. + """ + return [ + { + "role": RequiredRole.OWNER.value, + "description": "Full access (wildcard permissions).", + "permissions": list(OWNER_PERMISSIONS), + }, + { + "role": RequiredRole.ADMIN.value, + "description": ( + "Membership marker (no permissions)." + if scope == "organization" + else DefaultRole.get_description(DefaultRole.ADMIN) + ), + "permissions": _admin_permissions_for_scope(scope), + }, + { + "role": RequiredRole.VIEWER.value, + "description": ( + "Membership marker (no permissions)." + if scope == "organization" + else DefaultRole.get_description(DefaultRole.VIEWER) + ), + "permissions": _viewer_permissions_for_scope(scope), + }, + ] + + +def _default_roles() -> Dict[str, List[Dict[str, Any]]]: + """Return the code-default role catalog for each scope. + + Workspace and project scopes expose the code-default `DefaultRole` + entries on top of the minima — project membership stores the same role + slugs (`admin`/`developer`/`editor`/`annotator`), and the runtime + permission check resolves them through this map. Organization scope + only gets the minima today; new roles can be added per-scope via + `AGENTA_ACCESS_ROLES`. + """ + default_extras: List[Dict[str, Any]] = [] + minima_slugs = { + RequiredRole.OWNER.value, + RequiredRole.ADMIN.value, + RequiredRole.VIEWER.value, + } + for role in DefaultRole: + if role.value in minima_slugs: + continue + default_extras.append( + { + "role": role.value, + "description": DefaultRole.get_description(role), + "permissions": [p.value for p in Permission.default_permissions(role)], + } + ) + + return { + "organization": _minima_for("organization"), + "workspace": _minima_for("workspace") + default_extras, + "project": _minima_for("project") + default_extras, + } + + +def _validate_permission(slug: str) -> str: + if slug == "*": + return slug + try: + Permission(slug) + except ValueError as e: + raise ValueError(f"Unknown permission '{slug}'") from e + return slug + + +def _parse_roles_override(decoded: Any) -> Dict[str, List[Dict[str, Any]]]: + """Parse `AGENTA_ACCESS_ROLES` and merge with scope minima. + + Schema is `{scope: [role, ...]}` where scope is one of `organization`, + `workspace`, `project`. Missing scopes fall back to the code-default + minima for that scope. Within an overridden scope: + + - `owner`, `admin`, `viewer` are reserved slugs; if present, they're + rejected with a clear error so application invariants stay intact. The + minima for that scope are re-applied after parsing. + - Other roles are validated (known permissions, no duplicates) and + appended to the minima list. + + Empty `{}` or an empty per-scope list is rejected. + """ + if not isinstance(decoded, dict) or not decoded: + raise ValueError("AGENTA_ACCESS_ROLES must be a non-empty JSON object") + + # Start with code-default catalogs for every scope. Overrides only mutate + # the scopes they specify; non-overridden scopes keep their full code + # defaults (e.g. workspace's admin/developer/editor/annotator stay even + # when only `project` is overridden). + result: Dict[str, List[Dict[str, Any]]] = _default_roles() + + reserved = { + RequiredRole.OWNER.value, + RequiredRole.ADMIN.value, + RequiredRole.VIEWER.value, + } + + for scope, roles in decoded.items(): + if scope not in SCOPES: + raise ValueError( + f"Unknown role scope '{scope}' in AGENTA_ACCESS_ROLES " + f"(allowed: {list(SCOPES)})" + ) + + if not isinstance(roles, list) or not roles: + raise ValueError( + f"AGENTA_ACCESS_ROLES['{scope}'] must be a non-empty list of roles" + ) + + extras: List[Dict[str, Any]] = [] + seen: set[str] = set() + for entry in roles: + try: + role = _RoleOverride.model_validate(entry) + except ValidationError as e: + raise ValueError( + f"Invalid role override under scope '{scope}': {e}" + ) from e + + slug = role.role + if not slug: + raise ValueError(f"Empty role slug under scope '{scope}'") + if slug in reserved: + raise ValueError( + f"AGENTA_ACCESS_ROLES['{scope}'] cannot redefine reserved " + f"role '{slug}'; minima are always synthesized by the platform" + ) + if slug in seen: + raise ValueError(f"Duplicate role slug '{slug}' under scope '{scope}'") + seen.add(slug) + + permissions = [_validate_permission(p) for p in role.permissions] + + extras.append( + { + "role": slug, + "description": role.description, + "permissions": permissions, + } + ) + + # Minima first, then validated extras. + result[scope] = _minima_for(scope) + extras + + # TEMP: workspace and project use the same role set at runtime (the only + # caller of `workspace`-scope roles today is the Invite Members modal, + # which is really inviting to the underlying project). Operators almost + # always override only `project` via AGENTA_ACCESS_ROLES; without this + # mirror, custom roles silently disappear from the workspace catalog. + # Remove once the workspace/project scope split is reconciled. + if "project" in decoded and "workspace" not in decoded: + project_extras = [ + entry + for entry in result["project"] + if entry["role"] + not in { + RequiredRole.OWNER.value, + RequiredRole.ADMIN.value, + RequiredRole.VIEWER.value, + } + ] + result["workspace"] = _minima_for("workspace") + project_extras + + return result + + +# --------------------------------------------------------------------------- +# Roles overlay +# --------------------------------------------------------------------------- +# +# `AGENTA_ACCESS_ROLES_OVERLAY` lets operators tweak individual fields on +# existing default roles (`admin`, `developer`, `editor`, `annotator`) — or +# add new roles — without restating the whole scope catalog the way +# `AGENTA_ACCESS_ROLES` requires. See the parser docstring for the two accepted +# payload shapes. + + +def _parse_roles_overlay(decoded: Any) -> Dict[str, _RoleOverlayEntry]: + """Parse `AGENTA_ACCESS_ROLES_OVERLAY` and return per-slug entries. + + Two accepted payload shapes: + + 1. Project-focused shortcut — ``{: }``. Used when none + of the scope keys (``organization``, ``workspace``, ``project``) appear + at the root; the payload is interpreted as project-level role patches. + The scope names are therefore reserved and cannot be used as role slugs. + + 2. Full (scoped) form — ``{"project": {: }}``. Triggered + when any of ``organization``, ``workspace``, ``project`` appears at the + root. Only ``project`` is supported today; ``organization`` and + ``workspace`` are rejected. + + The result is the inner ``{: }`` dict; the caller decides + which scopes the patch applies to. + """ + if not isinstance(decoded, dict) or not decoded: + raise ValueError("AGENTA_ACCESS_ROLES_OVERLAY must be a non-empty JSON object") + + scope_keys = {"organization", "workspace", "project"} + + if scope_keys & set(decoded.keys()): + # Full parse: at least one scope key at root. + unsupported = (set(decoded.keys()) & scope_keys) - {"project"} + if unsupported: + raise ValueError( + f"AGENTA_ACCESS_ROLES_OVERLAY only supports the 'project' scope " + f"today (got: {sorted(unsupported)}). The patch is applied to " + "both workspace and project." + ) + unknown = set(decoded.keys()) - scope_keys + if unknown: + raise ValueError( + f"AGENTA_ACCESS_ROLES_OVERLAY mixes scope keys with non-scope " + f"keys at the root ({sorted(unknown)}). Either use the " + "project-focused shortcut (no scope keys, role slugs at the " + "root) or the full form ({'project': {...}})." + ) + project_payload = decoded.get("project") + if not isinstance(project_payload, dict) or not project_payload: + raise ValueError( + "AGENTA_ACCESS_ROLES_OVERLAY['project'] must be a non-empty " + "JSON object keyed by role slug" + ) + else: + # Project-focused shortcut: top-level dict is keyed by role slug. + project_payload = decoded + + reserved = { + RequiredRole.OWNER.value, + RequiredRole.ADMIN.value, + RequiredRole.VIEWER.value, + } + entries: Dict[str, _RoleOverlayEntry] = {} + for slug, patch in project_payload.items(): + if not slug or not isinstance(slug, str): + raise ValueError( + f"Invalid role slug '{slug}' in AGENTA_ACCESS_ROLES_OVERLAY" + ) + if slug in reserved: + raise ValueError( + f"AGENTA_ACCESS_ROLES_OVERLAY cannot patch reserved role " + f"'{slug}'; minima are platform-managed" + ) + try: + entry = _RoleOverlayEntry.model_validate(patch) + except ValidationError as e: + raise ValueError( + f"Invalid AGENTA_ACCESS_ROLES_OVERLAY['project']['{slug}']: {e}" + ) from e + if entry.permissions is not None: + for perm in entry.permissions: + _validate_permission(perm) + entries[slug] = entry + + return entries + + +def _apply_roles_overlay( + roles: Dict[str, List[Dict[str, Any]]], + overlay: Dict[str, _RoleOverlayEntry], +) -> Dict[str, List[Dict[str, Any]]]: + """Apply the parsed overlay to the workspace and project scopes. + + Returns a new dict; the input is not mutated. New roles (where the + slug doesn't already exist on the scope) require both ``description`` + and ``permissions``. + """ + result = {scope: [dict(entry) for entry in roles[scope]] for scope in roles} + + for scope_name in ("workspace", "project"): + scope_entries = result[scope_name] + by_slug = {entry["role"]: idx for idx, entry in enumerate(scope_entries)} + + for slug, patch in overlay.items(): + if slug in by_slug: + # Patch existing role: per-field replace. + idx = by_slug[slug] + if patch.description is not None: + scope_entries[idx]["description"] = patch.description + if patch.permissions is not None: + scope_entries[idx]["permissions"] = list(patch.permissions) + else: + # New role: both fields must be present. + if patch.permissions is None: + raise ValueError( + f"AGENTA_ACCESS_ROLES_OVERLAY['project']['{slug}']: " + f"new role requires 'permissions' (scope '{scope_name}' " + "has no existing role with this slug to patch)" + ) + scope_entries.append( + { + "role": slug, + "description": patch.description, + "permissions": list(patch.permissions), + } + ) + + return result + + +# --------------------------------------------------------------------------- +# Build (called once by ee.src.core.access.controls) +# --------------------------------------------------------------------------- + + +def build_role_controls() -> tuple[Dict[str, List[Dict[str, Any]]], str]: + """Build the per-scope role catalogs from defaults or env overrides. + + Returns ``(roles, source_label)`` where ``source_label`` is a short string + for startup logging (e.g. ``"roles=defaults roles_overlay=none"``). + """ + roles_payload = env.agenta.access.roles + roles_overlay_payload = env.agenta.access.roles_overlay + + if roles_payload is not None: + roles = _parse_roles_override(roles_payload) + roles_source = "env" + else: + roles = _default_roles() + roles_source = "defaults" + + roles_overlay_source = "none" + if roles_overlay_payload is not None: + roles_overlay = _parse_roles_overlay(roles_overlay_payload) + roles = _apply_roles_overlay(roles, roles_overlay) + roles_overlay_source = "env" + + source = f"roles={roles_source} roles_overlay={roles_overlay_source}" + return roles, source diff --git a/api/ee/src/utils/permissions.py b/api/ee/src/core/access/permissions/service.py similarity index 97% rename from api/ee/src/utils/permissions.py rename to api/ee/src/core/access/permissions/service.py index b18181fbf5..38dcffdebf 100644 --- a/api/ee/src/utils/permissions.py +++ b/api/ee/src/core/access/permissions/service.py @@ -10,16 +10,17 @@ WorkspaceDB, ProjectDB, ) -from ee.src.models.shared_models import ( - Permission, - WorkspaceRole, -) -from ee.src.core.entitlements.controls import get_role, get_role_permissions +from ee.src.core.access.permissions.types import Permission, RequiredRole +from ee.src.core.access.controls import get_role, get_role_permissions from oss.src.services import db_manager from ee.src.services import db_manager_ee -from ee.src.utils.entitlements import check_entitlements, scope_from, Flag -from ee.src.services.selectors import get_user_org_and_workspace_id +from ee.src.core.access.entitlements.service import ( + check_entitlements, + scope_from, + Flag, +) +from ee.src.services.db_manager_ee import get_user_org_and_workspace_id log = get_module_logger(__name__) @@ -65,12 +66,12 @@ def _project_is_owner( ) -> bool: """True if the user is OWNER in the project.""" role = _get_project_member_role(user_id, members) - return role == WorkspaceRole.OWNER + return role == RequiredRole.OWNER def _project_has_role( user_id: str, - role_to_check: WorkspaceRole, + role_to_check: RequiredRole, members: Sequence[Any], ) -> bool: """True if the user's role exactly matches role_to_check.""" diff --git a/api/ee/src/models/shared_models.py b/api/ee/src/core/access/permissions/types.py similarity index 81% rename from api/ee/src/models/shared_models.py rename to api/ee/src/core/access/permissions/types.py index fccba1affd..c3ab36b719 100644 --- a/api/ee/src/models/shared_models.py +++ b/api/ee/src/core/access/permissions/types.py @@ -1,10 +1,16 @@ from enum import Enum -from typing import List -from pydantic import BaseModel, Field +class DefaultRole(str, Enum): + """Code-default role catalog (the roles Agenta ships with). + + Used at all three scopes (organization / workspace / project) to seed the + default role catalog and their permission mappings. Env overrides + (`AGENTA_ACCESS_ROLES`) may add or customize roles on top of these. + + The minimal subset that must always exist in every scope is `RequiredRole`. + """ -class WorkspaceRole(str, Enum): OWNER = "owner" ADMIN = "admin" DEVELOPER = "developer" @@ -14,7 +20,7 @@ class WorkspaceRole(str, Enum): @classmethod def is_valid_role(cls, role: str) -> bool: - return role.upper() in list(WorkspaceRole.__members__.keys()) + return role.upper() in list(DefaultRole.__members__.keys()) @classmethod def get_description(cls, role): @@ -29,6 +35,29 @@ def get_description(cls, role): return descriptions.get(role, "Description not available, Role not found") +class RequiredRole(str, Enum): + """Required role slugs per scope (the minimal guaranteed subset). + + `owner`, `admin`, and `viewer` must exist in every scope (organization, + workspace, project) — they are merged in by the access-controls builder + regardless of `AGENTA_ACCESS_ROLES` content, so application code can depend + on these three slugs being valid in any scope. + + The rationale for the minimal set: `owner` is the single full-access owner, + `admin` is full access that can be granted to many people, and `viewer` is + minimal (read-only) access for many people. Without `admin` in the minimal + set, an env override could leave a scope with only owner + viewer — where no + non-owner can actually do anything. + + Env overrides may customize the permissions of these roles or add + additional roles, but cannot remove them. + """ + + OWNER = "owner" + ADMIN = "admin" + VIEWER = "viewer" + + class Permission(str, Enum): # general READ_SYSTEM = "read_system" @@ -242,17 +271,12 @@ def default_permissions(cls, role): cls.VIEW_WORKSPACE, ] defaults = { - WorkspaceRole.OWNER: [p for p in cls], - WorkspaceRole.ADMIN: ADMIN_PERMISSIONS, - WorkspaceRole.DEVELOPER: DEVELOPER_PERMISSIONS, - WorkspaceRole.EDITOR: EDITOR_PERMISSIONS, - WorkspaceRole.ANNOTATOR: ANNOTATOR_PERMISSIONS, - WorkspaceRole.VIEWER: VIEWER_PERMISSIONS, + DefaultRole.OWNER: [p for p in cls], + DefaultRole.ADMIN: ADMIN_PERMISSIONS, + DefaultRole.DEVELOPER: DEVELOPER_PERMISSIONS, + DefaultRole.EDITOR: EDITOR_PERMISSIONS, + DefaultRole.ANNOTATOR: ANNOTATOR_PERMISSIONS, + DefaultRole.VIEWER: VIEWER_PERMISSIONS, } return defaults.get(role, []) - - -class WorkspaceMember(BaseModel): - role_name: WorkspaceRole - permissions: List[Permission] = Field(default_factory=list) diff --git a/api/ee/src/core/entitlements/controls.py b/api/ee/src/core/entitlements/controls.py deleted file mode 100644 index edd4ec42ef..0000000000 --- a/api/ee/src/core/entitlements/controls.py +++ /dev/null @@ -1,832 +0,0 @@ -"""Access controls: effective plan/role accessors built from code defaults or env overrides. - -This module is the single runtime source of truth for: - -- the effective plan slug set; -- per-plan entitlement controls (flags, counters, gauges, throttles); -- per-scope role catalogs (organization, workspace, project). - -Code defaults live in `types.py` and `ee.src.models.shared_models`. Environment -overrides come from `AGENTA_ACCESS_PLANS` and `AGENTA_ACCESS_ROLES` (raw JSON -strings exposed via `env.agenta.access`). Parsing happens once at import time. -""" - -import hashlib -from json import dumps -from typing import Any, Dict, List, Optional - -from pydantic import BaseModel, ConfigDict, ValidationError - -from oss.src.utils.env import env -from oss.src.utils.logging import get_module_logger - -from ee.src.core.entitlements.types import ( - Category, - Counter, - DEFAULT_ENTITLEMENTS, - DefaultPlan, - DefaultRole, - Flag, - Gauge, - OWNER_PERMISSIONS, - Quota, - SCOPES, - Throttle, - Tracker, -) -from ee.src.models.shared_models import Permission, WorkspaceRole - - -log = get_module_logger(__name__) - - -# --------------------------------------------------------------------------- -# Override schemas -# --------------------------------------------------------------------------- - - -class _PlanOverride(BaseModel): - description: Optional[str] = None - flags: Optional[Dict[str, bool]] = None - counters: Optional[Dict[str, Quota]] = None - gauges: Optional[Dict[str, Quota]] = None - throttles: Optional[List[Throttle]] = None - - model_config = ConfigDict(extra="forbid") - - -class _RoleOverride(BaseModel): - role: str - description: Optional[str] = None - permissions: List[str] - - model_config = ConfigDict(extra="forbid") - - -# --------------------------------------------------------------------------- -# Plan entitlements + descriptions -# --------------------------------------------------------------------------- - -# Effective state is a dict[plan_slug, plan_entry] where plan_entry has the same -# shape as DEFAULT_ENTITLEMENTS values (Tracker -> mapping) plus an optional -# "description" key in the top-level plan entry's own description map. - -_DEFAULT_PLAN_DESCRIPTIONS: Dict[str, str] = {} - - -def _default_plans() -> Dict[str, Dict[Tracker, Any]]: - # Keys in DEFAULT_ENTITLEMENTS are `DefaultPlan` (str, Enum) members. - # Coerce to plain strings so the runtime plan map is uniformly keyed by slug. - return {str(plan.value): entry for plan, entry in DEFAULT_ENTITLEMENTS.items()} - - -def _validate_flag_key(key: str) -> Flag: - try: - return Flag(key) - except ValueError as e: - raise ValueError(f"Unknown flag '{key}'") from e - - -def _validate_counter_key(key: str) -> Counter: - try: - return Counter(key) - except ValueError as e: - raise ValueError(f"Unknown counter '{key}'") from e - - -def _validate_gauge_key(key: str) -> Gauge: - try: - return Gauge(key) - except ValueError as e: - raise ValueError(f"Unknown gauge '{key}'") from e - - -def _parse_plans_override( - decoded: Any, -) -> tuple[Dict[str, Dict[Tracker, Any]], Dict[str, str]]: - if not isinstance(decoded, dict) or not decoded: - raise ValueError("AGENTA_ACCESS_PLANS must be a non-empty JSON object") - - plans: Dict[str, Dict[Tracker, Any]] = {} - descriptions: Dict[str, str] = {} - - for slug, payload in decoded.items(): - if not slug or not isinstance(slug, str): - raise ValueError(f"Invalid plan slug '{slug}' in AGENTA_ACCESS_PLANS") - - try: - override = _PlanOverride.model_validate(payload) - except ValidationError as e: - raise ValueError( - f"Invalid plan override for '{slug}' in AGENTA_ACCESS_PLANS: {e}" - ) from e - - plan_entry: Dict[Tracker, Any] = {} - - if override.flags is not None: - plan_entry[Tracker.FLAGS] = { - _validate_flag_key(k): bool(v) for k, v in override.flags.items() - } - - if override.counters is not None: - plan_entry[Tracker.COUNTERS] = { - _validate_counter_key(k): v for k, v in override.counters.items() - } - - if override.gauges is not None: - plan_entry[Tracker.GAUGES] = { - _validate_gauge_key(k): v for k, v in override.gauges.items() - } - - if override.throttles is not None: - plan_entry[Tracker.THROTTLES] = list(override.throttles) - - # Plans with only a description are allowed — they represent custom / - # display-only plans (e.g. self-hosted Enterprise) that don't enforce - # quotas server-side. Downstream consumers must handle the empty - # entitlement map gracefully (e.g. `fetch_usage` returns no rows). - plans[slug] = plan_entry - - if override.description: - descriptions[slug] = override.description - - return plans, descriptions - - -# --------------------------------------------------------------------------- -# Role catalogs (scoped) -# --------------------------------------------------------------------------- -# -# Minima contract: every scope MUST expose `owner` and `viewer` with code- -# defined permissions. Env overrides may add roles or customize permissions -# of non-minima roles, but the minima are always present and their slugs -# cannot be re-bound to a different permission set. - - -def _read_only_permissions() -> List[str]: - """Read-only permission set sourced from the code-default `WorkspaceRole.VIEWER`. - - Used as the `viewer` minima permissions for the `workspace` and `project` - scopes where permissions are actually enforced. Organization-scope `viewer` - has no permissions (it's just a membership marker today). - """ - return [p.value for p in Permission.default_permissions(WorkspaceRole.VIEWER)] - - -def _viewer_permissions_for_scope(scope: str) -> List[str]: - """Per-scope code-default permissions for the `viewer` minima role. - - - `organization`: empty — orgs have no permission concept today. - - `workspace` and `project`: the code-default `WorkspaceRole.VIEWER` read-only set. - """ - if scope == "organization": - return [] - return _read_only_permissions() - - -def _minima_for(scope: str) -> List[Dict[str, Any]]: - """Return the required role entries for a scope (owner + viewer). - - Application code relies on these slugs being present in every scope; the - builder synthesizes them up front and re-applies them after any env - overrides so they can never be dropped or relabeled. - - The permission set for `viewer` varies per scope (see - `_viewer_permissions_for_scope`); `owner` is always wildcard. - """ - return [ - { - "role": DefaultRole.OWNER.value, - "description": "Full access (wildcard permissions).", - "permissions": list(OWNER_PERMISSIONS), - }, - { - "role": DefaultRole.VIEWER.value, - "description": ( - "Membership marker (no permissions)." - if scope == "organization" - else "Read-only access." - ), - "permissions": _viewer_permissions_for_scope(scope), - }, - ] - - -def _default_roles() -> Dict[str, List[Dict[str, Any]]]: - """Return the code-default role catalog for each scope. - - Workspace and project scopes expose the code-default `WorkspaceRole` - entries on top of the minima — project membership stores the same role - slugs (`admin`/`developer`/`editor`/`annotator`), and the runtime - permission check resolves them through this map. Organization scope - only gets the minima today; new roles can be added per-scope via - `AGENTA_ACCESS_ROLES`. - """ - default_extras: List[Dict[str, Any]] = [] - minima_slugs = {DefaultRole.OWNER.value, DefaultRole.VIEWER.value} - for role in WorkspaceRole: - if role.value in minima_slugs: - continue - default_extras.append( - { - "role": role.value, - "description": WorkspaceRole.get_description(role), - "permissions": [p.value for p in Permission.default_permissions(role)], - } - ) - - return { - "organization": _minima_for("organization"), - "workspace": _minima_for("workspace") + default_extras, - "project": _minima_for("project") + default_extras, - } - - -def _validate_permission(slug: str) -> str: - if slug == "*": - return slug - try: - Permission(slug) - except ValueError as e: - raise ValueError(f"Unknown permission '{slug}'") from e - return slug - - -def _parse_roles_override(decoded: Any) -> Dict[str, List[Dict[str, Any]]]: - """Parse `AGENTA_ACCESS_ROLES` and merge with scope minima. - - Schema is `{scope: [role, ...]}` where scope is one of `organization`, - `workspace`, `project`. Missing scopes fall back to the code-default - minima for that scope. Within an overridden scope: - - - `owner` and `viewer` are reserved slugs; if present, they're rejected - with a clear error so application invariants stay intact. The minima - for that scope are re-applied after parsing. - - Other roles are validated (known permissions, no duplicates) and - appended to the minima list. - - Empty `{}` or an empty per-scope list is rejected. - """ - if not isinstance(decoded, dict) or not decoded: - raise ValueError("AGENTA_ACCESS_ROLES must be a non-empty JSON object") - - # Start with code-default catalogs for every scope. Overrides only mutate - # the scopes they specify; non-overridden scopes keep their full code - # defaults (e.g. workspace's admin/developer/editor/annotator stay even - # when only `project` is overridden). - result: Dict[str, List[Dict[str, Any]]] = _default_roles() - - reserved = {DefaultRole.OWNER.value, DefaultRole.VIEWER.value} - - for scope, roles in decoded.items(): - if scope not in SCOPES: - raise ValueError( - f"Unknown role scope '{scope}' in AGENTA_ACCESS_ROLES " - f"(allowed: {list(SCOPES)})" - ) - - if not isinstance(roles, list) or not roles: - raise ValueError( - f"AGENTA_ACCESS_ROLES['{scope}'] must be a non-empty list of roles" - ) - - extras: List[Dict[str, Any]] = [] - seen: set[str] = set() - for entry in roles: - try: - role = _RoleOverride.model_validate(entry) - except ValidationError as e: - raise ValueError( - f"Invalid role override under scope '{scope}': {e}" - ) from e - - slug = role.role - if not slug: - raise ValueError(f"Empty role slug under scope '{scope}'") - if slug in reserved: - raise ValueError( - f"AGENTA_ACCESS_ROLES['{scope}'] cannot redefine reserved " - f"role '{slug}'; minima are always synthesized by the platform" - ) - if slug in seen: - raise ValueError(f"Duplicate role slug '{slug}' under scope '{scope}'") - seen.add(slug) - - permissions = [_validate_permission(p) for p in role.permissions] - - extras.append( - { - "role": slug, - "description": role.description, - "permissions": permissions, - } - ) - - # Minima first, then validated extras. - result[scope] = _minima_for(scope) + extras - - # TEMP: workspace and project use the same role set at runtime (the only - # caller of `workspace`-scope roles today is the Invite Members modal, - # which is really inviting to the underlying project). Operators almost - # always override only `project` via AGENTA_ACCESS_ROLES; without this - # mirror, custom roles silently disappear from the workspace catalog. - # Remove once the workspace/project scope split is reconciled. - if "project" in decoded and "workspace" not in decoded: - project_extras = [ - entry - for entry in result["project"] - if entry["role"] not in {DefaultRole.OWNER.value, DefaultRole.VIEWER.value} - ] - result["workspace"] = _minima_for("workspace") + project_extras - - return result - - -# --------------------------------------------------------------------------- -# Roles overlay -# --------------------------------------------------------------------------- -# -# `AGENTA_ACCESS_ROLES_OVERLAY` lets operators tweak individual fields on -# existing default roles (`admin`, `developer`, `editor`, `annotator`) — or -# add new roles — without restating the whole scope catalog the way -# `AGENTA_ACCESS_ROLES` requires. -# -# The overlay accepts two shapes: -# -# - Project-focused shortcut (preferred for the common case): -# {: , ...} -# The whole dict is interpreted as project-level role patches. The scope -# names (`organization`, `workspace`, `project`) are reserved and cannot -# be used as role slugs in this form. -# -# - Full form, scoped: -# {"project": {: , ...}} -# Triggered when any of `organization`/`workspace`/`project` appears at -# the root. Today only `project` is supported; `organization` and -# `workspace` are rejected — silent ignore would mislead operators. -# -# In both shapes the patch is applied to both the `workspace` and `project` -# scopes because the two scopes share the same role set in the code defaults. -# -# Merge semantics per role slug: -# - role exists in the scope: per-field replace (`permissions` and/or -# `description`). -# - role does not exist: append as a new role (must include both -# `description` and `permissions`). -# - `owner` and `viewer` minima cannot be patched (platform-managed). - - -class _RoleOverlayEntry(BaseModel): - """Partial role update. ``permissions`` and ``description`` are both - optional; whatever is set replaces the matching field on the existing - role entry. To add a new role both fields must be supplied — that - constraint is enforced in :func:`_apply_roles_overlay`. - """ - - description: Optional[str] = None - permissions: Optional[List[str]] = None - - model_config = ConfigDict(extra="forbid") - - -def _parse_roles_overlay(decoded: Any) -> Dict[str, _RoleOverlayEntry]: - """Parse `AGENTA_ACCESS_ROLES_OVERLAY` and return per-slug entries. - - Two accepted payload shapes: - - 1. Project-focused shortcut — ``{: }``. Used when none - of the scope keys (``organization``, ``workspace``, ``project``) appear - at the root; the payload is interpreted as project-level role patches. - The scope names are therefore reserved and cannot be used as role slugs. - - 2. Full (scoped) form — ``{"project": {: }}``. Triggered - when any of ``organization``, ``workspace``, ``project`` appears at the - root. Only ``project`` is supported today; ``organization`` and - ``workspace`` are rejected. - - The result is the inner ``{: }`` dict; the caller decides - which scopes the patch applies to. - """ - if not isinstance(decoded, dict) or not decoded: - raise ValueError("AGENTA_ACCESS_ROLES_OVERLAY must be a non-empty JSON object") - - scope_keys = {"organization", "workspace", "project"} - - if scope_keys & set(decoded.keys()): - # Full parse: at least one scope key at root. - unsupported = (set(decoded.keys()) & scope_keys) - {"project"} - if unsupported: - raise ValueError( - f"AGENTA_ACCESS_ROLES_OVERLAY only supports the 'project' scope " - f"today (got: {sorted(unsupported)}). The patch is applied to " - "both workspace and project." - ) - unknown = set(decoded.keys()) - scope_keys - if unknown: - raise ValueError( - f"AGENTA_ACCESS_ROLES_OVERLAY mixes scope keys with non-scope " - f"keys at the root ({sorted(unknown)}). Either use the " - "project-focused shortcut (no scope keys, role slugs at the " - "root) or the full form ({'project': {...}})." - ) - project_payload = decoded.get("project") - if not isinstance(project_payload, dict) or not project_payload: - raise ValueError( - "AGENTA_ACCESS_ROLES_OVERLAY['project'] must be a non-empty " - "JSON object keyed by role slug" - ) - else: - # Project-focused shortcut: top-level dict is keyed by role slug. - project_payload = decoded - - reserved = {DefaultRole.OWNER.value, DefaultRole.VIEWER.value} - entries: Dict[str, _RoleOverlayEntry] = {} - for slug, patch in project_payload.items(): - if not slug or not isinstance(slug, str): - raise ValueError( - f"Invalid role slug '{slug}' in AGENTA_ACCESS_ROLES_OVERLAY" - ) - if slug in reserved: - raise ValueError( - f"AGENTA_ACCESS_ROLES_OVERLAY cannot patch reserved role " - f"'{slug}'; minima are platform-managed" - ) - try: - entry = _RoleOverlayEntry.model_validate(patch) - except ValidationError as e: - raise ValueError( - f"Invalid AGENTA_ACCESS_ROLES_OVERLAY['project']['{slug}']: {e}" - ) from e - if entry.permissions is not None: - for perm in entry.permissions: - _validate_permission(perm) - entries[slug] = entry - - return entries - - -def _apply_roles_overlay( - roles: Dict[str, List[Dict[str, Any]]], - overlay: Dict[str, _RoleOverlayEntry], -) -> Dict[str, List[Dict[str, Any]]]: - """Apply the parsed overlay to the workspace and project scopes. - - Returns a new dict; the input is not mutated. New roles (where the - slug doesn't already exist on the scope) require both ``description`` - and ``permissions``. - """ - result = {scope: [dict(entry) for entry in roles[scope]] for scope in roles} - - for scope_name in ("workspace", "project"): - scope_entries = result[scope_name] - by_slug = {entry["role"]: idx for idx, entry in enumerate(scope_entries)} - - for slug, patch in overlay.items(): - if slug in by_slug: - # Patch existing role: per-field replace. - idx = by_slug[slug] - if patch.description is not None: - scope_entries[idx]["description"] = patch.description - if patch.permissions is not None: - scope_entries[idx]["permissions"] = list(patch.permissions) - else: - # New role: both fields must be present. - if patch.permissions is None: - raise ValueError( - f"AGENTA_ACCESS_ROLES_OVERLAY['project']['{slug}']: " - f"new role requires 'permissions' (scope '{scope_name}' " - "has no existing role with this slug to patch)" - ) - scope_entries.append( - { - "role": slug, - "description": patch.description, - "permissions": list(patch.permissions), - } - ) - - return result - - -# --------------------------------------------------------------------------- -# Default-plan overlay -# --------------------------------------------------------------------------- -# -# `AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY` lets self-hosted operators tweak individual -# entitlement values on the default plan without restating the entire plan in -# `AGENTA_ACCESS_PLANS`. Common cases: bumping trace retention, raising the -# standard-throttle rate, flipping a flag. -# -# Shape mirrors a plan entry (same keys, same units) with one divergence: -# `throttles` is a map keyed by category slug instead of a list, so per- -# category patches don't require restating the whole list. Throttles that -# combine multiple categories or use `endpoints` cannot be addressed via the -# overlay — operators who need that should use `AGENTA_ACCESS_PLANS`. - - -class _ThrottleOverlay(BaseModel): - """Partial throttle update keyed by a single category. - - Every field is optional; only fields explicitly set on the overlay - replace the matching field on the existing throttle entry. - """ - - bucket: Optional[Dict[str, Any]] = None - mode: Optional[str] = None - - model_config = ConfigDict(extra="forbid") - - -class _DefaultPlanOverlay(BaseModel): - """Partial overlay for the default plan. Same shape as `_PlanOverride` - except `throttles` is a map keyed by category slug.""" - - description: Optional[str] = None - flags: Optional[Dict[str, bool]] = None - counters: Optional[Dict[str, Dict[str, Any]]] = None - gauges: Optional[Dict[str, Dict[str, Any]]] = None - throttles: Optional[Dict[str, _ThrottleOverlay]] = None - - model_config = ConfigDict(extra="forbid") - - -def _merge_quota(existing: Optional[Quota], patch: Dict[str, Any]) -> Quota: - """Patch a Quota field-by-field, preserving fields the overlay didn't set.""" - if existing is None: - # Patching an entitlement key that isn't defined on the base plan: - # treat the overlay as the full definition. - return Quota.model_validate(patch) - merged = existing.model_dump() - merged.update(patch) - return Quota.model_validate(merged) - - -def _merge_throttle(existing: Throttle, patch: _ThrottleOverlay) -> Throttle: - """Patch one throttle entry. `bucket` is field-merged; `mode` replaces.""" - base = existing.model_dump() - if patch.bucket is not None: - bucket = dict(base.get("bucket") or {}) - bucket.update(patch.bucket) - base["bucket"] = bucket - if patch.mode is not None: - base["mode"] = patch.mode - return Throttle.model_validate(base) - - -def _parse_default_plan_overlay(decoded: Any) -> _DefaultPlanOverlay: - if not isinstance(decoded, dict) or not decoded: - raise ValueError( - "AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY must be a non-empty JSON object" - ) - try: - overlay = _DefaultPlanOverlay.model_validate(decoded) - except ValidationError as e: - raise ValueError(f"Invalid AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY: {e}") from e - - # Validate slugs upfront so the error message points at the bad key - # rather than failing inside the merge. - if overlay.flags is not None: - for key in overlay.flags: - _validate_flag_key(key) - if overlay.counters is not None: - for key in overlay.counters: - _validate_counter_key(key) - if overlay.gauges is not None: - for key in overlay.gauges: - _validate_gauge_key(key) - if overlay.throttles is not None: - valid_categories = {c.value for c in Category} - for key in overlay.throttles: - if key not in valid_categories: - raise ValueError( - f"AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY.throttles['{key}'] is not a " - f"valid throttle category. Allowed: {sorted(valid_categories)}." - ) - return overlay - - -def _apply_default_plan_overlay( - plans: Dict[str, Dict[Tracker, Any]], - descriptions: Dict[str, str], - overlay: _DefaultPlanOverlay, - default_plan_slug: str, -) -> tuple[Dict[str, Dict[Tracker, Any]], Dict[str, str]]: - """Apply the overlay to the resolved default plan in-place (returning new dicts).""" - if default_plan_slug not in plans: - raise ValueError( - f"AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY targets the default plan " - f"'{default_plan_slug}', which is not in the effective plan set " - f"({sorted(plans.keys())}). Add the slug to AGENTA_ACCESS_PLANS or " - "unset AGENTA_DEFAULT_PLAN." - ) - - plans = {slug: dict(entry) for slug, entry in plans.items()} - descriptions = dict(descriptions) - entry = plans[default_plan_slug] - - if overlay.description is not None: - descriptions[default_plan_slug] = overlay.description - - if overlay.flags is not None: - flags = dict(entry.get(Tracker.FLAGS) or {}) - for key, value in overlay.flags.items(): - flags[Flag(key)] = bool(value) - entry[Tracker.FLAGS] = flags - - if overlay.counters is not None: - counters = dict(entry.get(Tracker.COUNTERS) or {}) - for key, patch in overlay.counters.items(): - counter = Counter(key) - counters[counter] = _merge_quota(counters.get(counter), patch) - entry[Tracker.COUNTERS] = counters - - if overlay.gauges is not None: - gauges = dict(entry.get(Tracker.GAUGES) or {}) - for key, patch in overlay.gauges.items(): - gauge = Gauge(key) - gauges[gauge] = _merge_quota(gauges.get(gauge), patch) - entry[Tracker.GAUGES] = gauges - - if overlay.throttles is not None: - existing_throttles = list(entry.get(Tracker.THROTTLES) or []) - - for category_key, patch in overlay.throttles.items(): - category = Category(category_key) - # Find the single-category throttle entry that matches. - target_idx = next( - ( - idx - for idx, t in enumerate(existing_throttles) - if t.categories - and len(t.categories) == 1 - and t.categories[0] == category - ), - None, - ) - if target_idx is None: - raise ValueError( - f"AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY.throttles['{category_key}']: " - f"the default plan '{default_plan_slug}' has no single-" - f"category throttle entry for '{category_key}'. Use " - "AGENTA_ACCESS_PLANS to define multi-category or endpoint-" - "keyed throttles." - ) - existing_throttles[target_idx] = _merge_throttle( - existing_throttles[target_idx], patch - ) - entry[Tracker.THROTTLES] = existing_throttles - - plans[default_plan_slug] = entry - return plans, descriptions - - -def _resolve_default_plan_slug(plans: Dict[str, Dict[Tracker, Any]]) -> str: - """Resolve the default plan slug for overlay targeting. - - Mirrors `subscriptions.types.get_default_plan()` without importing it (to - avoid pulling subscription/Stripe code into the access-controls layer). - """ - raw = env.agenta.access.default_plan - if raw: - return raw - if env.stripe.enabled: - return DefaultPlan.CLOUD_V0_HOBBY.value - return DefaultPlan.SELF_HOSTED_ENTERPRISE.value - - -# --------------------------------------------------------------------------- -# Effective controls (built once at import time) -# --------------------------------------------------------------------------- - - -def _build_controls() -> tuple[ - Dict[str, Dict[Tracker, Any]], - Dict[str, str], - Dict[str, List[Dict[str, Any]]], - str, -]: - plans_payload = env.agenta.access.plans - roles_payload = env.agenta.access.roles - roles_overlay_payload = env.agenta.access.roles_overlay - plan_overlay_payload = env.agenta.access.default_plan_overlay - - if plans_payload is not None: - plans, descriptions = _parse_plans_override(plans_payload) - plans_source = "env" - else: - plans = _default_plans() - descriptions = dict(_DEFAULT_PLAN_DESCRIPTIONS) - plans_source = "defaults" - - plan_overlay_source = "none" - if plan_overlay_payload is not None: - plan_overlay = _parse_default_plan_overlay(plan_overlay_payload) - default_plan_slug = _resolve_default_plan_slug(plans) - plans, descriptions = _apply_default_plan_overlay( - plans, descriptions, plan_overlay, default_plan_slug - ) - plan_overlay_source = f"env→{default_plan_slug}" - - if roles_payload is not None: - roles = _parse_roles_override(roles_payload) - roles_source = "env" - else: - roles = _default_roles() - roles_source = "defaults" - - roles_overlay_source = "none" - if roles_overlay_payload is not None: - roles_overlay = _parse_roles_overlay(roles_overlay_payload) - roles = _apply_roles_overlay(roles, roles_overlay) - roles_overlay_source = "env" - - payload = dumps( - { - "plans": sorted(plans.keys()), - "descriptions": descriptions, - "roles": {scope: [r["role"] for r in roles[scope]] for scope in SCOPES}, - }, - sort_keys=True, - default=str, - ) - controls_hash = hashlib.sha256(payload.encode()).hexdigest()[:12] - - log.info( - "[access-controls] plans=%s roles=%s plan_overlay=%s roles_overlay=%s hash=%s", - plans_source, - roles_source, - plan_overlay_source, - roles_overlay_source, - controls_hash, - ) - - return plans, descriptions, roles, controls_hash - - -_PLANS, _PLAN_DESCRIPTIONS, _ROLES, _CONTROLS_HASH = _build_controls() - - -# --------------------------------------------------------------------------- -# Public accessors -# --------------------------------------------------------------------------- - - -def get_plans() -> Dict[str, Dict[Tracker, Any]]: - """Return the effective plan map (slug -> entitlement controls).""" - return _PLANS - - -def get_plan(slug: Optional[str]) -> Optional[Dict[Tracker, Any]]: - """Return the entitlement controls for a plan slug, or None if missing.""" - if not slug: - return None - return _PLANS.get(slug) - - -def get_plan_entitlements(slug: Optional[str]) -> Optional[Dict[Tracker, Any]]: - """Alias for `get_plan`. Kept distinct for readability at call sites.""" - if not slug: - return None - return _PLANS.get(slug) - - -def get_plan_description(slug: Optional[str]) -> Optional[str]: - """Return the operator-facing description for a plan, if any.""" - if not slug: - return None - return _PLAN_DESCRIPTIONS.get(slug) - - -def get_roles(scope: str) -> List[Dict[str, Any]]: - """Return the effective role catalog for a scope.""" - if scope not in SCOPES: - return [] - return _ROLES[scope] - - -def get_role(scope: str, slug: str) -> Optional[Dict[str, Any]]: - """Return a single role entry within a scope.""" - for entry in get_roles(scope): - if entry["role"] == slug: - return entry - return None - - -def get_role_permissions(scope: str, slug: str) -> List[str]: - """Return the permission slugs for a role within a scope.""" - role = get_role(scope, slug) - if not role: - return [] - return list(role["permissions"]) - - -def get_role_description(scope: str, slug: str) -> Optional[str]: - role = get_role(scope, slug) - if not role: - return None - return role.get("description") - - -def get_controls_hash() -> str: - """Stable short hash of the effective controls; useful in logs.""" - return _CONTROLS_HASH diff --git a/api/ee/src/core/entitlements/service.py b/api/ee/src/core/entitlements/service.py deleted file mode 100644 index 5cff086467..0000000000 --- a/api/ee/src/core/entitlements/service.py +++ /dev/null @@ -1,98 +0,0 @@ -from typing import Optional, Dict, List - -from ee.src.core.entitlements.types import ( - Tracker, - Constraint, - CONSTRAINTS, -) -from ee.src.core.entitlements.types import Quota, Gauge -from ee.src.core.entitlements.controls import get_plan_entitlements -from ee.src.core.meters.service import MetersService -from ee.src.core.meters.types import MeterDTO - - -class ConstaintsException(Exception): - issues: Dict[Gauge, int] = {} - - -class EntitlementsService: - def __init__( - self, - meters_service: MetersService, - ): - self.meters_service = meters_service - - async def enforce( - self, - *, - organization_id: str, - plan: str, - force: Optional[bool] = False, - ) -> None: - issues = await self.check( - organization_id=organization_id, - plan=plan, - ) - - if issues: - if not force: - raise ConstaintsException( - issues=issues, - ) - - await self.fix( - organization_id=organization_id, - issues=issues, - ) - - async def check( - self, - *, - organization_id: str, - plan: str, - ) -> Dict[Gauge, int]: - issues = {} - - entitlements = get_plan_entitlements(plan) or {} - - for key in CONSTRAINTS[Constraint.BLOCKED][Tracker.GAUGES]: - quotas: List[Quota] = entitlements.get(Tracker.GAUGES) or {} - - if key in quotas: - meter = MeterDTO( - organization_id=organization_id, - key=key, - ) - quota: Quota = quotas[key] - - check, meter = await self.meters_service.check( - meter=meter, - quota=quota, - ) - - if not check: - issues[key] = quota.limit - - return issues - - async def fix( - self, - *, - organization_id: str, - issues: Dict[Gauge, int], - ) -> None: - # TODO: Implement fix - pass - - -# TODO: -# -- P0 / MUST -# - Add active : Optional[bool] = None to all scopes and users -# -- P1 / SHOULD -# - Add parent scopes to all child scope -# - Add parent scopes membership on child scope membership creation -# - Remove children scopes membership on parent scope membership removal -# -- P2 / COULD -# - Add created_at / updated_at to all scopes -# - Set updated_at on all updates + on creation -# - Move organization roles to memberships diff --git a/api/ee/src/core/events/service.py b/api/ee/src/core/events/service.py index eef6d17e47..b52b292c20 100644 --- a/api/ee/src/core/events/service.py +++ b/api/ee/src/core/events/service.py @@ -13,8 +13,8 @@ from oss.src.utils.logging import get_module_logger -from ee.src.core.entitlements.types import Tracker, Counter -from ee.src.core.entitlements.controls import get_plans +from ee.src.core.access.entitlements.types import Tracker, Counter +from ee.src.core.access.controls import get_plans from ee.src.dbs.postgres.events.dao import EventsRetentionDAO diff --git a/api/ee/src/core/meters/interfaces.py b/api/ee/src/core/meters/interfaces.py index 924aafff92..52a46de4b4 100644 --- a/api/ee/src/core/meters/interfaces.py +++ b/api/ee/src/core/meters/interfaces.py @@ -1,6 +1,6 @@ from typing import Tuple, Callable, Optional -from ee.src.core.entitlements.types import Quota +from ee.src.core.access.entitlements.types import Quota from ee.src.core.meters.types import MeterDTO, MeterScope, MeterPeriod, Meters diff --git a/api/ee/src/core/meters/service.py b/api/ee/src/core/meters/service.py index 8d6d9588d5..9a76add021 100644 --- a/api/ee/src/core/meters/service.py +++ b/api/ee/src/core/meters/service.py @@ -1,13 +1,11 @@ from typing import Awaitable, Tuple, Callable, List, Optional from uuid import uuid4 -import stripe - from oss.src.utils.logging import get_module_logger -from oss.src.utils.env import env +from oss.src.utils.lazy import _load_stripe -from ee.src.core.entitlements.types import Quota -from ee.src.core.entitlements.types import Counter, Gauge, REPORTS, STRIPE_METER_NAMES +from ee.src.core.access.entitlements.types import Quota +from ee.src.core.access.entitlements.types import Counter, Gauge, REPORTS from ee.src.core.subscriptions.settings import get_stripe_meter_price from ee.src.core.meters.types import MeterDTO, MeterScope, MeterPeriod, Meters from ee.src.core.meters.interfaces import MetersDAOInterface @@ -22,13 +20,6 @@ _GAUGE_SLUGS: frozenset[str] = frozenset(g.value for g in Gauge) _COUNTER_SLUGS: frozenset[str] = frozenset(c.value for c in Counter) -# Initialize Stripe only if enabled -if env.stripe.enabled: - stripe.api_key = env.stripe.api_key - log.info("✓ Stripe enabled:", target=env.stripe.webhook_target) -else: - log.info("✗ Stripe disabled") - class MetersService: def __init__( @@ -85,8 +76,9 @@ async def report( self, renew: Optional[Callable[[], Awaitable[bool]]] = None, ): - if not env.stripe.enabled: - log.warn("✗ Stripe disabled") + stripe = _load_stripe() + if stripe is None: + log.warn("✗ Stripe unavailable") return log.info("[report] ============================================") @@ -177,8 +169,8 @@ async def report( if meter.key.value in _GAUGE_SLUGS: try: - stripe_meter_name = STRIPE_METER_NAMES.get(meter.key.value) - if not stripe_meter_name: + meter_name = REPORTS.get(meter.key.value) + if not meter_name: log.warn( f"[report] Skipping meter {meter.organization_id}/{meter.key} - no Stripe meter mapping" ) @@ -188,7 +180,7 @@ async def report( price_id = get_stripe_meter_price( meter.subscription.plan, - stripe_meter_name, + meter_name, ) if not price_id: @@ -256,7 +248,7 @@ async def report( if meter.key.value in _COUNTER_SLUGS: try: - event_name = STRIPE_METER_NAMES.get(meter.key.value) + event_name = REPORTS.get(meter.key.value) if not event_name: log.warn( f"[report] Skipping meter {meter.organization_id}/{meter.key} - no Stripe meter mapping" diff --git a/api/ee/src/core/meters/types.py b/api/ee/src/core/meters/types.py index 3b9cbc8f04..aa228d54c1 100644 --- a/api/ee/src/core/meters/types.py +++ b/api/ee/src/core/meters/types.py @@ -8,7 +8,7 @@ from oss.src.utils.logging import get_module_logger -from ee.src.core.entitlements.types import Counter, Gauge +from ee.src.core.access.entitlements.types import Counter, Gauge from ee.src.core.subscriptions.types import SubscriptionDTO diff --git a/api/ee/src/core/subscriptions/service.py b/api/ee/src/core/subscriptions/service.py index 75edaf313f..bd240397ca 100644 --- a/api/ee/src/core/subscriptions/service.py +++ b/api/ee/src/core/subscriptions/service.py @@ -2,12 +2,10 @@ from uuid import getnode from datetime import datetime, timezone, timedelta - -import stripe - from oss.src.utils.logging import get_module_logger from oss.src.utils.env import env from oss.src.utils.caching import invalidate_cache +from oss.src.utils.lazy import _load_stripe from ee.src.core.subscriptions.types import ( SubscriptionDTO, @@ -23,18 +21,9 @@ trial_enabled, ) from ee.src.core.subscriptions.interfaces import SubscriptionsDAOInterface -from ee.src.core.entitlements.service import EntitlementsService -from ee.src.core.meters.service import MetersService log = get_module_logger(__name__) -# Initialize Stripe only if enabled -if env.stripe.enabled: - stripe.api_key = env.stripe.api_key - log.info("✓ Stripe enabled:", target=env.stripe.webhook_target) -else: - log.info("✗ Stripe disabled") - MAC_ADDRESS = ":".join(f"{(getnode() >> ele) & 0xFF:02x}" for ele in range(40, -1, -8)) @@ -50,11 +39,8 @@ class SubscriptionsService: def __init__( self, subscriptions_dao: SubscriptionsDAOInterface, - meters_service: MetersService, ): self.subscriptions_dao = subscriptions_dao - self.meters_service = meters_service - self.entitlements_service = EntitlementsService(meters_service=meters_service) async def create( self, @@ -84,8 +70,9 @@ async def start_reverse_trial( organization_name: str, organization_email: str, ) -> Optional[SubscriptionDTO]: - if not env.stripe.enabled: - raise EventException("Reverse trial requires Stripe to be enabled") + stripe = _load_stripe() + if stripe is None: + raise EventException("Reverse trial requires Stripe to be available") if not trial_enabled(): raise EventException( @@ -299,9 +286,10 @@ async def process_event( subscription = await self.update(subscription=subscription) elif subscription.plan != free_plan and event == Event.SUBSCRIPTION_SWITCHED: - if not env.stripe.enabled: - log.warn("✗ Stripe disabled") - return None + stripe = _load_stripe() + if stripe is None: + log.warn("✗ Stripe unavailable") + raise EventException("Stripe is not available for plan switching") if subscription.plan == plan: log.warn("Subscription already on the plan: %s", plan) @@ -331,12 +319,6 @@ async def process_event( subscription.active = True subscription.plan = plan - # await self.entitlements_service.enforce( - # organization_id=organization_id, - # plan=plan, - # force=force, - # ) - stripe.Subscription.modify( subscription.subscription_id, items=[ @@ -356,12 +338,6 @@ async def process_event( subscription.subscription_id = None subscription.anchor = anchor - # await self.entitlements_service.enforce( - # organization_id=organization_id, - # plan=free_plan, - # force=True, - # ) - subscription = await self.update(subscription=subscription) elif subscription.plan == free_plan and event == Event.SUBSCRIPTION_CANCELLED: diff --git a/api/ee/src/core/subscriptions/settings.py b/api/ee/src/core/subscriptions/settings.py index 8eec37e230..8c101977ec 100644 --- a/api/ee/src/core/subscriptions/settings.py +++ b/api/ee/src/core/subscriptions/settings.py @@ -17,8 +17,8 @@ from oss.src.utils.env import env from oss.src.utils.logging import get_module_logger -from ee.src.core.entitlements.controls import get_plans -from ee.src.core.entitlements.types import ( +from ee.src.core.access.controls import get_plans +from ee.src.core.access.entitlements.types import ( DEFAULT_CATALOG, DefaultPlan, ) @@ -124,7 +124,7 @@ def _normalize_pricing_entry(slug: str, entry: Any) -> Dict[str, Any]: plus an optional `"quantity"` (default `1` when omitted). The internal `Counter` / `Gauge` slug → Stripe-side slot name map lives - in `ee.src.core.entitlements.types.STRIPE_METER_NAMES`. The runtime in + in `ee.src.core.access.entitlements.types.REPORTS`. The runtime in `ee.src.core.meters.service` resolves the internal slug through that map before looking up the price ID here; that is why the operator's pricing JSON uses Stripe-side names (matching their Stripe dashboard) @@ -459,7 +459,7 @@ def get_stripe_meter_price( `meter` is a **Stripe-side meter slot name** (e.g. `"users"`, `"traces"`) — the operator-configured identifier on the Stripe dashboard, not the internal `Counter` / `Gauge` slug. Callers in `meters/service.py` - resolve the internal slug through `STRIPE_METER_NAMES` before calling + resolve the internal slug through `REPORTS` before calling this. Returns `None` when the plan has no pricing, the meter slot is absent, or the slot carries no `"price"`. """ diff --git a/api/ee/src/core/subscriptions/types.py b/api/ee/src/core/subscriptions/types.py index 46e330c42a..5033b8e536 100644 --- a/api/ee/src/core/subscriptions/types.py +++ b/api/ee/src/core/subscriptions/types.py @@ -7,7 +7,7 @@ from oss.src.utils.env import env from pydantic import BaseModel -from ee.src.core.entitlements.types import DefaultPlan +from ee.src.core.access.entitlements.types import DefaultPlan class Event(str, Enum): diff --git a/api/ee/src/core/tracing/service.py b/api/ee/src/core/tracing/service.py index 3be3aede78..e4ff413a4b 100644 --- a/api/ee/src/core/tracing/service.py +++ b/api/ee/src/core/tracing/service.py @@ -2,8 +2,8 @@ from oss.src.utils.logging import get_module_logger -from ee.src.core.entitlements.types import Tracker, Counter -from ee.src.core.entitlements.controls import get_plans +from ee.src.core.access.entitlements.types import Tracker, Counter +from ee.src.core.access.controls import get_plans from ee.src.dbs.postgres.tracing.dao import TracingRetentionDAO diff --git a/api/ee/src/core/workspaces/types.py b/api/ee/src/core/workspaces/types.py index 4016a564df..3dfa04f9cc 100644 --- a/api/ee/src/core/workspaces/types.py +++ b/api/ee/src/core/workspaces/types.py @@ -3,13 +3,13 @@ from pydantic import BaseModel -from ee.src.models.shared_models import Permission +from ee.src.core.access.permissions.types import Permission class WorkspacePermission(BaseModel): # Role slugs are dynamic (env-overridable via AGENTA_ACCESS_ROLES); # validation against the effective scope catalog happens at the API - # boundary via `ee.src.core.entitlements.controls.get_role`. + # boundary via `ee.src.core.access.controls.get_role`. role_name: str role_description: Optional[str] = None permissions: Optional[List[Permission]] = None diff --git a/api/ee/src/dbs/postgres/events/dao.py b/api/ee/src/dbs/postgres/events/dao.py index b64a8fb486..53d500fa3a 100644 --- a/api/ee/src/dbs/postgres/events/dao.py +++ b/api/ee/src/dbs/postgres/events/dao.py @@ -11,7 +11,7 @@ """ from datetime import datetime -from typing import List +from typing import List, Optional from uuid import UUID from sqlalchemy import bindparam, delete, func, literal, select, tuple_ @@ -22,7 +22,12 @@ from oss.src.models.db_models import ProjectDB -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + AnalyticsEngine, + get_transactions_engine, + get_analytics_engine, +) from oss.src.dbs.postgres.events.dbes import EventDBE from ee.src.dbs.postgres.subscriptions.dbes import SubscriptionDBE @@ -38,6 +43,18 @@ class EventsRetentionDAO: owns retention only. """ + def __init__( + self, + transactions_engine: Optional[TransactionsEngine] = None, + analytics_engine: Optional[AnalyticsEngine] = None, + ): + if transactions_engine is None: + transactions_engine = get_transactions_engine() + if analytics_engine is None: + analytics_engine = get_analytics_engine() + self.transactions_engine = transactions_engine + self.analytics_engine = analytics_engine + async def fetch_projects_with_plan( self, *, @@ -46,7 +63,7 @@ async def fetch_projects_with_plan( max_projects: int, ) -> List[UUID]: """Page through projects whose org subscribes to the given plan.""" - async with engine.core_session() as session: + async with self.transactions_engine.session() as session: stmt = ( select(ProjectDB.id) .select_from( @@ -87,7 +104,7 @@ async def delete_events_before_cutoff( if not project_ids: return 0 - async with engine.tracing_session() as session: + async with self.analytics_engine.session() as session: project_ids_param = bindparam( "project_ids", value=project_ids, diff --git a/api/ee/src/dbs/postgres/meters/dao.py b/api/ee/src/dbs/postgres/meters/dao.py index 100e8f0f16..26eb462577 100644 --- a/api/ee/src/dbs/postgres/meters/dao.py +++ b/api/ee/src/dbs/postgres/meters/dao.py @@ -8,14 +8,17 @@ from oss.src.utils.logging import get_module_logger -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) -from ee.src.core.entitlements.types import Quota +from ee.src.core.access.entitlements.types import Quota from ee.src.core.meters.types import MeterDTO, MeterScope, MeterPeriod, Meters from ee.src.core.subscriptions.types import SubscriptionDTO from ee.src.core.meters.interfaces import MetersDAOInterface from ee.src.dbs.postgres.meters.dbes import MeterDBE -from ee.src.utils.entitlements import period_from +from ee.src.core.access.entitlements.service import period_from log = get_module_logger(__name__) @@ -90,8 +93,10 @@ def _format_meter_for_log(meter: MeterDTO) -> str: class MetersDAO(MetersDAOInterface): - def __init__(self): - pass + def __init__(self, engine: TransactionsEngine = None): + if engine is None: + engine = get_transactions_engine() + self.engine = engine async def dump( self, @@ -99,7 +104,7 @@ async def dump( ) -> list[MeterDTO]: log.info(f"[report] [dump] Starting (limit={limit or 'none'})") - async with engine.core_session() as session: + async with self.engine.session() as session: try: stmt = ( select(MeterDBE) @@ -252,7 +257,7 @@ async def _bump_commit_chunk( missing_count = 0 missing_samples: list[str] = [] - async with engine.core_session() as session: + async with self.engine.session() as session: for meter in meters: stmt = ( update(MeterDBE) @@ -300,7 +305,7 @@ async def fetch( pins lifetime/gauge-sentinel rows (`year/month/day IS NULL`). Any other `MeterPeriod` binds every period dim uniformly. """ - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(MeterDBE).options( joinedload(MeterDBE.subscription) ) # NO RISK OF DEADLOCK @@ -341,7 +346,7 @@ async def check( ) -> Tuple[bool, MeterDTO]: meter = _normalize_period_on_meter(meter, quota, anchor) - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(MeterDBE).filter_by( meter_id=meter.meter_id, ) # NO RISK OF DEADLOCK @@ -458,7 +463,7 @@ async def adjust( where = where | where_clause # 4. Build SQL statement (atomic upsert with RETURNING) - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( insert(MeterDBE) .values( diff --git a/api/ee/src/dbs/postgres/organizations/dao.py b/api/ee/src/dbs/postgres/organizations/dao.py index 00658837db..ead881360c 100644 --- a/api/ee/src/dbs/postgres/organizations/dao.py +++ b/api/ee/src/dbs/postgres/organizations/dao.py @@ -1,9 +1,13 @@ from typing import Optional, List +from datetime import datetime, timezone from sqlalchemy import select, and_ from sqlalchemy.ext.asyncio import AsyncSession -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) from ee.src.dbs.postgres.organizations.dbes import ( OrganizationDomainDBE, OrganizationProviderDBE, @@ -18,8 +22,15 @@ class OrganizationDomainsDAO: 2. Without a session (creates own sessions): OrganizationDomainsDAO() """ - def __init__(self, session: Optional[AsyncSession] = None): + def __init__( + self, + session: Optional[AsyncSession] = None, + engine: Optional[TransactionsEngine] = None, + ): self.session = session + if engine is None: + engine = get_transactions_engine() + self.engine = engine async def create( self, @@ -54,7 +65,7 @@ async def create( return domain else: - async with engine.core_session() as session: + async with self.engine.session() as session: domain = OrganizationDomainDBE( organization_id=organization_id, slug=slug, @@ -92,7 +103,7 @@ async def get_by_id( return result.scalars().first() else: - async with engine.core_session() as session: + async with self.engine.session() as session: result = await session.execute( select(OrganizationDomainDBE).where( and_( @@ -125,7 +136,7 @@ async def get_by_slug( return result.scalars().first() else: - async with engine.core_session() as session: + async with self.engine.session() as session: result = await session.execute( select(OrganizationDomainDBE).where( and_( @@ -158,7 +169,7 @@ async def get_verified_by_slug( return result.scalars().first() else: - async with engine.core_session() as session: + async with self.engine.session() as session: result = await session.execute( select(OrganizationDomainDBE).where( and_( @@ -186,7 +197,7 @@ async def list_by_organization( return list(result.scalars().all()) else: - async with engine.core_session() as session: + async with self.engine.session() as session: result = await session.execute( select(OrganizationDomainDBE).where( OrganizationDomainDBE.organization_id == organization_id @@ -207,9 +218,12 @@ async def update_flags( """Update domain flags (e.g., mark as verified).""" domain = await self.get_by_id(domain_id=domain_id, organization_id="") + now = datetime.now(timezone.utc) + if self.session: if domain: domain.flags = flags + domain.updated_at = now domain.updated_by_id = updated_by_id await self.session.flush() @@ -218,13 +232,14 @@ async def update_flags( return domain else: - async with engine.core_session() as session: + async with self.engine.session() as session: if domain: # Re-attach to new session domain = await session.get(OrganizationDomainDBE, domain_id) if domain: domain.flags = flags + domain.updated_at = now domain.updated_by_id = updated_by_id await session.commit() @@ -252,7 +267,7 @@ async def delete( return False else: - async with engine.core_session() as session: + async with self.engine.session() as session: domain = await session.get(OrganizationDomainDBE, domain_id) if domain: @@ -272,8 +287,15 @@ class OrganizationProvidersDAO: 2. Without a session (creates own sessions): OrganizationProvidersDAO() """ - def __init__(self, session: Optional[AsyncSession] = None): + def __init__( + self, + session: Optional[AsyncSession] = None, + engine: Optional[TransactionsEngine] = None, + ): self.session = session + if engine is None: + engine = get_transactions_engine() + self.engine = engine async def create( self, @@ -311,7 +333,7 @@ async def create( return provider else: - async with engine.core_session() as session: + async with self.engine.session() as session: provider = OrganizationProviderDBE( organization_id=organization_id, slug=slug, @@ -350,7 +372,7 @@ async def get_by_id( return result.scalars().first() else: - async with engine.core_session() as session: + async with self.engine.session() as session: result = await session.execute( select(OrganizationProviderDBE).where( and_( @@ -378,7 +400,7 @@ async def get_by_id_any( return result.scalars().first() else: - async with engine.core_session() as session: + async with self.engine.session() as session: result = await session.execute( select(OrganizationProviderDBE).where( OrganizationProviderDBE.id == provider_id @@ -408,7 +430,7 @@ async def get_by_slug( return result.scalars().first() else: - async with engine.core_session() as session: + async with self.engine.session() as session: result = await session.execute( select(OrganizationProviderDBE).where( and_( @@ -436,7 +458,7 @@ async def list_by_organization( return list(result.scalars().all()) else: - async with engine.core_session() as session: + async with self.engine.session() as session: result = await session.execute( select(OrganizationProviderDBE).where( OrganizationProviderDBE.organization_id == organization_id @@ -458,6 +480,8 @@ async def update( # ) -> Optional[OrganizationProviderDBE]: """Update a provider's secret reference or flags.""" + now = datetime.now(timezone.utc) + if self.session: provider = await self.session.get(OrganizationProviderDBE, provider_id) @@ -466,6 +490,7 @@ async def update( provider.secret_id = secret_id if flags is not None: provider.flags = flags + provider.updated_at = now if updated_by_id: provider.updated_by_id = updated_by_id @@ -475,7 +500,7 @@ async def update( return provider else: - async with engine.core_session() as session: + async with self.engine.session() as session: provider = await session.get(OrganizationProviderDBE, provider_id) if provider: @@ -483,6 +508,7 @@ async def update( provider.secret_id = secret_id if flags is not None: provider.flags = flags + provider.updated_at = now if updated_by_id: provider.updated_by_id = updated_by_id @@ -511,7 +537,7 @@ async def delete( return False else: - async with engine.core_session() as session: + async with self.engine.session() as session: provider = await session.get(OrganizationProviderDBE, provider_id) if provider: diff --git a/api/ee/src/dbs/postgres/subscriptions/dao.py b/api/ee/src/dbs/postgres/subscriptions/dao.py index 93d67dcac4..58f7f26f03 100644 --- a/api/ee/src/dbs/postgres/subscriptions/dao.py +++ b/api/ee/src/dbs/postgres/subscriptions/dao.py @@ -5,7 +5,10 @@ from ee.src.core.subscriptions.types import SubscriptionDTO from ee.src.core.subscriptions.interfaces import SubscriptionsDAOInterface -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) from ee.src.dbs.postgres.subscriptions.dbes import SubscriptionDBE from ee.src.dbs.postgres.subscriptions.mappings import ( map_dbe_to_dto, @@ -14,15 +17,17 @@ class SubscriptionsDAO(SubscriptionsDAOInterface): - def __init__(self): - pass + def __init__(self, engine: TransactionsEngine = None): + if engine is None: + engine = get_transactions_engine() + self.engine = engine async def create( self, *, subscription: SubscriptionDTO, ) -> SubscriptionDTO: - async with engine.core_session() as session: + async with self.engine.session() as session: subscription_dbe = map_dto_to_dbe(subscription) session.add(subscription_dbe) @@ -38,7 +43,7 @@ async def read( *, organization_id: str, ) -> Optional[SubscriptionDTO]: - async with engine.core_session() as session: + async with self.engine.session() as session: result = await session.execute( select(SubscriptionDBE).where( SubscriptionDBE.organization_id == organization_id, @@ -59,7 +64,7 @@ async def update( *, subscription: SubscriptionDTO, ) -> Optional[SubscriptionDTO]: - async with engine.core_session() as session: + async with self.engine.session() as session: result = await session.execute( select(SubscriptionDBE).where( SubscriptionDBE.organization_id == subscription.organization_id, diff --git a/api/ee/src/dbs/postgres/tracing/dao.py b/api/ee/src/dbs/postgres/tracing/dao.py index 34143a24a1..5f6911912e 100644 --- a/api/ee/src/dbs/postgres/tracing/dao.py +++ b/api/ee/src/dbs/postgres/tracing/dao.py @@ -10,7 +10,12 @@ from oss.src.models.db_models import ProjectDB -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + AnalyticsEngine, + get_transactions_engine, + get_analytics_engine, +) from oss.src.dbs.postgres.tracing.dbes import SpanDBE from ee.src.dbs.postgres.subscriptions.dbes import SubscriptionDBE @@ -70,6 +75,18 @@ class TracingRetentionDAO: + def __init__( + self, + transactions_engine: TransactionsEngine = None, + analytics_engine: AnalyticsEngine = None, + ): + if transactions_engine is None: + transactions_engine = get_transactions_engine() + if analytics_engine is None: + analytics_engine = get_analytics_engine() + self.transactions_engine = transactions_engine + self.analytics_engine = analytics_engine + # ---------------- # # Raw-SQL versions # ---------------- # @@ -81,7 +98,7 @@ async def _fetch_projects_with_plan( project_id: Optional[UUID], max_projects: int, ) -> List[UUID]: - async with engine.core_session() as session: + async with self.transactions_engine.session() as session: result = await session.execute( CORE_PROJECTS_PAGE_SQL, { @@ -105,7 +122,7 @@ async def _delete_traces_before_cutoff( if not project_ids: return (0, 0) - async with engine.tracing_session() as session: + async with self.analytics_engine.session() as session: result = await session.execute( TRACING_DELETE_SQL, { @@ -135,7 +152,7 @@ async def fetch_projects_with_plan( project_id: Optional[UUID], max_projects: int, ) -> List[UUID]: - async with engine.core_session() as session: + async with self.transactions_engine.session() as session: stmt = ( select(ProjectDB.id) .select_from( @@ -167,7 +184,7 @@ async def delete_traces_before_cutoff( if not project_ids: return (0, 0) - async with engine.tracing_session() as session: + async with self.analytics_engine.session() as session: project_ids_param = bindparam( "project_ids", value=project_ids, diff --git a/api/ee/src/main.py b/api/ee/src/main.py index c1f583f7e6..b5b13e094e 100644 --- a/api/ee/src/main.py +++ b/api/ee/src/main.py @@ -3,6 +3,11 @@ from oss.src.utils.env import env from oss.src.utils.logging import get_module_logger + +from oss.src.dbs.postgres.shared.engine import ( + get_transactions_engine, + get_analytics_engine, +) from oss.src.dbs.postgres.events.dao import EventsDAO from oss.src.core.events.service import EventsService @@ -14,6 +19,7 @@ from ee.src.dbs.postgres.meters.dao import MetersDAO from ee.src.dbs.postgres.tracing.dao import TracingRetentionDAO from ee.src.dbs.postgres.subscriptions.dao import SubscriptionsDAO +from ee.src.dbs.postgres.organizations.dao import OrganizationDomainsDAO from ee.src.dbs.postgres.events.dao import EventsRetentionDAO from ee.src.core.meters.service import MetersService @@ -28,18 +34,30 @@ from ee.src.apis.fastapi.organizations.router import ( router as organization_router, ) -from ee.src.utils.entitlements import bootstrap_entitlements_services +from ee.src.core.access.entitlements.service import bootstrap_entitlements_services # DBS -------------------------------------------------------------------------- -meters_dao = MetersDAO() +# Get engines from shared initialization (instantiated in routers.py) +_transactions_engine = get_transactions_engine() +_analytics_engine = get_analytics_engine() + +meters_dao = MetersDAO(engine=_transactions_engine) + +tracing_retention_dao = TracingRetentionDAO( + transactions_engine=_transactions_engine, + analytics_engine=_analytics_engine, +) -tracing_retention_dao = TracingRetentionDAO() +subscriptions_dao = SubscriptionsDAO(engine=_transactions_engine) -subscriptions_dao = SubscriptionsDAO() +organization_domains_dao = OrganizationDomainsDAO(engine=_transactions_engine) -events_retention_dao = EventsRetentionDAO() -query_events_dao = EventsDAO() +events_dao = EventsDAO(engine=_analytics_engine) +events_retention_dao = EventsRetentionDAO( + transactions_engine=_transactions_engine, + analytics_engine=_analytics_engine, +) # CORE ------------------------------------------------------------------------- @@ -51,16 +69,16 @@ tracing_retention_dao=tracing_retention_dao, ) +events_service = EventsService( + events_dao=events_dao, +) + events_retention_service = EventsRetentionService( events_retention_dao=events_retention_dao, ) -query_events_service = EventsService( - events_dao=query_events_dao, -) subscription_service = SubscriptionsService( subscriptions_dao=subscriptions_dao, - meters_service=meters_service, ) # Wire entitlements module against the freshly-built services so the @@ -84,7 +102,7 @@ ) events_router = EventsRouter( - events_service=query_events_service, + events_service=events_service, ) events_retention_router = EventsRetentionRouter( diff --git a/api/ee/src/services/throttling_service.py b/api/ee/src/middlewares/throttling.py similarity index 96% rename from api/ee/src/services/throttling_service.py rename to api/ee/src/middlewares/throttling.py index ed6ceffcb3..d8acc6afe2 100644 --- a/api/ee/src/services/throttling_service.py +++ b/api/ee/src/middlewares/throttling.py @@ -8,7 +8,7 @@ from oss.src.utils.logging import get_module_logger from oss.src.utils.throttling import Algorithm, check_throttles -from ee.src.core.entitlements.types import ( +from ee.src.core.access.entitlements.types import ( ENDPOINTS, Category, Method, @@ -16,11 +16,9 @@ Throttle, Tracker, ) -from ee.src.core.entitlements.controls import get_plan_entitlements, get_plans +from ee.src.core.access.controls import get_plan_entitlements, get_plans from ee.src.core.subscriptions.settings import get_free_plan -from ee.src.core.meters.service import MetersService from ee.src.core.subscriptions.service import SubscriptionsService -from ee.src.dbs.postgres.meters.dao import MetersDAO from ee.src.dbs.postgres.subscriptions.dao import SubscriptionsDAO log = get_module_logger(__name__) @@ -29,13 +27,8 @@ _warned_no_throttles = False _warned_fallback_pairs: set[tuple[str | None, str | None]] = set() -meters_service = MetersService( - meters_dao=MetersDAO(), -) - subscriptions_service = SubscriptionsService( subscriptions_dao=SubscriptionsDAO(), - meters_service=meters_service, ) diff --git a/api/ee/src/models/api/workspace_models.py b/api/ee/src/models/api/workspace_models.py index f8f9ebacde..9dfe03c00f 100644 --- a/api/ee/src/models/api/workspace_models.py +++ b/api/ee/src/models/api/workspace_models.py @@ -4,13 +4,13 @@ from pydantic import BaseModel from ee.src.models.api.api_models import TimestampModel -from ee.src.models.shared_models import Permission +from ee.src.core.access.permissions.types import Permission class WorkspacePermission(BaseModel): # Role slugs are dynamic (env-overridable via AGENTA_ACCESS_ROLES); # validation against the effective scope catalog happens at the API - # boundary via `ee.src.core.entitlements.controls.get_role`. + # boundary via `ee.src.core.access.controls.get_role`. role_name: str role_description: Optional[str] = None permissions: Optional[List[Permission]] = None diff --git a/api/ee/src/routers/organization_router.py b/api/ee/src/routers/organization_router.py index 717163639a..488e9aeb4c 100644 --- a/api/ee/src/routers/organization_router.py +++ b/api/ee/src/routers/organization_router.py @@ -8,11 +8,11 @@ from oss.src.services import db_manager -from ee.src.utils.permissions import ( +from ee.src.core.access.permissions.service import ( check_user_org_access, check_rbac_permission, ) -from ee.src.utils.entitlements import ( +from ee.src.core.access.entitlements.service import ( check_entitlements, scope_from, Tracker, @@ -24,8 +24,8 @@ db_manager_ee, workspace_manager, ) -from ee.src.services.selectors import get_user_org_and_workspace_id -from ee.src.models.shared_models import Permission +from ee.src.services.db_manager_ee import get_user_org_and_workspace_id +from ee.src.core.access.permissions.types import Permission from ee.src.models.api.workspace_models import ( CreateWorkspace, diff --git a/api/ee/src/routers/workspace_router.py b/api/ee/src/routers/workspace_router.py index 5ed85d6cf7..e46578b58a 100644 --- a/api/ee/src/routers/workspace_router.py +++ b/api/ee/src/routers/workspace_router.py @@ -5,14 +5,14 @@ from oss.src.utils.logging import get_module_logger from oss.src.utils.common import APIRouter -from ee.src.utils.permissions import check_action_access +from ee.src.core.access.permissions.service import check_action_access from ee.src.services import workspace_manager, db_manager_ee from ee.src.models.api.workspace_models import ( UserRole, ) -from ee.src.models.shared_models import Permission -from ee.src.core.entitlements.controls import get_role +from ee.src.core.access.permissions.types import Permission +from ee.src.core.access.controls import get_role router = APIRouter() diff --git a/api/ee/src/services/admin_manager.py b/api/ee/src/services/admin_manager.py index 2aa827c03e..4c8fee002f 100644 --- a/api/ee/src/services/admin_manager.py +++ b/api/ee/src/services/admin_manager.py @@ -7,7 +7,7 @@ from oss.src.utils.logging import get_module_logger -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import get_transactions_engine from oss.src.models.db_models import UserDB from oss.src.services.api_key_service import create_api_key @@ -88,7 +88,7 @@ class ProjectRequest(BaseModel): # Role slugs are dynamic at runtime (env-overridable via AGENTA_ACCESS_ROLES); # validation against the effective per-scope catalog happens at the handler -# boundary via `ee.src.core.entitlements.controls.get_role`. +# boundary via `ee.src.core.access.controls.get_role`. class OrganizationMembershipRequest(BaseModel): role: str is_demo: bool @@ -119,7 +119,9 @@ class ProjectMembershipRequest(BaseModel): async def check_user( request: UserRequest, ) -> Optional[UserRequest]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(UserDB).filter_by( email=request.email, @@ -136,7 +138,9 @@ async def check_user( async def create_user( request: UserRequest, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: user_db = UserDB( # id=uuid7() # use default # @@ -162,7 +166,9 @@ async def create_user( async def create_organization( request: OrganizationRequest, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: organization_db = OrganizationDB( name=request.name, description=request.description, @@ -190,7 +196,9 @@ async def create_organization( async def create_workspace( request: WorkspaceRequest, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: workspace_db = WorkspaceDB( # id=uuid7() # use default # @@ -219,7 +227,9 @@ async def create_workspace( async def create_project( request: ProjectRequest, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: project_db = ProjectDB( # id=uuid7() # use default # @@ -250,7 +260,9 @@ async def create_project( async def create_organization_membership( request: OrganizationMembershipRequest, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: membership_db = OrganizationMembershipDB( # id=uuid7() # use default # @@ -293,7 +305,9 @@ async def create_organization_membership( async def create_workspace_membership( request: WorkspaceMembershipRequest, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: workspace = await session.execute( select(WorkspaceDB).filter_by( id=request.workspace_ref.id, @@ -334,7 +348,9 @@ async def create_workspace_membership( async def create_project_membership( request: ProjectMembershipRequest, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: project = await session.execute( select(ProjectDB).filter_by( id=request.project_ref.id, diff --git a/api/ee/src/services/commoners.py b/api/ee/src/services/commoners.py index 1c2d2c66bf..5a5920aa87 100644 --- a/api/ee/src/services/commoners.py +++ b/api/ee/src/services/commoners.py @@ -7,7 +7,7 @@ from pydantic import BaseModel from oss.src.utils.logging import get_module_logger -from oss.src.utils.caching import acquire_lock, release_lock +from oss.src.utils.locking import acquire_lock, release_lock from oss.src.utils.common import env from oss.src.services import db_manager @@ -24,26 +24,23 @@ delete_user, ) from oss.src.models.db_models import OrganizationDB -from ee.src.services.email_helper import ( - add_contact_to_loops, -) +from oss.src.utils import emailing from oss.src.core.auth.service import AuthService from ee.src.dbs.postgres.subscriptions.dao import SubscriptionsDAO from ee.src.core.subscriptions.service import SubscriptionsService from ee.src.core.subscriptions.types import get_default_plan -from ee.src.dbs.postgres.meters.dao import MetersDAO -from ee.src.core.meters.service import MetersService -from ee.src.utils.entitlements import check_entitlements, scope_from, Gauge +from ee.src.core.access.entitlements.service import ( + check_entitlements, + scope_from, + Gauge, +) from ee.src.core.organizations.exceptions import OrganizationCreationNotAllowedError log = get_module_logger(__name__) subscription_service = SubscriptionsService( subscriptions_dao=SubscriptionsDAO(), - meters_service=MetersService( - meters_dao=MetersDAO(), - ), ) DEMOS = "AGENTA_DEMOS" @@ -247,7 +244,7 @@ async def create_accounts( if is_ee(): try: # Adds contact to loops for marketing emails. TODO: Add opt-in checkbox to supertokens - add_contact_to_loops(user_dict["email"]) # type: ignore + emailing.add_contact(user_dict["email"]) # type: ignore except ConnectionError as ex: log.warn("error adding contact to loops %s", ex) diff --git a/api/ee/src/services/converters.py b/api/ee/src/services/converters.py deleted file mode 100644 index e0520b68e5..0000000000 --- a/api/ee/src/services/converters.py +++ /dev/null @@ -1,158 +0,0 @@ -from typing import List, Any -from datetime import datetime, timezone - -from oss.src.services import db_manager -from ee.src.services import db_manager_ee -from ee.src.core.workspaces.types import WorkspaceResponse -from ee.src.core.entitlements.controls import ( - get_role_description, - get_role_permissions, -) -from ee.src.models.shared_models import Permission -from oss.src.models.db_models import WorkspaceDB - - -def _role_slug(role: Any) -> str: - """Normalize an enum or string role to its slug form.""" - return role.value if hasattr(role, "value") else str(role) - - -def _expand_permissions(slugs: List[str]) -> List[str]: - """Expand the `"*"` wildcard to the full list of Permission enum values. - - Why: `WorkspacePermission.permissions` is typed as `List[Permission]` and - the owner role stores `["*"]` as a wildcard. Pydantic rejects `"*"` since - it's not an enum member, so we materialize it at the API boundary. - """ - if "*" not in slugs: - return slugs - return [p.value for p in Permission] - - -async def get_workspace_in_format( - workspace: WorkspaceDB, - include_members: bool = True, -) -> WorkspaceResponse: - """Converts the workspace object to the WorkspaceResponse model. - - Arguments: - workspace (WorkspaceDB): The workspace object - include_members (bool): Whether to include workspace members. Defaults to True. - - Returns: - WorkspaceResponse: The workspace object in the WorkspaceResponse model - """ - - members = [] - - if include_members: - project = await db_manager_ee.get_project_by_workspace( - workspace_id=str(workspace.id) - ) - project_members = await db_manager_ee.get_project_members( - project_id=str(project.id) - ) - invitations = await db_manager_ee.get_project_invitations( - project_id=str(project.id), invitation_used=False - ) - - if len(invitations) > 0: - for invitation in invitations: - if not invitation.used and str(invitation.project_id) == str( - project.id - ): - user = await db_manager.get_user_with_email(invitation.email) - member_dict = { - "user": { - "id": str(user.id) if user else invitation.email, - "email": user.email if user else invitation.email, - "username": ( - user.username - if user - else invitation.email.split("@")[0] - ), - "status": ( - "pending" - if invitation.expiration_date - > datetime.now(timezone.utc) - else "expired" - ), - "created_at": ( - str(user.created_at) - if user - else ( - str(invitation.created_at) - if str(invitation.created_at) - else None - ) - ), - }, - "roles": [ - { - "role_name": invitation.role, - "role_description": get_role_description( - "workspace", _role_slug(invitation.role) - ), - } - ], - } - members.append(member_dict) - - for project_member in project_members: - member_role = project_member.role - member_dict = { - "user": { - "id": str(project_member.user.id), - "email": project_member.user.email, - "username": project_member.user.username, - "status": "member", - "created_at": str(project_member.user.created_at), - }, - "roles": ( - [ - { - "role_name": member_role, - "role_description": get_role_description( - "project", _role_slug(member_role) - ), - "permissions": _expand_permissions( - get_role_permissions("project", _role_slug(member_role)) - ), - } - ] - if member_role - else [] - ), - } - members.append(member_dict) - - workspace_response = WorkspaceResponse( - id=str(workspace.id), - name=workspace.name, - description=workspace.description, - type=workspace.type, - members=members, - organization=str(workspace.organization_id), - created_at=str(workspace.created_at), - updated_at=str(workspace.updated_at), - ) - return workspace_response - - -async def get_all_workspace_permissions() -> List[Permission]: - """ - Retrieve all workspace permissions. - - Returns: - List[Permission]: A list of all workspace permissions in the DB. - """ - workspace_permissions = list(Permission) - return workspace_permissions - - -def get_all_workspace_permissions_by_role(role_name: str) -> List[str]: - """Retrieve all permissions assigned to a workspace role. - - Resolved via access-controls (env-overridable via AGENTA_ACCESS_ROLES). - """ - return _expand_permissions(get_role_permissions("workspace", role_name)) diff --git a/api/ee/src/services/db_manager.py b/api/ee/src/services/db_manager.py deleted file mode 100644 index a97b52af6d..0000000000 --- a/api/ee/src/services/db_manager.py +++ /dev/null @@ -1,35 +0,0 @@ -import uuid - -from oss.src.dbs.postgres.shared.engine import engine -from oss.src.models.db_models import DeploymentDB - - -async def create_deployment( - app_id: str, - project_id: str, - uri: str, -) -> DeploymentDB: - """Create a new deployment. - Args: - app_id (str): The app variant to create the deployment for. - project_id (str): The project variant to create the deployment for. - uri (str): The URI of the service. - Returns: - DeploymentDB: The created deployment. - """ - - async with engine.core_session() as session: - try: - deployment = DeploymentDB( - app_id=uuid.UUID(app_id), - project_id=uuid.UUID(project_id), - uri=uri, - ) - - session.add(deployment) - await session.commit() - await session.refresh(deployment) - - return deployment - except Exception as e: - raise Exception(f"Error while creating deployment: {e}") diff --git a/api/ee/src/services/db_manager_ee.py b/api/ee/src/services/db_manager_ee.py index 9b57ebf3c8..405e65e2a6 100644 --- a/api/ee/src/services/db_manager_ee.py +++ b/api/ee/src/services/db_manager_ee.py @@ -1,8 +1,7 @@ -from typing import List, Set, Union, NoReturn, Optional, Tuple +from typing import Any, Dict, List, Set, Union, NoReturn, Optional, Tuple import uuid from datetime import datetime, timezone -import sendgrid from fastapi import HTTPException from sqlalchemy import delete, func, update @@ -14,7 +13,9 @@ from oss.src.utils.logging import get_module_logger -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + get_transactions_engine, +) from oss.src.services import db_manager from ee.src.core.workspaces.types import ( UserRole, @@ -27,8 +28,12 @@ CreateOrganization, OrganizationUpdate, ) -from ee.src.models.shared_models import WorkspaceRole -from ee.src.core.entitlements.controls import get_roles +from ee.src.core.access.permissions.types import Permission, RequiredRole +from ee.src.core.access.controls import ( + get_roles, + get_role_description, + get_role_permissions, +) from oss.src.models.db_models import ( OrganizationDB, @@ -44,6 +49,7 @@ from oss.src.models.db_models import ( UserDB, InvitationDB, + DeploymentDB, ) from ee.src.core.organizations.exceptions import ( @@ -54,15 +60,10 @@ OrganizationProvidersDAO, OrganizationDomainsDAO, ) -from ee.src.services.converters import get_workspace_in_format -from ee.src.services.selectors import get_org_default_workspace from oss.src.utils.env import env -# Initialize sendgrid api client -sg = sendgrid.SendGridAPIClient(api_key=env.sendgrid.api_key) - log = get_module_logger(__name__) @@ -77,7 +78,9 @@ async def get_organization(organization_id: str) -> OrganizationDB: OrganizationDB: The fetched organization. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(OrganizationDB).filter_by(id=uuid.UUID(organization_id)) ) @@ -96,7 +99,9 @@ async def get_organizations_by_list_ids(organization_ids: List) -> List[Organiza List: A list of dictionaries representing the retrieved organizations. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: organization_uuids = [ uuid.UUID(organization_id) for organization_id in organization_ids ] @@ -116,7 +121,9 @@ async def count_organizations_by_owner(owner_id: str) -> int: Returns: int: The count of organizations owned by the user. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(func.count(OrganizationDB.id)).where( OrganizationDB.owner_id == uuid.UUID(owner_id) @@ -136,7 +143,9 @@ async def get_default_workspace_id(user_id: str) -> str: str: The default workspace ID. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(WorkspaceMemberDB) .filter_by(user_id=uuid.UUID(user_id)) @@ -157,7 +166,7 @@ async def get_default_workspace_id(user_id: str) -> str: ( membership for membership in memberships - if membership.role == WorkspaceRole.OWNER + if membership.role == RequiredRole.OWNER ), None, ) @@ -181,7 +190,9 @@ async def get_organization_workspaces(organization_id: str): organization_id (str): The ID of the organization """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(WorkspaceDB) .filter_by(organization_id=uuid.UUID(organization_id)) @@ -199,7 +210,9 @@ async def get_workspace_members(workspace_id: str) -> List[WorkspaceMemberDB]: Used by RBAC / admin helpers to derive roles and permissions. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(WorkspaceMemberDB).where( WorkspaceMemberDB.workspace_id == workspace_id @@ -221,7 +234,7 @@ async def get_workspace_administrators(workspace: WorkspaceDB) -> List[UserDB]: admin_user_ids = [ str(member.user_id) for member in members - if member.role in (WorkspaceRole.ADMIN, WorkspaceRole.OWNER) + if member.role in (RequiredRole.ADMIN, RequiredRole.OWNER) ] administrators: List[UserDB] = [] @@ -385,7 +398,8 @@ async def _sync(db_session: AsyncSession) -> None: await _sync(session) return - async with engine.core_session() as new_session: + engine = get_transactions_engine() + async with engine.session() as new_session: await _sync(new_session) @@ -402,7 +416,9 @@ async def get_default_workspace_id_from_organization( str: The default (first) workspace ID. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: workspace_query = await session.execute( select(WorkspaceDB) .where( @@ -433,7 +449,9 @@ async def get_project_by_workspace( """ assert workspace_id is not None, "Workspace ID is required to retrieve project" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: stmt = select(ProjectDB).where( ProjectDB.workspace_id == uuid.UUID(workspace_id), ) @@ -494,7 +512,9 @@ async def create_project_member( async def fetch_project_memberships_by_user_id( user_id: str, ) -> List[ProjectMemberDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(ProjectMemberDB) .filter_by(user_id=uuid.UUID(user_id)) @@ -622,7 +642,9 @@ async def create_workspace( user = await db_manager.get_user(user_uid) organization = await get_organization(organization_id) - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: user_result = await session.execute(select(UserDB).filter_by(uid=user_uid)) user = user_result.scalars().first() @@ -652,7 +674,9 @@ async def update_workspace( payload (UpdateWorkspace): The data to update the workspace with. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(WorkspaceDB).filter_by(id=workspace.id)) workspace = result.scalars().first() @@ -681,7 +705,9 @@ async def check_user_in_workspace_with_email(email: str, workspace_id: str) -> b Exception: If there is an error checking if the user belongs to the workspace. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(WorkspaceMemberDB) .join(UserDB, UserDB.id == WorkspaceMemberDB.user_id) @@ -721,7 +747,9 @@ async def update_user_roles( f"No projects found for the provided workspace_id {workspace_id}" ) - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: workspace_member_result = await session.execute( select(WorkspaceMemberDB).filter_by( workspace_id=uuid.UUID(workspace_id), user_id=user.id @@ -804,7 +832,9 @@ async def add_user_to_workspace_and_org( if project and str(project.workspace_id) != str(workspace.id): raise ValueError("Project does not belong to the provided workspace") - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: # create joined organization for user user_organization = OrganizationMemberDB( user_id=user.id, organization_id=organization.id @@ -910,7 +940,9 @@ async def remove_user_from_workspace( ) project_ids = [project.id for project in projects] - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: if not user: # User is an invited user who has not yet created an account and therefore does not have a user object pass else: @@ -1048,7 +1080,9 @@ async def create_organization( Exception: If there is an error creating the organization. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: create_org_data = payload.model_dump(exclude_unset=True) is_demo = create_org_data.pop("is_demo", False) @@ -1137,7 +1171,9 @@ async def update_organization( Exception: If there is an error updating the organization. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(OrganizationDB).filter_by(id=uuid.UUID(organization_id)) ) @@ -1299,7 +1335,9 @@ async def delete_organization(organization_id: str) -> bool: Raises: NoResultFound: If organization not found. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(OrganizationDB).filter_by(id=uuid.UUID(organization_id)) ) @@ -1324,7 +1362,9 @@ async def delete_invitation(invitation_id: str) -> bool: bool: True if the invitation was successfully deleted, False otherwise. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB).filter_by(id=uuid.UUID(invitation_id)) ) @@ -1386,7 +1426,9 @@ async def mark_invitation_as_used( HTTPException: If there is an error marking the invitation as used. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB).filter_by( project_id=uuid.UUID(project_id), token=invitation.token @@ -1477,7 +1519,9 @@ async def get_project_invitations(project_id: str, **kwargs): project_id (str): The ID of the project """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: stmt = select(InvitationDB).filter( InvitationDB.project_id == uuid.UUID(project_id) ) @@ -1497,7 +1541,9 @@ async def get_all_pending_invitations(email: str): email (str): The email address of the user. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB).filter( InvitationDB.email == email, @@ -1522,7 +1568,9 @@ async def get_project_invitation( InvitationDB: invitation object """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB).filter_by( project_id=uuid.UUID(project_id), token=token, email=email @@ -1539,7 +1587,9 @@ async def get_project_members(project_id: str): project_id (str): The ID of the project """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: members_query = await session.execute( select(ProjectMemberDB) .filter(ProjectMemberDB.project_id == uuid.UUID(project_id)) @@ -1567,7 +1617,9 @@ async def project_member_exists( True if the user belongs to the project, False otherwise. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: stmt = select( select(ProjectMemberDB.id) .filter( @@ -1598,7 +1650,9 @@ async def workspace_member_exists( True if the user belongs to the workspace, False otherwise. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: stmt = select( select(WorkspaceMemberDB.id) .filter( @@ -1645,7 +1699,9 @@ async def create_org_workspace_invitation( if not project: raise Exception(f"No project found with ID {project_id}") - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: invitation = InvitationDB( token=token, email=email, @@ -1686,7 +1742,9 @@ async def add_user_to_organization( role: str = "viewer", # is_demo: bool = False, ) -> None: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: organization_member = OrganizationMemberDB( user_id=user_id, organization_id=organization_id, @@ -1712,7 +1770,9 @@ async def add_user_to_workspace( role: str, # is_demo: bool = False, ) -> None: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: # fetch workspace by workspace_id (SQL) stmt = select(WorkspaceDB).filter_by(id=workspace_id) workspace = await session.execute(stmt) @@ -1753,7 +1813,9 @@ async def add_user_to_project( if not project: raise Exception(f"No project found with ID {project_id}") - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: project_member = ProjectMemberDB( user_id=user_id, project_id=project_id, @@ -1793,7 +1855,9 @@ async def transfer_organization_ownership( Raises: ValueError: If new owner is not a member of the organization """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: # Verify organization exists org_result = await session.execute( select(OrganizationDB).filter_by(id=uuid.UUID(organization_id)) @@ -1925,7 +1989,9 @@ async def transfer_organization_ownership( async def admin_delete_org_membership(membership_id: uuid.UUID) -> bool: """Delete an org membership by ID. Returns False if not found.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(OrganizationMemberDB).filter_by(id=membership_id) ) @@ -1939,7 +2005,9 @@ async def admin_delete_org_membership(membership_id: uuid.UUID) -> bool: async def admin_delete_workspace_membership(membership_id: uuid.UUID) -> bool: """Delete a workspace membership by ID. Returns False if not found.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(WorkspaceMemberDB).filter_by(id=membership_id) ) @@ -1953,7 +2021,9 @@ async def admin_delete_workspace_membership(membership_id: uuid.UUID) -> bool: async def admin_delete_project_membership(membership_id: uuid.UUID) -> bool: """Delete a project membership by ID. Returns False if not found.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(ProjectMemberDB).filter_by(id=membership_id) ) @@ -1970,7 +2040,9 @@ async def admin_get_member_org_ids( org_ids: List[uuid.UUID], ) -> Set[uuid.UUID]: """Return the subset of org_ids where the user has a membership row.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: rows = ( ( await session.execute( @@ -1997,7 +2069,9 @@ async def admin_swap_org_memberships( a membership row. For each qualifying org, target gets source's role and source gets target's prior role. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: source_rows = ( ( await session.execute( @@ -2059,7 +2133,9 @@ async def admin_swap_workspace_memberships( have a membership row. For each qualifying workspace, target gets source's role and source gets target's prior role. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: source_rows = ( ( await session.execute( @@ -2121,7 +2197,9 @@ async def admin_swap_project_memberships( have a membership row. For each qualifying project, target gets source's role and source gets target's prior role. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: source_rows = ( ( await session.execute( @@ -2177,7 +2255,9 @@ async def admin_delete_user_memberships(user_id: uuid.UUID) -> None: Called before hard-deleting a user so FK constraints are not violated. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: await session.execute( delete(OrganizationMemberDB).where(OrganizationMemberDB.user_id == user_id) ) @@ -2188,3 +2268,258 @@ async def admin_delete_user_memberships(user_id: uuid.UUID) -> None: delete(ProjectMemberDB).where(ProjectMemberDB.user_id == user_id) ) await session.commit() + + +# Merged from ee/src/services/selectors.py +async def get_user_org_and_workspace_id(user_uid) -> Dict[str, Union[str, List[str]]]: + """ + Retrieves the user ID and organization IDs associated with a given user UID. + + Args: + user_uid (str): The UID of the user. + + Returns: + dict: A dictionary containing the user UID, ID, list of workspace IDS and list of organization IDS associated with a user. + If the user is not found, returns None + + Example Usage: + result = await get_user_org_and_workspace_id("user123") + + Output: + { "id": "123", "uid": "user123", "organization_ids": [], "workspace_ids": []} + """ + + engine = get_transactions_engine() + + async with engine.session() as session: + user = await db_manager.get_user_with_id(user_id=user_uid) + if not user: + raise NoResultFound(f"User with uid {user_uid} not found") + + user_org_result = await session.execute( + select(OrganizationMemberDB) + .filter_by(user_id=user.id) + .options(load_only(OrganizationMemberDB.organization_id)) # type: ignore + ) + orgs = user_org_result.scalars().all() + organization_ids = [str(user_org.organization_id) for user_org in orgs] + + member_in_workspaces_result = await session.execute( + select(WorkspaceMemberDB) + .filter_by(user_id=user.id) + .options(load_only(WorkspaceMemberDB.workspace_id)) # type: ignore + ) + workspaces_ids = [ + str(user_workspace.workspace_id) + for user_workspace in member_in_workspaces_result.scalars().all() + ] + + return { + "id": str(user.id), + "uid": str(user.uid), + "workspace_ids": workspaces_ids, + "organization_ids": organization_ids, + } + + +async def user_exists(user_email: str) -> bool: + """Check if user exists in the database. + + Arguments: + user_email (str): The email address of the logged-in user + + Returns: + bool: confirming if the user exists or not. + """ + + user = await db_manager.get_user_with_email(email=user_email) + return False if not user else True + + +async def get_org_default_workspace(organization: Organization) -> WorkspaceDB: + """Get's the default workspace for an organization from the database. + + Arguments: + organization (Organization): The organization + + Returns: + WorkspaceDB: Instance of WorkspaceDB + """ + + engine = get_transactions_engine() + + async with engine.session() as session: + result = await session.execute( + select(WorkspaceDB).filter_by( + organization_id=organization.id, + type="default", + ) + ) + workspace = result.scalars().first() + if workspace is not None: + return workspace + + result = await session.execute( + select(WorkspaceDB).filter_by( + organization_id=organization.id, + ) + ) + return result.scalars().first() + + +# Merged from ee/src/services/db_manager.py +async def create_deployment( + app_id: str, + project_id: str, + uri: str, +) -> DeploymentDB: + """Create a new deployment. + Args: + app_id (str): The app variant to create the deployment for. + project_id (str): The project variant to create the deployment for. + uri (str): The URI of the service. + Returns: + DeploymentDB: The created deployment. + """ + + engine = get_transactions_engine() + + async with engine.session() as session: + try: + deployment = DeploymentDB( + app_id=uuid.UUID(app_id), + project_id=uuid.UUID(project_id), + uri=uri, + ) + + session.add(deployment) + await session.commit() + await session.refresh(deployment) + + return deployment + except Exception as e: + raise Exception(f"Error while creating deployment: {e}") + + +# Merged from ee/src/services/converters.py +def _role_slug(role: Any) -> str: + """Normalize an enum or string role to its slug form.""" + return role.value if hasattr(role, "value") else str(role) + + +def _expand_permissions(slugs: List[str]) -> List[str]: + """Expand the `"*"` wildcard to the full list of Permission enum values. + + Why: `WorkspacePermission.permissions` is typed as `List[Permission]` and + the owner role stores `["*"]` as a wildcard. Pydantic rejects `"*"` since + it's not an enum member, so we materialize it at the API boundary. + """ + if "*" not in slugs: + return slugs + return [p.value for p in Permission] + + +async def get_workspace_in_format( + workspace: WorkspaceDB, + include_members: bool = True, +) -> WorkspaceResponse: + """Converts the workspace object to the WorkspaceResponse model. + + Arguments: + workspace (WorkspaceDB): The workspace object + include_members (bool): Whether to include workspace members. Defaults to True. + + Returns: + WorkspaceResponse: The workspace object in the WorkspaceResponse model + """ + + members = [] + + if include_members: + project = await get_project_by_workspace(workspace_id=str(workspace.id)) + project_members = await get_project_members(project_id=str(project.id)) + invitations = await get_project_invitations( + project_id=str(project.id), invitation_used=False + ) + + if len(invitations) > 0: + for invitation in invitations: + if not invitation.used and str(invitation.project_id) == str( + project.id + ): + user = await db_manager.get_user_with_email(invitation.email) + member_dict = { + "user": { + "id": str(user.id) if user else invitation.email, + "email": user.email if user else invitation.email, + "username": ( + user.username + if user + else invitation.email.split("@")[0] + ), + "status": ( + "pending" + if invitation.expiration_date + > datetime.now(timezone.utc) + else "expired" + ), + "created_at": ( + str(user.created_at) + if user + else ( + str(invitation.created_at) + if str(invitation.created_at) + else None + ) + ), + }, + "roles": [ + { + "role_name": invitation.role, + "role_description": get_role_description( + "workspace", _role_slug(invitation.role) + ), + } + ], + } + members.append(member_dict) + + for project_member in project_members: + member_role = project_member.role + member_dict = { + "user": { + "id": str(project_member.user.id), + "email": project_member.user.email, + "username": project_member.user.username, + "status": "member", + "created_at": str(project_member.user.created_at), + }, + "roles": ( + [ + { + "role_name": member_role, + "role_description": get_role_description( + "project", _role_slug(member_role) + ), + "permissions": _expand_permissions( + get_role_permissions("project", _role_slug(member_role)) + ), + } + ] + if member_role + else [] + ), + } + members.append(member_dict) + + workspace_response = WorkspaceResponse( + id=str(workspace.id), + name=workspace.name, + description=workspace.description, + type=workspace.type, + members=members, + organization=str(workspace.organization_id), + created_at=str(workspace.created_at), + updated_at=str(workspace.updated_at), + ) + return workspace_response diff --git a/api/ee/src/services/email_helper.py b/api/ee/src/services/email_helper.py deleted file mode 100644 index 0c91e666cd..0000000000 --- a/api/ee/src/services/email_helper.py +++ /dev/null @@ -1,54 +0,0 @@ -import time - -import httpx - -from oss.src.utils.env import env -from oss.src.utils.logging import get_module_logger - -log = get_module_logger(__name__) - - -def add_contact_to_loops(email, max_retries=5, initial_delay=1): - """ - Add a contact to Loops audience with retry and exponential backoff. - - Args: - email (str): Email address of the contact to be added. - max_retries (int): Maximum number of retries in case of rate limiting. - initial_delay (int): Initial delay in seconds before retrying. - - Raises: - ConnectionError: If max retries reached and unable to connect to Loops API. - - Returns: - httpx.Response: Response object from the Loops API. - """ - - # Endpoint URL - url = "https://app.loops.so/api/v1/contacts/create" - - # Request headers - headers = {"Authorization": f"Bearer {env.loops.api_key}"} - - # Request payload/body - data = {"email": email} - - retries = 0 - delay = initial_delay - - while retries < max_retries: - # Making the POST request - response = httpx.post(url, json=data, headers=headers, timeout=20) - - # If response code is 429, it indicates rate limiting - if response.status_code == 429: - log.warning(f"[LOOPS] Rate limit hit. Retrying in {delay} seconds...") - time.sleep(delay) - retries += 1 - delay *= 2 # Double the delay for exponential backoff - else: - # If response is not 429, return it - return response - - # If max retries reached, raise an exception or handle as needed - raise ConnectionError("Max retries reached. Unable to connect to Loops API.") diff --git a/api/ee/src/services/organization_service.py b/api/ee/src/services/organization_service.py index 5378008ae3..4f05f1bee0 100644 --- a/api/ee/src/services/organization_service.py +++ b/api/ee/src/services/organization_service.py @@ -12,7 +12,7 @@ from oss.src.utils.env import env from oss.src.utils.logging import get_module_logger -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import get_transactions_engine from oss.src.core.secrets.dtos import ( CreateSecretDTO, UpdateSecretDTO, @@ -38,7 +38,7 @@ ) from ee.src.services import db_manager_ee -from oss.src.services import email_service +from oss.src.utils import emailing from oss.src.models.db_models import UserDB from oss.src.models.db_models import ( WorkspaceDB, @@ -87,8 +87,6 @@ async def send_invitation_email( bool: True if the email was sent successfully, False otherwise. """ - html_template = email_service.read_email_template("./templates/send_email.html") - token_param = quote(token, safe="") email_param = quote(email, safe="") org_param = quote(str(organization.id), safe="") @@ -108,22 +106,18 @@ async def send_invitation_email( if not env.sendgrid.enabled: return invite_link - html_content = html_template.format( - username_placeholder=user.username, - action_placeholder="invited you to join", - workspace_placeholder=organization.name, + await emailing.send_email( + from_email="account@hello.agenta.ai", + to_email=email, + subject=f"{user.username} invited you to join {organization.name}", + username=user.username, + action="invited you to join", + workspace=organization.name, call_to_action=( "Click the link below to accept the invitation:


" f'Accept Invitation' ), ) - - await email_service.send_email( - from_email="account@hello.agenta.ai", - to_email=email, - subject=f"{user.username} invited you to join {organization.name}", - html_content=html_content, - ) return True @@ -141,24 +135,21 @@ async def notify_org_admin_invitation(workspace: WorkspaceDB, user: UserDB) -> b organization = await db_manager_ee.get_organization(str(workspace.organization_id)) project = await db_manager_ee.get_project_by_workspace(str(workspace.id)) - html_template = email_service.read_email_template("./templates/send_email.html") - html_content = html_template.format( - username_placeholder=user.username, - action_placeholder="joined your Workspace", - workspace_placeholder=f'"{organization.name}"', - call_to_action=( - "Click the link below to view your Organization:


" - f'View Organization' - ), - ) workspace_admins = await db_manager_ee.get_workspace_administrators(workspace) + for workspace_admin in workspace_admins: - await email_service.send_email( + await emailing.send_email( from_email="account@hello.agenta.ai", to_email=workspace_admin.email, subject=f"New Member Joined {organization.name}", - html_content=html_content, + username=user.username, + action="joined your Workspace", + workspace=f'"{organization.name}"', + call_to_action=( + "Click the link below to view your Organization:


" + f'View Organization' + ), ) return True @@ -280,7 +271,9 @@ async def create_domain( Token expires after 48 hours and can be refreshed. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationDomainsDAO(session) # Block if a verified domain already exists anywhere @@ -337,7 +330,9 @@ async def verify_domain( self, organization_id: str, domain_id: str, user_id: str ) -> OrganizationDomain: """Verify a domain via DNS check.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationDomainsDAO(session) domain = await dao.get_by_id( @@ -403,7 +398,9 @@ async def list_domains(self, organization_id: str) -> List[OrganizationDomain]: Tokens are returned for unverified domains (within expiry period). Verified domains have token=None (cleared after verification). """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationDomainsDAO(session) domains = await dao.list_by_organization(organization_id=organization_id) @@ -430,7 +427,9 @@ async def refresh_token( Generates a new token and resets the 48-hour expiry window. For verified domains, this marks them as unverified for re-verification. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationDomainsDAO(session) domain = await dao.get_by_id( @@ -470,7 +469,9 @@ async def reset_domain( Generates a new token and marks the domain as unverified. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationDomainsDAO(session) domain = await dao.get_by_id( @@ -506,7 +507,9 @@ async def delete_domain( self, organization_id: str, domain_id: str, user_id: str ) -> bool: """Delete a domain.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationDomainsDAO(session) domain = await dao.get_by_id( @@ -573,7 +576,9 @@ async def create_provider( user_id: str, ) -> OrganizationProvider: """Create a new SSO provider.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationProvidersDAO(session) # Use the slug from payload (already validated to be lowercase letters and hyphens) @@ -648,7 +653,9 @@ async def update_provider( user_id: str, ) -> OrganizationProvider: """Update an SSO provider.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationProvidersDAO(session) provider = await dao.get_by_id( @@ -735,7 +742,9 @@ async def update_provider( async def list_providers(self, organization_id: str) -> List[OrganizationProvider]: """List all SSO providers for an organization.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationProvidersDAO(session) providers = await dao.list_by_organization(organization_id=organization_id) @@ -748,7 +757,9 @@ async def get_provider( self, organization_id: str, provider_id: str ) -> OrganizationProvider: """Get a single SSO provider by ID.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationProvidersDAO(session) provider = await dao.get_by_id( provider_id=provider_id, organization_id=organization_id @@ -761,7 +772,9 @@ async def test_provider( self, organization_id: str, provider_id: str, user_id: str ) -> OrganizationProvider: """Test SSO provider connection and mark as valid if successful.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationProvidersDAO(session) provider = await dao.get_by_id( @@ -806,7 +819,9 @@ async def delete_provider( self, organization_id: str, provider_id: str, user_id: str ) -> bool: """Delete an SSO provider.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: dao = OrganizationProvidersDAO(session) provider = await dao.get_by_id( diff --git a/api/ee/src/services/selectors.py b/api/ee/src/services/selectors.py deleted file mode 100644 index 017229d7bb..0000000000 --- a/api/ee/src/services/selectors.py +++ /dev/null @@ -1,113 +0,0 @@ -from typing import Dict, List, Union - -from sqlalchemy.future import select -from sqlalchemy.exc import NoResultFound -from sqlalchemy.orm import load_only - -from oss.src.services import db_manager -from oss.src.utils.logging import get_module_logger - -from oss.src.dbs.postgres.shared.engine import engine -from ee.src.core.organizations.types import Organization - -from oss.src.models.db_models import ( - WorkspaceDB, -) -from ee.src.models.db_models import ( - OrganizationMemberDB, - WorkspaceMemberDB, -) - -log = get_module_logger(__name__) - - -async def get_user_org_and_workspace_id(user_uid) -> Dict[str, Union[str, List[str]]]: - """ - Retrieves the user ID and organization IDs associated with a given user UID. - - Args: - user_uid (str): The UID of the user. - - Returns: - dict: A dictionary containing the user UID, ID, list of workspace IDS and list of organization IDS associated with a user. - If the user is not found, returns None - - Example Usage: - result = await get_user_org_and_workspace_id("user123") - - Output: - { "id": "123", "uid": "user123", "organization_ids": [], "workspace_ids": []} - """ - - async with engine.core_session() as session: - user = await db_manager.get_user_with_id(user_id=user_uid) - if not user: - raise NoResultFound(f"User with uid {user_uid} not found") - - user_org_result = await session.execute( - select(OrganizationMemberDB) - .filter_by(user_id=user.id) - .options(load_only(OrganizationMemberDB.organization_id)) # type: ignore - ) - orgs = user_org_result.scalars().all() - organization_ids = [str(user_org.organization_id) for user_org in orgs] - - member_in_workspaces_result = await session.execute( - select(WorkspaceMemberDB) - .filter_by(user_id=user.id) - .options(load_only(WorkspaceMemberDB.workspace_id)) # type: ignore - ) - workspaces_ids = [ - str(user_workspace.workspace_id) - for user_workspace in member_in_workspaces_result.scalars().all() - ] - - return { - "id": str(user.id), - "uid": str(user.uid), - "workspace_ids": workspaces_ids, - "organization_ids": organization_ids, - } - - -async def user_exists(user_email: str) -> bool: - """Check if user exists in the database. - - Arguments: - user_email (str): The email address of the logged-in user - - Returns: - bool: confirming if the user exists or not. - """ - - user = await db_manager.get_user_with_email(email=user_email) - return False if not user else True - - -async def get_org_default_workspace(organization: Organization) -> WorkspaceDB: - """Get's the default workspace for an organization from the database. - - Arguments: - organization (Organization): The organization - - Returns: - WorkspaceDB: Instance of WorkspaceDB - """ - - async with engine.core_session() as session: - result = await session.execute( - select(WorkspaceDB).filter_by( - organization_id=organization.id, - type="default", - ) - ) - workspace = result.scalars().first() - if workspace is not None: - return workspace - - result = await session.execute( - select(WorkspaceDB).filter_by( - organization_id=organization.id, - ) - ) - return result.scalars().first() diff --git a/api/ee/src/services/templates/send_email.html b/api/ee/src/services/templates/send_email.html deleted file mode 100644 index 7d124ffd8a..0000000000 --- a/api/ee/src/services/templates/send_email.html +++ /dev/null @@ -1,7 +0,0 @@ -

Hello,

-

- {username_placeholder} has {action_placeholder} {workspace_placeholder} on - Agenta. -

-

{call_to_action}

-

Thank you for using Agenta!

diff --git a/api/ee/src/services/workspace_manager.py b/api/ee/src/services/workspace_manager.py index c3b9c11896..6e50ff2351 100644 --- a/api/ee/src/services/workspace_manager.py +++ b/api/ee/src/services/workspace_manager.py @@ -5,7 +5,7 @@ from oss.src.utils.logging import get_module_logger from oss.src.utils.caching import invalidate_cache from oss.src.services import db_manager -from ee.src.services import db_manager_ee, converters +from ee.src.services import db_manager_ee from oss.src.models.db_models import ( OrganizationDB, WorkspaceDB, @@ -20,8 +20,8 @@ CreateWorkspace, UpdateWorkspace, ) -from ee.src.core.entitlements.controls import get_role -from ee.src.models.shared_models import Permission, WorkspaceRole +from ee.src.core.access.controls import get_role +from ee.src.core.access.permissions.types import Permission, RequiredRole from oss.src.services.organization_service import ( create_invitation, check_existing_invitation, @@ -111,11 +111,10 @@ async def get_all_workspace_permissions() -> List[Permission]: Retrieve all workspace permissions. Returns: - List[Permission]: A list of all workspace permissions in the DB. + List[Permission]: A list of all workspace permissions. """ - workspace_permissions_from_db = await converters.get_all_workspace_permissions() - return workspace_permissions_from_db + return list(Permission) async def invite_user_to_workspace( @@ -214,7 +213,7 @@ async def invite_user_to_workspace( else ( str(payload_invite.roles[0]) if payload_invite.roles - else WorkspaceRole.VIEWER.value + else RequiredRole.VIEWER.value ) ) # Validate against the effective workspace catalog diff --git a/api/ee/tests/manual/evaluations/sdk/test_sdk_evaluation.py b/api/ee/tests/manual/evaluations/sdk/test_loop.py similarity index 100% rename from api/ee/tests/manual/evaluations/sdk/test_sdk_evaluation.py rename to api/ee/tests/manual/evaluations/sdk/test_loop.py diff --git a/api/ee/tests/manual/test_billing_period.py b/api/ee/tests/manual/test_billing_period.py index c4a66b4b22..1242c4e491 100644 --- a/api/ee/tests/manual/test_billing_period.py +++ b/api/ee/tests/manual/test_billing_period.py @@ -11,7 +11,7 @@ import pytest from datetime import datetime, timezone -from ee.src.utils.entitlements import monthly_period_from +from ee.src.core.access.entitlements.service import monthly_period_from # ---- Helpers ---- diff --git a/api/ee/tests/pytest/unit/services/test_db_manager_ee.py b/api/ee/tests/pytest/unit/services/test_db_manager_ee.py index 89bbea5e31..48f64261a5 100644 --- a/api/ee/tests/pytest/unit/services/test_db_manager_ee.py +++ b/api/ee/tests/pytest/unit/services/test_db_manager_ee.py @@ -5,7 +5,7 @@ import pytest from sqlalchemy.exc import NoResultFound -from ee.src.models.shared_models import WorkspaceRole +from ee.src.core.access.permissions.types import DefaultRole from ee.src.services import db_manager_ee @@ -45,10 +45,14 @@ async def __aexit__(self, exc_type, exc, tb): def _patch_core_session(monkeypatch, memberships): + # db_manager_ee calls get_transactions_engine() — patch where it's called + mock_engine = type( + "MockEngine", (), {"session": lambda self: _SessionContext(memberships)} + )() monkeypatch.setattr( - db_manager_ee.engine, - "core_session", - lambda: _SessionContext(memberships), + db_manager_ee, + "get_transactions_engine", + lambda: mock_engine, ) @@ -89,12 +93,12 @@ async def test_get_default_workspace_id_prefers_owner_membership(monkeypatch): [ SimpleNamespace( workspace_id=editor_workspace_id, - role=WorkspaceRole.EDITOR, + role=DefaultRole.EDITOR, created_at=datetime(2026, 4, 9, tzinfo=timezone.utc), ), SimpleNamespace( workspace_id=owner_workspace_id, - role=WorkspaceRole.OWNER, + role=DefaultRole.OWNER, created_at=datetime(2026, 4, 10, tzinfo=timezone.utc), ), ], @@ -115,12 +119,12 @@ async def test_get_default_workspace_id_falls_back_to_oldest_membership(monkeypa [ SimpleNamespace( workspace_id=newer_workspace_id, - role=WorkspaceRole.EDITOR, + role=DefaultRole.EDITOR, created_at=datetime(2026, 4, 10, tzinfo=timezone.utc), ), SimpleNamespace( workspace_id=oldest_workspace_id, - role=WorkspaceRole.VIEWER, + role=DefaultRole.VIEWER, created_at=datetime(2026, 4, 9, tzinfo=timezone.utc), ), ], @@ -180,10 +184,15 @@ async def fail_if_nested_invitation_delete_is_used(invitation_id): "delete_invitation", fail_if_nested_invitation_delete_is_used, ) + mock_engine = type( + "MockEngine", + (), + {"session": lambda self: _PendingInviteSessionContext(session)}, + )() monkeypatch.setattr( - db_manager_ee.engine, - "core_session", - lambda: _PendingInviteSessionContext(session), + db_manager_ee, + "get_transactions_engine", + lambda: mock_engine, ) result = await db_manager_ee.remove_user_from_workspace( diff --git a/api/ee/tests/pytest/unit/test_access_controls.py b/api/ee/tests/pytest/unit/test_access_controls.py index bef7461366..7a66835c74 100644 --- a/api/ee/tests/pytest/unit/test_access_controls.py +++ b/api/ee/tests/pytest/unit/test_access_controls.py @@ -1,5 +1,5 @@ """Unit tests for the access-controls parsers in -``ee.src.core.entitlements.controls``. +``ee.src.core.access.controls``. These exercise the pure parser functions (`_parse_plans_override`, `_parse_roles_override`) so we don't have to manipulate process env vars at @@ -11,9 +11,11 @@ import pytest -from ee.src.core.entitlements import controls -from ee.src.core.entitlements.types import DefaultPlan, DefaultRole, Tracker -from ee.src.models.shared_models import Permission, WorkspaceRole +from ee.src.core.access import controls +from ee.src.core.access.entitlements import controls as entitlement_controls +from ee.src.core.access.permissions import controls as permission_controls +from ee.src.core.access.entitlements.types import DefaultPlan, Tracker +from ee.src.core.access.permissions.types import Permission, DefaultRole, RequiredRole # --------------------------------------------------------------------------- @@ -36,26 +38,28 @@ def test_get_plan_entitlements_returns_none_for_empty_slug(self): def test_get_roles_returns_workspace_role_set(self): ws = controls.get_roles("workspace") slugs = {r["role"] for r in ws} - # Workspace exposes the code-default WorkspaceRole enum set on top of the + # Workspace exposes the code-default DefaultRole enum set on top of the # owner/viewer minima. - assert slugs == {r.value for r in WorkspaceRole} + assert slugs == {r.value for r in DefaultRole} def test_get_roles_returns_empty_for_unknown_scope(self): assert controls.get_roles("garbage") == [] def test_minima_present_in_every_scope(self): - # Every scope must always expose `owner` and `viewer`. + # Every scope must always expose `owner`, `admin`, and `viewer`. for scope in ("organization", "workspace", "project"): slugs = {r["role"] for r in controls.get_roles(scope)} - assert DefaultRole.OWNER.value in slugs - assert DefaultRole.VIEWER.value in slugs + assert RequiredRole.OWNER.value in slugs + assert RequiredRole.ADMIN.value in slugs + assert RequiredRole.VIEWER.value in slugs def test_organization_defaults_to_minima_only(self): # Organization scope has no permission concept today; it stays at the - # minima while workspace and project expose the code-default WorkspaceRole set. + # minima while workspace and project expose the code-default DefaultRole set. assert {r["role"] for r in controls.get_roles("organization")} == { - DefaultRole.OWNER.value, - DefaultRole.VIEWER.value, + RequiredRole.OWNER.value, + RequiredRole.ADMIN.value, + RequiredRole.VIEWER.value, } def test_project_default_mirrors_workspace_role_set(self): @@ -63,7 +67,7 @@ def test_project_default_mirrors_workspace_role_set(self): # (admin/developer/editor/annotator), so the project scope must # surface the same permission map for non-overridden deployments. assert {r["role"] for r in controls.get_roles("project")} == { - r.value for r in WorkspaceRole + r.value for r in DefaultRole } def test_owner_role_is_wildcard(self): @@ -73,7 +77,7 @@ def test_owner_role_is_wildcard(self): def test_viewer_in_workspace_and_project_is_read_only(self): # Viewer permissions in workspace/project come from the code-default - # `WorkspaceRole.VIEWER` set — every entry is a real Permission. + # `DefaultRole.VIEWER` set — every entry is a real Permission. valid = {p.value for p in Permission} for scope in ("workspace", "project"): perms = controls.get_role_permissions(scope, "viewer") @@ -99,7 +103,7 @@ def test_controls_hash_is_stable(self): class TestParsePlansOverride: def test_minimal_valid_override_with_flags(self): - plans, descriptions = controls._parse_plans_override( + plans, descriptions = entitlement_controls._parse_plans_override( { "plan_a": { "description": "Test plan", @@ -117,7 +121,7 @@ def test_minimal_valid_override_with_flags(self): assert descriptions["plan_a"] == "Test plan" def test_counters_and_gauges_validated(self): - plans, _ = controls._parse_plans_override( + plans, _ = entitlement_controls._parse_plans_override( { "p": { "counters": { @@ -132,21 +136,21 @@ def test_counters_and_gauges_validated(self): def test_empty_dict_rejected(self): with pytest.raises(ValueError, match="non-empty"): - controls._parse_plans_override({}) + entitlement_controls._parse_plans_override({}) def test_non_dict_rejected(self): with pytest.raises(ValueError, match="non-empty JSON object"): - controls._parse_plans_override([]) + entitlement_controls._parse_plans_override([]) def test_plan_with_no_entitlements_allowed(self): # Display-only plans (e.g. custom/self-hosted) may carry no # entitlement trackers. The runtime returns an empty entitlement # map for those plans rather than treating them as unknown. - plans, _ = controls._parse_plans_override({"empty_plan": {}}) + plans, _ = entitlement_controls._parse_plans_override({"empty_plan": {}}) assert plans == {"empty_plan": {}} def test_plan_with_only_description_allowed(self): - plans, descriptions = controls._parse_plans_override( + plans, descriptions = entitlement_controls._parse_plans_override( {"p": {"description": "display only"}} ) assert plans == {"p": {}} @@ -154,19 +158,25 @@ def test_plan_with_only_description_allowed(self): def test_unknown_flag_key_rejected(self): with pytest.raises(ValueError, match="Unknown flag"): - controls._parse_plans_override({"p": {"flags": {"bogus": True}}}) + entitlement_controls._parse_plans_override( + {"p": {"flags": {"bogus": True}}} + ) def test_unknown_counter_key_rejected(self): with pytest.raises(ValueError, match="Unknown counter"): - controls._parse_plans_override({"p": {"counters": {"bogus": {"limit": 1}}}}) + entitlement_controls._parse_plans_override( + {"p": {"counters": {"bogus": {"limit": 1}}}} + ) def test_unknown_gauge_key_rejected(self): with pytest.raises(ValueError, match="Unknown gauge"): - controls._parse_plans_override({"p": {"gauges": {"bogus": {"limit": 1}}}}) + entitlement_controls._parse_plans_override( + {"p": {"gauges": {"bogus": {"limit": 1}}}} + ) def test_extra_field_in_plan_rejected(self): with pytest.raises(ValueError, match="Invalid plan override"): - controls._parse_plans_override({"p": {"surprise": "yes"}}) + entitlement_controls._parse_plans_override({"p": {"surprise": "yes"}}) # --------------------------------------------------------------------------- @@ -189,47 +199,49 @@ def test_project_override_is_mirrored_to_workspace_today(self): # workspace role catalog is used by the Invite Members flow for project # membership. A project-only override therefore intentionally replaces # workspace extras too, instead of leaving workspace defaults intact. - result = controls._parse_roles_override( + result = permission_controls._parse_roles_override( {"project": [_custom_role("reviewer", ["read_system"])]} ) - proj_slugs = [r["role"] for r in result["project"]] - ws_slugs = [r["role"] for r in result["workspace"]] + prj_sclugs = [r["role"] for r in result["project"]] + wrk_sclugs = [r["role"] for r in result["workspace"]] org_slugs = [r["role"] for r in result["organization"]] # Project: minima + override (env overrides REPLACE default extras in # the overridden scope). - assert proj_slugs == ["owner", "viewer", "reviewer"] - assert ws_slugs == ["owner", "viewer", "reviewer"] + assert prj_sclugs == ["owner", "admin", "viewer", "reviewer"] + assert wrk_sclugs == ["owner", "admin", "viewer", "reviewer"] # Organization: untouched (minima-only by default). - assert org_slugs == ["owner", "viewer"] + assert org_slugs == ["owner", "admin", "viewer"] def test_empty_dict_rejected(self): with pytest.raises(ValueError, match="non-empty"): - controls._parse_roles_override({}) + permission_controls._parse_roles_override({}) def test_unknown_scope_rejected(self): with pytest.raises(ValueError, match="Unknown role scope"): - controls._parse_roles_override( + permission_controls._parse_roles_override( {"galaxy": [_custom_role("ranger", ["read_system"])]} ) def test_empty_scope_list_rejected(self): with pytest.raises(ValueError, match="non-empty list of roles"): - controls._parse_roles_override({"project": []}) + permission_controls._parse_roles_override({"project": []}) def test_owner_reserved_cannot_be_redefined(self): with pytest.raises(ValueError, match="cannot redefine reserved role 'owner'"): - controls._parse_roles_override({"project": [_custom_role("owner", ["*"])]}) + permission_controls._parse_roles_override( + {"project": [_custom_role("owner", ["*"])]} + ) def test_viewer_reserved_cannot_be_redefined(self): with pytest.raises(ValueError, match="cannot redefine reserved role 'viewer'"): - controls._parse_roles_override( + permission_controls._parse_roles_override( {"project": [_custom_role("viewer", ["read_system"])]} ) def test_duplicate_custom_role_slug_rejected(self): with pytest.raises(ValueError, match="Duplicate role slug"): - controls._parse_roles_override( + permission_controls._parse_roles_override( { "project": [ _custom_role("reviewer", ["read_system"]), @@ -240,19 +252,19 @@ def test_duplicate_custom_role_slug_rejected(self): def test_empty_role_slug_rejected(self): with pytest.raises(ValueError, match="Empty role slug|Invalid role override"): - controls._parse_roles_override( + permission_controls._parse_roles_override( {"project": [{"role": "", "permissions": []}]} ) def test_unknown_permission_rejected(self): with pytest.raises(ValueError, match="Unknown permission"): - controls._parse_roles_override( + permission_controls._parse_roles_override( {"project": [_custom_role("custom", ["totally_made_up_perm"])]} ) def test_known_permission_accepted(self): valid_perm = next(iter(Permission)).value - result = controls._parse_roles_override( + result = permission_controls._parse_roles_override( {"project": [_custom_role("custom", [valid_perm])]} ) # Last entry is the custom role; first two are the minima. @@ -260,13 +272,12 @@ def test_known_permission_accepted(self): assert result["project"][-1]["permissions"] == [valid_perm] def test_minima_always_present_after_override(self): - result = controls._parse_roles_override( + result = permission_controls._parse_roles_override( {"organization": [_custom_role("auditor", ["read_system"])]} ) slugs = [r["role"] for r in result["organization"]] - # Minima are always re-applied at the front of each scope. - assert slugs[0] == "owner" - assert slugs[1] == "viewer" + # Minima are always re-applied at the front of each scope, in order. + assert slugs[:3] == ["owner", "admin", "viewer"] assert "auditor" in slugs @@ -275,7 +286,7 @@ def test_minima_always_present_after_override(self): # --------------------------------------------------------------------------- -from ee.src.core.entitlements.types import ( # noqa: E402 +from ee.src.core.access.entitlements.types import ( # noqa: E402 Category, Counter, Flag, @@ -290,23 +301,25 @@ def test_minima_always_present_after_override(self): class TestDefaultPlanOverlayParse: def test_empty_payload_rejected(self): with pytest.raises(ValueError, match="non-empty"): - controls._parse_default_plan_overlay({}) + entitlement_controls._parse_default_plan_overlay({}) def test_non_dict_rejected(self): with pytest.raises(ValueError, match="non-empty JSON object"): - controls._parse_default_plan_overlay([]) + entitlement_controls._parse_default_plan_overlay([]) def test_unknown_flag_rejected(self): with pytest.raises(ValueError, match="Unknown flag"): - controls._parse_default_plan_overlay({"flags": {"bogus": True}}) + entitlement_controls._parse_default_plan_overlay({"flags": {"bogus": True}}) def test_unknown_counter_rejected(self): with pytest.raises(ValueError, match="Unknown counter"): - controls._parse_default_plan_overlay({"counters": {"bogus": {"limit": 1}}}) + entitlement_controls._parse_default_plan_overlay( + {"counters": {"bogus": {"limit": 1}}} + ) def test_unknown_throttle_category_rejected(self): with pytest.raises(ValueError, match="not a valid throttle category"): - controls._parse_default_plan_overlay( + entitlement_controls._parse_default_plan_overlay( {"throttles": {"galaxy": {"bucket": {"rate": 1}}}} ) @@ -314,7 +327,7 @@ def test_extra_field_rejected(self): with pytest.raises( ValueError, match="Invalid AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY" ): - controls._parse_default_plan_overlay({"surprise": "x"}) + entitlement_controls._parse_default_plan_overlay({"surprise": "x"}) class TestDefaultPlanOverlayApply: @@ -344,10 +357,10 @@ def _base_plan(self) -> dict: def test_quota_field_merge_preserves_other_fields(self): plans = {"the_plan": self._base_plan()} descriptions: dict = {} - overlay = controls._parse_default_plan_overlay( + overlay = entitlement_controls._parse_default_plan_overlay( {"counters": {"traces_ingested": {"retention": 525600}}} ) - plans, _ = controls._apply_default_plan_overlay( + plans, _ = entitlement_controls._apply_default_plan_overlay( plans, descriptions, overlay, "the_plan" ) traces: Quota = plans["the_plan"][Tracker.COUNTERS][Counter.TRACES_INGESTED] @@ -358,10 +371,12 @@ def test_quota_field_merge_preserves_other_fields(self): def test_throttle_category_patch_preserves_other_throttles(self): plans = {"the_plan": self._base_plan()} - overlay = controls._parse_default_plan_overlay( + overlay = entitlement_controls._parse_default_plan_overlay( {"throttles": {"standard": {"bucket": {"rate": 7200}}}} ) - plans, _ = controls._apply_default_plan_overlay(plans, {}, overlay, "the_plan") + plans, _ = entitlement_controls._apply_default_plan_overlay( + plans, {}, overlay, "the_plan" + ) throttles = plans["the_plan"][Tracker.THROTTLES] # Standard throttle: rate patched, capacity preserved. standard = next(t for t in throttles if t.categories == [Category.STANDARD]) @@ -373,26 +388,32 @@ def test_throttle_category_patch_preserves_other_throttles(self): def test_overlay_targeting_unknown_plan_fails(self): plans = {"the_plan": self._base_plan()} - overlay = controls._parse_default_plan_overlay({"flags": {"access": True}}) + overlay = entitlement_controls._parse_default_plan_overlay( + {"flags": {"access": True}} + ) with pytest.raises(ValueError, match="not in the effective plan set"): - controls._apply_default_plan_overlay(plans, {}, overlay, "ghost_plan") + entitlement_controls._apply_default_plan_overlay( + plans, {}, overlay, "ghost_plan" + ) def test_overlay_targeting_throttle_with_no_match_fails(self): plans = {"the_plan": self._base_plan()} - overlay = controls._parse_default_plan_overlay( + overlay = entitlement_controls._parse_default_plan_overlay( {"throttles": {"ai_services": {"bucket": {"rate": 99}}}} ) with pytest.raises( ValueError, match="no single-category throttle entry for 'ai_services'" ): - controls._apply_default_plan_overlay(plans, {}, overlay, "the_plan") + entitlement_controls._apply_default_plan_overlay( + plans, {}, overlay, "the_plan" + ) def test_description_replaces(self): plans = {"the_plan": self._base_plan()} - overlay = controls._parse_default_plan_overlay( + overlay = entitlement_controls._parse_default_plan_overlay( {"description": "Self-hosted override"} ) - _, descriptions = controls._apply_default_plan_overlay( + _, descriptions = entitlement_controls._apply_default_plan_overlay( plans, {}, overlay, "the_plan" ) assert descriptions["the_plan"] == "Self-hosted override" @@ -401,8 +422,12 @@ def test_flag_patch_only_overwrites_named_keys(self): base = self._base_plan() base[Tracker.FLAGS] = {Flag.ACCESS: False, Flag.RBAC: True} plans = {"the_plan": base} - overlay = controls._parse_default_plan_overlay({"flags": {"access": True}}) - plans, _ = controls._apply_default_plan_overlay(plans, {}, overlay, "the_plan") + overlay = entitlement_controls._parse_default_plan_overlay( + {"flags": {"access": True}} + ) + plans, _ = entitlement_controls._apply_default_plan_overlay( + plans, {}, overlay, "the_plan" + ) flags = plans["the_plan"][Tracker.FLAGS] assert flags[Flag.ACCESS] is True assert flags[Flag.RBAC] is True # untouched @@ -416,21 +441,21 @@ def test_flag_patch_only_overwrites_named_keys(self): class TestRolesOverlayParse: def test_empty_payload_rejected(self): with pytest.raises(ValueError, match="non-empty"): - controls._parse_roles_overlay({}) + permission_controls._parse_roles_overlay({}) def test_non_dict_rejected(self): with pytest.raises(ValueError, match="non-empty JSON object"): - controls._parse_roles_overlay([]) + permission_controls._parse_roles_overlay([]) def test_non_project_scope_rejected(self): with pytest.raises(ValueError, match="only supports the 'project' scope"): - controls._parse_roles_overlay( + permission_controls._parse_roles_overlay( {"workspace": {"editor": {"permissions": ["read_system"]}}} ) def test_multiple_scopes_rejected_lists_offenders(self): with pytest.raises(ValueError, match="organization"): - controls._parse_roles_overlay( + permission_controls._parse_roles_overlay( { "project": {"editor": {"permissions": ["read_system"]}}, "organization": {"foo": {"permissions": []}}, @@ -439,33 +464,35 @@ def test_multiple_scopes_rejected_lists_offenders(self): def test_empty_project_block_rejected(self): with pytest.raises(ValueError, match="must be a non-empty"): - controls._parse_roles_overlay({"project": {}}) + permission_controls._parse_roles_overlay({"project": {}}) def test_reserved_role_patch_rejected(self): with pytest.raises(ValueError, match="cannot patch reserved role 'owner'"): - controls._parse_roles_overlay( + permission_controls._parse_roles_overlay( {"project": {"owner": {"permissions": ["read_system"]}}} ) def test_reserved_viewer_patch_rejected(self): with pytest.raises(ValueError, match="cannot patch reserved role 'viewer'"): - controls._parse_roles_overlay( + permission_controls._parse_roles_overlay( {"project": {"viewer": {"permissions": ["read_system"]}}} ) def test_unknown_permission_rejected(self): with pytest.raises(ValueError, match="Unknown permission"): - controls._parse_roles_overlay( + permission_controls._parse_roles_overlay( {"project": {"editor": {"permissions": ["bogus_perm"]}}} ) def test_extra_field_rejected(self): with pytest.raises(ValueError, match="Invalid AGENTA_ACCESS_ROLES_OVERLAY"): - controls._parse_roles_overlay({"project": {"editor": {"surprise": "yes"}}}) + permission_controls._parse_roles_overlay( + {"project": {"editor": {"surprise": "yes"}}} + ) def test_project_focused_shortcut_accepted(self): # Top-level keys are role slugs (no scope wrapper). - overlay = controls._parse_roles_overlay( + overlay = permission_controls._parse_roles_overlay( {"editor": {"permissions": ["read_system"]}} ) assert set(overlay.keys()) == {"editor"} @@ -473,23 +500,25 @@ def test_project_focused_shortcut_accepted(self): def test_project_focused_shortcut_rejects_reserved_role(self): with pytest.raises(ValueError, match="cannot patch reserved role 'owner'"): - controls._parse_roles_overlay({"owner": {"permissions": ["read_system"]}}) + permission_controls._parse_roles_overlay( + {"owner": {"permissions": ["read_system"]}} + ) def test_full_form_with_organization_scope_rejected(self): with pytest.raises(ValueError, match="only supports the 'project' scope"): - controls._parse_roles_overlay( + permission_controls._parse_roles_overlay( {"organization": {"editor": {"permissions": ["read_system"]}}} ) def test_full_form_with_workspace_scope_rejected(self): with pytest.raises(ValueError, match="only supports the 'project' scope"): - controls._parse_roles_overlay( + permission_controls._parse_roles_overlay( {"workspace": {"editor": {"permissions": ["read_system"]}}} ) def test_mixing_scope_and_role_keys_rejected(self): with pytest.raises(ValueError, match="mixes scope keys with non-scope"): - controls._parse_roles_overlay( + permission_controls._parse_roles_overlay( { "project": {"editor": {"permissions": ["read_system"]}}, "auditor": {"permissions": ["read_system"]}, @@ -501,14 +530,14 @@ class TestRolesOverlayApply: def _base_roles(self) -> dict: # Mirror the code-default catalog (minima + default extras for # workspace and project; minima only for organization). - return controls._default_roles() + return permission_controls._default_roles() def test_patch_existing_role_replaces_permissions_in_both_scopes(self): roles = self._base_roles() - overlay = controls._parse_roles_overlay( + overlay = permission_controls._parse_roles_overlay( {"project": {"editor": {"permissions": ["read_system"]}}} ) - result = controls._apply_roles_overlay(roles, overlay) + result = permission_controls._apply_roles_overlay(roles, overlay) for scope in ("workspace", "project"): editor = next(r for r in result[scope] if r["role"] == "editor") @@ -519,10 +548,10 @@ def test_patch_existing_role_preserves_description_when_not_set(self): original_description = next( r for r in roles["project"] if r["role"] == "editor" )["description"] - overlay = controls._parse_roles_overlay( + overlay = permission_controls._parse_roles_overlay( {"project": {"editor": {"permissions": ["read_system"]}}} ) - result = controls._apply_roles_overlay(roles, overlay) + result = permission_controls._apply_roles_overlay(roles, overlay) for scope in ("workspace", "project"): editor = next(r for r in result[scope] if r["role"] == "editor") @@ -533,10 +562,10 @@ def test_patch_existing_role_preserves_permissions_when_not_set(self): original_perms = list( next(r for r in roles["project"] if r["role"] == "editor")["permissions"] ) - overlay = controls._parse_roles_overlay( + overlay = permission_controls._parse_roles_overlay( {"project": {"editor": {"description": "Custom description"}}} ) - result = controls._apply_roles_overlay(roles, overlay) + result = permission_controls._apply_roles_overlay(roles, overlay) for scope in ("workspace", "project"): editor = next(r for r in result[scope] if r["role"] == "editor") @@ -545,7 +574,7 @@ def test_patch_existing_role_preserves_permissions_when_not_set(self): def test_new_role_added_to_both_scopes(self): roles = self._base_roles() - overlay = controls._parse_roles_overlay( + overlay = permission_controls._parse_roles_overlay( { "project": { "auditor": { @@ -555,7 +584,7 @@ def test_new_role_added_to_both_scopes(self): } } ) - result = controls._apply_roles_overlay(roles, overlay) + result = permission_controls._apply_roles_overlay(roles, overlay) for scope in ("workspace", "project"): slugs = [r["role"] for r in result[scope]] @@ -563,16 +592,16 @@ def test_new_role_added_to_both_scopes(self): def test_new_role_without_permissions_rejected(self): roles = self._base_roles() - overlay = controls._parse_roles_overlay( + overlay = permission_controls._parse_roles_overlay( {"project": {"auditor": {"description": "Only description"}}} ) with pytest.raises(ValueError, match="new role requires 'permissions'"): - controls._apply_roles_overlay(roles, overlay) + permission_controls._apply_roles_overlay(roles, overlay) def test_organization_scope_untouched(self): roles = self._base_roles() original_org_slugs = [r["role"] for r in roles["organization"]] - overlay = controls._parse_roles_overlay( + overlay = permission_controls._parse_roles_overlay( { "project": { "auditor": { @@ -582,6 +611,6 @@ def test_organization_scope_untouched(self): } } ) - result = controls._apply_roles_overlay(roles, overlay) + result = permission_controls._apply_roles_overlay(roles, overlay) assert [r["role"] for r in result["organization"]] == original_org_slugs diff --git a/api/ee/tests/pytest/unit/test_billing_router.py b/api/ee/tests/pytest/unit/test_billing_router.py index f0b8b9b372..2e4b5c8d09 100644 --- a/api/ee/tests/pytest/unit/test_billing_router.py +++ b/api/ee/tests/pytest/unit/test_billing_router.py @@ -6,7 +6,7 @@ from ee.src.apis.fastapi.billing import router as billing_router_module from ee.src.apis.fastapi.billing.router import BillingRouter -from ee.src.core.entitlements.types import DefaultPlan +from ee.src.core.access.entitlements.types import DefaultPlan from ee.src.core.subscriptions.types import Event @@ -28,18 +28,22 @@ async def test_fetch_subscription_reads_periods_from_stripe_objects(monkeypatch) monkeypatch.setattr(billing_router_module.env.stripe, "api_key", "sk_test_123") monkeypatch.setattr( - billing_router_module.stripe.Subscription, - "retrieve", - lambda id: SimpleNamespace( - items=SimpleNamespace( - data=[ - SimpleNamespace( - current_period_start=1710000000, - current_period_end=1712592000, - ) - ] + billing_router_module, + "_load_stripe", + lambda: SimpleNamespace( + Subscription=SimpleNamespace( + retrieve=lambda id: SimpleNamespace( + items=SimpleNamespace( + data=[ + SimpleNamespace( + current_period_start=1710000000, + current_period_end=1712592000, + ) + ] + ), + status="trialing", + ), ), - status="trialing", ), ) @@ -81,20 +85,25 @@ async def test_handle_events_reads_subscription_created_metadata_from_stripe_obj monkeypatch.setattr(billing_router_module.env.stripe, "api_key", "sk_test_123") monkeypatch.setattr(billing_router_module.env.stripe, "webhook_secret", None) monkeypatch.setattr( - billing_router_module.stripe.Event, - "construct_from", - lambda payload, api_key: SimpleNamespace( - type="customer.subscription.created", - data=SimpleNamespace( - object=SimpleNamespace( - id="sub_123", - billing_cycle_anchor=1710000000, - metadata=SimpleNamespace( - target=billing_router_module.env.stripe.webhook_target, - organization_id="org_123", - plan=DefaultPlan.CLOUD_V0_PRO.value, + billing_router_module, + "_load_stripe", + lambda: SimpleNamespace( + api_key="sk_test_123", + Event=SimpleNamespace( + construct_from=lambda payload, api_key: SimpleNamespace( + type="customer.subscription.created", + data=SimpleNamespace( + object=SimpleNamespace( + id="sub_123", + billing_cycle_anchor=1710000000, + metadata=SimpleNamespace( + target=billing_router_module.env.stripe.webhook_target, + organization_id="org_123", + plan=DefaultPlan.CLOUD_V0_PRO.value, + ), + ) ), - ) + ), ), ), ) @@ -126,19 +135,24 @@ async def test_handle_events_reads_invoice_metadata_from_stripe_objects(monkeypa monkeypatch.setattr(billing_router_module.env.stripe, "api_key", "sk_test_123") monkeypatch.setattr(billing_router_module.env.stripe, "webhook_secret", None) monkeypatch.setattr( - billing_router_module.stripe.Event, - "construct_from", - lambda payload, api_key: SimpleNamespace( - type="invoice.payment_succeeded", - data=SimpleNamespace( - object=SimpleNamespace( - subscription_details=SimpleNamespace( - metadata=SimpleNamespace( - target=billing_router_module.env.stripe.webhook_target, - organization_id="org_456", + billing_router_module, + "_load_stripe", + lambda: SimpleNamespace( + api_key="sk_test_123", + Event=SimpleNamespace( + construct_from=lambda payload, api_key: SimpleNamespace( + type="invoice.payment_succeeded", + data=SimpleNamespace( + object=SimpleNamespace( + subscription_details=SimpleNamespace( + metadata=SimpleNamespace( + target=billing_router_module.env.stripe.webhook_target, + organization_id="org_456", + ) + ) ) - ) - ) + ), + ), ), ), ) @@ -175,8 +189,13 @@ async def test_cancel_subscription_updates_local_state_after_stripe_cancel(monke retrieve = Mock(return_value=SimpleNamespace(status="active")) cancel = Mock() - monkeypatch.setattr(billing_router_module.stripe.Subscription, "retrieve", retrieve) - monkeypatch.setattr(billing_router_module.stripe.Subscription, "cancel", cancel) + monkeypatch.setattr( + billing_router_module, + "_load_stripe", + lambda: SimpleNamespace( + Subscription=SimpleNamespace(retrieve=retrieve, cancel=cancel), + ), + ) response = await router.cancel_subscription(organization_id="org_123") @@ -233,8 +252,13 @@ async def test_cancel_subscription_reconciles_when_stripe_is_already_canceled( retrieve = Mock(return_value=SimpleNamespace(status="canceled")) cancel = Mock() - monkeypatch.setattr(billing_router_module.stripe.Subscription, "retrieve", retrieve) - monkeypatch.setattr(billing_router_module.stripe.Subscription, "cancel", cancel) + monkeypatch.setattr( + billing_router_module, + "_load_stripe", + lambda: SimpleNamespace( + Subscription=SimpleNamespace(retrieve=retrieve, cancel=cancel), + ), + ) response = await router.cancel_subscription(organization_id="org_123") diff --git a/api/ee/tests/pytest/unit/test_billing_settings.py b/api/ee/tests/pytest/unit/test_billing_settings.py index 3a5076aa1a..384c7ffdc6 100644 --- a/api/ee/tests/pytest/unit/test_billing_settings.py +++ b/api/ee/tests/pytest/unit/test_billing_settings.py @@ -9,7 +9,7 @@ import pytest from ee.src.core.subscriptions import settings -from ee.src.core.entitlements.types import DefaultPlan +from ee.src.core.access.entitlements.types import DefaultPlan # --------------------------------------------------------------------------- diff --git a/api/ee/tests/pytest/unit/test_compute_meter_id.py b/api/ee/tests/pytest/unit/test_compute_meter_id.py index 6075d8f7ba..c0b1db1c44 100644 --- a/api/ee/tests/pytest/unit/test_compute_meter_id.py +++ b/api/ee/tests/pytest/unit/test_compute_meter_id.py @@ -8,9 +8,10 @@ rows that the database cannot detect. These tests pin format invariants (determinism, equivalence, distinctness) -plus the namespace derivation. Any change that alters the canonical form will -surface as a failure in the distinctness/equivalence tests or a re-keying of -the namespace test. +plus the namespace derivation. The absolute output values are NOT pinned — +they depend on `AGENTA_UUID_NAMESPACE` which is environment-derived. Any +change that alters the canonical form will surface as a failure in the +distinctness/equivalence tests or a re-keying of the namespace test. """ import uuid @@ -23,8 +24,7 @@ MeterPeriod, compute_meter_id, ) -from ee.src.core.entitlements.types import Counter - +from ee.src.core.access.entitlements.types import Counter # Fixed UUIDs used across the table — keep them stable. ORG = uuid.UUID("a1111111-1111-1111-1111-111111111111") @@ -34,13 +34,12 @@ # --------------------------------------------------------------------------- -# Namespace UUID — derived from uuid5(NAMESPACE_DNS, "agenta"). +# Namespace UUID — derived from the fixed project namespace root. # --------------------------------------------------------------------------- -def test_namespace_uuid_is_derived_from_uuid_namespace(): - """The meters namespace must be a stable derivative of the project-wide - agenta namespace UUID. Changing either side forces a full re-backfill.""" +def test_namespace_uuid_is_derived_from_fixed_root_namespace(): + """The meters namespace must remain a stable derivative of the fixed root.""" expected = uuid.uuid5(uuid.uuid5(uuid.NAMESPACE_DNS, "agenta"), "meters") assert AGENTA_METERS_NAMESPACE_UUID == expected diff --git a/api/ee/tests/pytest/unit/test_controls_env_override.py b/api/ee/tests/pytest/unit/test_controls_env_override.py index 67cf6a4556..04bb36c37d 100644 --- a/api/ee/tests/pytest/unit/test_controls_env_override.py +++ b/api/ee/tests/pytest/unit/test_controls_env_override.py @@ -56,12 +56,12 @@ def test_no_env_uses_defaults(self): # Plans count should equal DefaultPlan enum size; catalog count # should match (one entry per plan in DEFAULT_CATALOG plus the # Enterprise contact-sales tier which has no `plan` field). - from ee.src.core.entitlements.types import DEFAULT_CATALOG, DefaultPlan + from ee.src.core.access.entitlements.types import DEFAULT_CATALOG, DefaultPlan expected_plans = len(list(DefaultPlan)) expected_catalog = len(DEFAULT_CATALOG) out = _ok( - "from ee.src.core.entitlements.controls import get_plans; " + "from ee.src.core.access.controls import get_plans; " "from ee.src.core.subscriptions.settings import get_catalog; " "print(len(get_plans())); print(len(get_catalog()))" ) @@ -155,7 +155,7 @@ class TestPlansOverride: def test_consistent_override_works_end_to_end(self): out = _ok( - "from ee.src.core.entitlements.controls import get_plans, get_plan_description; " + "from ee.src.core.access.controls import get_plans, get_plan_description; " "from ee.src.core.subscriptions.settings import get_catalog, get_free_plan; " "print(','.join(sorted(get_plans()))); " "print(get_plan_description('only_plan')); " @@ -171,21 +171,21 @@ def test_consistent_override_works_end_to_end(self): def test_invalid_json_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_plans", + "from ee.src.core.access.controls import get_plans", {"AGENTA_ACCESS_PLANS": "{not json"}, "AGENTA_ACCESS_PLANS is not valid JSON", ) def test_wrong_top_level_type_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_plans", + "from ee.src.core.access.controls import get_plans", {"AGENTA_ACCESS_PLANS": "[1,2,3]"}, "must be a JSON object", ) def test_empty_object_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_plans", + "from ee.src.core.access.controls import get_plans", {"AGENTA_ACCESS_PLANS": "{}"}, "non-empty", ) @@ -194,7 +194,7 @@ def test_plan_with_only_description_allowed(self): # Display-only plans (no enforced trackers) are accepted. They show # up in the effective plan map with an empty entitlements dict. out = _ok( - "from ee.src.core.entitlements.controls import get_plans, get_plan_entitlements; " + "from ee.src.core.access.controls import get_plans, get_plan_entitlements; " "print(sorted(get_plans())); print(get_plan_entitlements('x'))", env_extra={"AGENTA_ACCESS_PLANS": '{"x":{"description":"y"}}'}, ) @@ -517,7 +517,7 @@ class TestRolesOverride: def test_custom_role_with_known_permission_appended_to_minima(self): out = _ok( - "from ee.src.core.entitlements.controls import get_roles, get_role_permissions; " + "from ee.src.core.access.controls import get_roles, get_role_permissions; " "print(','.join(r['role'] for r in get_roles('project'))); " "print(','.join(get_role_permissions('project','reviewer')))", env_extra={ @@ -528,7 +528,7 @@ def test_custom_role_with_known_permission_appended_to_minima(self): ) lines = out.splitlines() # owner + viewer minima first, custom role last. - assert lines[0] == "owner,viewer,reviewer" + assert lines[0] == "owner,admin,viewer,reviewer" assert lines[1] == "read_system" def test_project_override_is_mirrored_to_workspace_today(self): @@ -537,7 +537,7 @@ def test_project_override_is_mirrored_to_workspace_today(self): # membership. A project-only override therefore intentionally replaces # workspace extras too, instead of leaving workspace defaults intact. out = _ok( - "from ee.src.core.entitlements.controls import get_roles; " + "from ee.src.core.access.controls import get_roles; " "print(','.join(r['role'] for r in get_roles('workspace')))", env_extra={ "AGENTA_ACCESS_ROLES": json.dumps( @@ -545,11 +545,11 @@ def test_project_override_is_mirrored_to_workspace_today(self): ) }, ) - assert out.strip() == "owner,viewer,reviewer" + assert out.strip() == "owner,admin,viewer,reviewer" def test_unknown_permission_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", { "AGENTA_ACCESS_ROLES": json.dumps( {"project": [{"role": "x", "permissions": ["bogus_perm_id"]}]} @@ -560,7 +560,7 @@ def test_unknown_permission_fails_startup(self): def test_redefining_owner_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", { "AGENTA_ACCESS_ROLES": json.dumps( {"project": [{"role": "owner", "permissions": ["*"]}]} @@ -571,7 +571,7 @@ def test_redefining_owner_fails_startup(self): def test_redefining_viewer_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", { "AGENTA_ACCESS_ROLES": json.dumps( {"project": [{"role": "viewer", "permissions": ["read_system"]}]} @@ -582,7 +582,7 @@ def test_redefining_viewer_fails_startup(self): def test_duplicate_custom_role_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", { "AGENTA_ACCESS_ROLES": json.dumps( { @@ -598,14 +598,14 @@ def test_duplicate_custom_role_fails_startup(self): def test_empty_roles_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", {"AGENTA_ACCESS_ROLES": "{}"}, "non-empty", ) def test_empty_scope_list_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", {"AGENTA_ACCESS_ROLES": json.dumps({"project": []})}, "non-empty list of roles", ) @@ -649,8 +649,8 @@ def test_overlay_patches_traces_retention(self): # Retention enum values, so we use one of those rather than an # arbitrary minute count. out = _ok( - "from ee.src.core.entitlements.controls import get_plan_entitlements; " - "from ee.src.core.entitlements.types import Tracker, Counter; " + "from ee.src.core.access.controls import get_plan_entitlements; " + "from ee.src.core.access.entitlements.types import Tracker, Counter; " "ent = get_plan_entitlements('cloud_v0_hobby'); " "print(ent[Tracker.COUNTERS][Counter.TRACES_INGESTED].retention.value)", env_extra={ @@ -667,8 +667,8 @@ def test_overlay_preserves_other_quota_fields(self): # retention=Retention.MONTHLY. Overlay sets only retention → free # and period stay. out = _ok( - "from ee.src.core.entitlements.controls import get_plan_entitlements; " - "from ee.src.core.entitlements.types import Tracker, Counter; " + "from ee.src.core.access.controls import get_plan_entitlements; " + "from ee.src.core.access.entitlements.types import Tracker, Counter; " "q = get_plan_entitlements('cloud_v0_hobby')" "[Tracker.COUNTERS][Counter.TRACES_INGESTED]; " "print(q.retention.value, q.free, q.period.value)", @@ -683,8 +683,8 @@ def test_overlay_preserves_other_quota_fields(self): def test_overlay_patches_throttle_rate_only(self): out = _ok( - "from ee.src.core.entitlements.controls import get_plan_entitlements; " - "from ee.src.core.entitlements.types import Tracker, Category; " + "from ee.src.core.access.controls import get_plan_entitlements; " + "from ee.src.core.access.entitlements.types import Tracker, Category; " "ent = get_plan_entitlements('cloud_v0_hobby'); " "t = next(t for t in ent[Tracker.THROTTLES] " " if t.categories == [Category.STANDARD]); " @@ -701,7 +701,7 @@ def test_overlay_patches_throttle_rate_only(self): def test_overlay_invalid_field_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_plans", + "from ee.src.core.access.controls import get_plans", { "AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY": json.dumps( {"flags": {"bogus_flag": True}} @@ -712,7 +712,7 @@ def test_overlay_invalid_field_fails_startup(self): def test_overlay_targeting_unknown_plan_fails(self): _fails( - "from ee.src.core.entitlements.controls import get_plans", + "from ee.src.core.access.controls import get_plans", { "AGENTA_ACCESS_DEFAULT_PLAN": "ghost_plan", "AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY": json.dumps( @@ -724,7 +724,7 @@ def test_overlay_targeting_unknown_plan_fails(self): def test_overlay_empty_object_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_plans", + "from ee.src.core.access.controls import get_plans", {"AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY": "{}"}, "non-empty", ) @@ -794,7 +794,7 @@ class TestRolesOverlay: def test_overlay_patches_editor_permissions_in_both_scopes(self): out = _ok( - "from ee.src.core.entitlements.controls import get_role_permissions; " + "from ee.src.core.access.controls import get_role_permissions; " "print(get_role_permissions('workspace', 'editor')); " "print(get_role_permissions('project', 'editor'))", env_extra={ @@ -809,7 +809,7 @@ def test_overlay_patches_editor_permissions_in_both_scopes(self): def test_overlay_adds_new_role_to_both_scopes(self): out = _ok( - "from ee.src.core.entitlements.controls import get_roles; " + "from ee.src.core.access.controls import get_roles; " "print('auditor' in [r['role'] for r in get_roles('workspace')]); " "print('auditor' in [r['role'] for r in get_roles('project')])", env_extra={ @@ -831,7 +831,7 @@ def test_overlay_organization_scope_untouched(self): # Organization scope only has the minima (owner + viewer) by default. # The overlay must not change that — it targets workspace + project. out = _ok( - "from ee.src.core.entitlements.controls import get_roles; " + "from ee.src.core.access.controls import get_roles; " "print(','.join(r['role'] for r in get_roles('organization')))", env_extra={ "AGENTA_ACCESS_ROLES_OVERLAY": json.dumps( @@ -846,11 +846,11 @@ def test_overlay_organization_scope_untouched(self): ) }, ) - assert out.strip() == "owner,viewer" + assert out.strip() == "owner,admin,viewer" def test_overlay_non_project_scope_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", { "AGENTA_ACCESS_ROLES_OVERLAY": json.dumps( {"workspace": {"editor": {"permissions": ["read_system"]}}} @@ -861,7 +861,7 @@ def test_overlay_non_project_scope_fails_startup(self): def test_overlay_reserved_role_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", { "AGENTA_ACCESS_ROLES_OVERLAY": json.dumps( {"project": {"owner": {"permissions": ["*"]}}} @@ -872,7 +872,7 @@ def test_overlay_reserved_role_fails_startup(self): def test_overlay_unknown_permission_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", { "AGENTA_ACCESS_ROLES_OVERLAY": json.dumps( {"project": {"editor": {"permissions": ["bogus_perm"]}}} @@ -883,14 +883,14 @@ def test_overlay_unknown_permission_fails_startup(self): def test_overlay_empty_object_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", {"AGENTA_ACCESS_ROLES_OVERLAY": "{}"}, "non-empty", ) def test_overlay_new_role_without_permissions_fails_startup(self): _fails( - "from ee.src.core.entitlements.controls import get_roles", + "from ee.src.core.access.controls import get_roles", { "AGENTA_ACCESS_ROLES_OVERLAY": json.dumps( {"project": {"auditor": {"description": "x"}}} diff --git a/api/ee/tests/pytest/unit/test_events_retention.py b/api/ee/tests/pytest/unit/test_events_retention.py index a3a794ff74..10b7fd4d12 100644 --- a/api/ee/tests/pytest/unit/test_events_retention.py +++ b/api/ee/tests/pytest/unit/test_events_retention.py @@ -13,7 +13,7 @@ import pytest -from ee.src.core.entitlements.types import ( +from ee.src.core.access.entitlements.types import ( Counter, Period, Quota, diff --git a/api/ee/tests/pytest/unit/test_meters_dao_fetch.py b/api/ee/tests/pytest/unit/test_meters_dao_fetch.py index 676f1d4c90..ecdeba0982 100644 --- a/api/ee/tests/pytest/unit/test_meters_dao_fetch.py +++ b/api/ee/tests/pytest/unit/test_meters_dao_fetch.py @@ -68,14 +68,16 @@ async def __aexit__(self, exc_type, exc, tb): return False -def _patch_session(monkeypatch, session: _Session): - from ee.src.dbs.postgres.meters import dao as dao_module +class _Engine: + def __init__(self, session: _Session): + self._session = session - monkeypatch.setattr( - dao_module.engine, - "core_session", - lambda: _SessionContext(session), - ) + def session(self): + return _SessionContext(self._session) + + +def _dao_with_session(session: _Session) -> MetersDAO: + return MetersDAO(engine=_Engine(session)) def _where_sql(stmt) -> str: @@ -103,12 +105,11 @@ def _where_sql(stmt) -> str: class TestFetchScopeFilters: @pytest.mark.asyncio - async def test_org_only_scope_binds_finer_dims_to_is_null(self, monkeypatch): + async def test_org_only_scope_binds_finer_dims_to_is_null(self): """`MeterScope(organization_id=X)` → finer dims IS NULL.""" session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch(scope=MeterScope(organization_id=ORG)) assert len(session.executed_statements) == 1 @@ -118,12 +119,11 @@ async def test_org_only_scope_binds_finer_dims_to_is_null(self, monkeypatch): assert "user_id is null" in sql @pytest.mark.asyncio - async def test_workspace_scope_binds_below_to_is_null(self, monkeypatch): + async def test_workspace_scope_binds_below_to_is_null(self): """workspace-scoped read should not match project/user rows.""" session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch( scope=MeterScope(organization_id=ORG, workspace_id=WRK), ) @@ -134,12 +134,11 @@ async def test_workspace_scope_binds_below_to_is_null(self, monkeypatch): assert "user_id is null" in sql @pytest.mark.asyncio - async def test_user_scope_binds_every_dim(self, monkeypatch): + async def test_user_scope_binds_every_dim(self): """fully-bound user scope → all four dims bound to concrete values.""" session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch( scope=MeterScope( organization_id=ORG, @@ -155,12 +154,11 @@ async def test_user_scope_binds_every_dim(self, monkeypatch): assert "is null" not in sql @pytest.mark.asyncio - async def test_scope_none_applies_no_filter(self, monkeypatch): + async def test_scope_none_applies_no_filter(self): """`scope=None` escape hatch — no scope filter at all.""" session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch(scope=None) sql = _where_sql(session.executed_statements[0]).lower() @@ -170,12 +168,11 @@ async def test_scope_none_applies_no_filter(self, monkeypatch): assert "user_id" not in sql @pytest.mark.asyncio - async def test_empty_scope_is_equivalent_to_scope_none(self, monkeypatch): + async def test_empty_scope_is_equivalent_to_scope_none(self): """`MeterScope()` (all dims unset) is treated as the same admin escape as `scope=None`.""" session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch(scope=MeterScope()) sql = _where_sql(session.executed_statements[0]).lower() @@ -187,12 +184,11 @@ async def test_empty_scope_is_equivalent_to_scope_none(self, monkeypatch): class TestFetchPeriodFilters: @pytest.mark.asyncio - async def test_monthly_period_binds_day_to_is_null(self, monkeypatch): + async def test_monthly_period_binds_day_to_is_null(self): """MONTHLY read (year, month, day=None) must not match DAILY rows.""" session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch(period=MeterPeriod(year=2026, month=5)) sql = _where_sql(session.executed_statements[0]).lower() @@ -201,12 +197,11 @@ async def test_monthly_period_binds_day_to_is_null(self, monkeypatch): assert "day is null" in sql @pytest.mark.asyncio - async def test_daily_period_binds_every_dim(self, monkeypatch): + async def test_daily_period_binds_every_dim(self): """DAILY read binds year+month+day to concrete values.""" session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch(period=MeterPeriod(year=2026, month=5, day=19)) sql = _where_sql(session.executed_statements[0]).lower() @@ -217,12 +212,11 @@ async def test_daily_period_binds_every_dim(self, monkeypatch): assert "day is null" not in sql @pytest.mark.asyncio - async def test_empty_period_binds_all_to_is_null(self, monkeypatch): + async def test_empty_period_binds_all_to_is_null(self): """`MeterPeriod()` (no period) → all three IS NULL — pins lifetime rows.""" session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch(period=MeterPeriod()) sql = _where_sql(session.executed_statements[0]).lower() @@ -231,12 +225,11 @@ async def test_empty_period_binds_all_to_is_null(self, monkeypatch): assert "day is null" in sql @pytest.mark.asyncio - async def test_period_none_applies_no_filter(self, monkeypatch): + async def test_period_none_applies_no_filter(self): """`period=None` escape hatch — no period filter at all.""" session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch(period=None) sql = _where_sql(session.executed_statements[0]).lower() @@ -247,22 +240,20 @@ async def test_period_none_applies_no_filter(self, monkeypatch): class TestFetchKeyFilter: @pytest.mark.asyncio - async def test_key_bound(self, monkeypatch): + async def test_key_bound(self): session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch(key=Meters.TRACES_RETRIEVED) sql = _where_sql(session.executed_statements[0]).lower() assert "key" in sql @pytest.mark.asyncio - async def test_key_none_applies_no_filter(self, monkeypatch): + async def test_key_none_applies_no_filter(self): session = _Session() - _patch_session(monkeypatch, session) - dao = MetersDAO() + dao = _dao_with_session(session) await dao.fetch(key=None) sql = _where_sql(session.executed_statements[0]).lower() diff --git a/api/ee/tests/pytest/unit/test_meters_dao_strict_soft.py b/api/ee/tests/pytest/unit/test_meters_dao_strict_soft.py index 318e9a9321..91fb80afe6 100644 --- a/api/ee/tests/pytest/unit/test_meters_dao_strict_soft.py +++ b/api/ee/tests/pytest/unit/test_meters_dao_strict_soft.py @@ -26,7 +26,7 @@ import pytest -from ee.src.core.entitlements.types import Quota +from ee.src.core.access.entitlements.types import Quota from ee.src.core.meters.types import MeterDTO, Meters from ee.src.dbs.postgres.meters.dao import MetersDAO @@ -104,14 +104,16 @@ async def __aexit__(self, exc_type, exc, tb): return False -def _patch_session(monkeypatch, session: _Session): - from ee.src.dbs.postgres.meters import dao as dao_module +class _MockEngine: + def __init__(self, session: _Session): + self._session = session - monkeypatch.setattr( - dao_module.engine, - "core_session", - lambda: _SessionContext(session), - ) + def session(self): + return _SessionContext(self._session) + + +def _mock_engine(session: _Session) -> _MockEngine: + return _MockEngine(session) # --------------------------------------------------------------------------- @@ -121,12 +123,10 @@ def _patch_session(monkeypatch, session: _Session): class TestCheck: @pytest.mark.asyncio - async def test_below_limit_with_delta_allows(self, monkeypatch): + async def test_below_limit_with_delta_allows(self): """current=5, delta=3, limit=10 → 8 <= 10 → allowed.""" session = _Session(scalar=SimpleNamespace(value=5, synced=0)) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, dto = await dao.check( meter=_meter(delta=3), quota=Quota(limit=10), @@ -136,12 +136,10 @@ async def test_below_limit_with_delta_allows(self, monkeypatch): assert dto.value == 5 # returned current value @pytest.mark.asyncio - async def test_at_limit_with_zero_delta_allows(self, monkeypatch): + async def test_at_limit_with_zero_delta_allows(self): """current=10, delta=0, limit=10 → 10 <= 10 → allowed (boundary).""" session = _Session(scalar=SimpleNamespace(value=10, synced=0)) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _ = await dao.check( meter=_meter(delta=0), quota=Quota(limit=10), @@ -150,12 +148,10 @@ async def test_at_limit_with_zero_delta_allows(self, monkeypatch): assert allowed is True @pytest.mark.asyncio - async def test_would_exceed_limit_blocks(self, monkeypatch): + async def test_would_exceed_limit_blocks(self): """current=10, delta=2, limit=10 → 12 > 10 → blocked.""" session = _Session(scalar=SimpleNamespace(value=10, synced=0)) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _ = await dao.check( meter=_meter(delta=2), quota=Quota(limit=10), @@ -164,12 +160,10 @@ async def test_would_exceed_limit_blocks(self, monkeypatch): assert allowed is False @pytest.mark.asyncio - async def test_negative_delta_clamps_at_zero(self, monkeypatch): + async def test_negative_delta_clamps_at_zero(self): """current=3, delta=-10, limit=10 → max(-7, 0)=0 <= 10 → allowed.""" session = _Session(scalar=SimpleNamespace(value=3, synced=0)) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _ = await dao.check( meter=_meter(delta=-10), quota=Quota(limit=10), @@ -178,12 +172,10 @@ async def test_negative_delta_clamps_at_zero(self, monkeypatch): assert allowed is True @pytest.mark.asyncio - async def test_no_existing_row_treats_current_as_zero(self, monkeypatch): + async def test_no_existing_row_treats_current_as_zero(self): """No row in DB, delta=5, limit=10 → 0+5=5 <= 10 → allowed.""" session = _Session(scalar=None) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, dto = await dao.check( meter=_meter(delta=5), quota=Quota(limit=10), @@ -194,12 +186,10 @@ async def test_no_existing_row_treats_current_as_zero(self, monkeypatch): assert dto.synced == 0 @pytest.mark.asyncio - async def test_no_limit_always_allows(self, monkeypatch): + async def test_no_limit_always_allows(self): """limit=None → unconditional pass even at huge values.""" session = _Session(scalar=SimpleNamespace(value=10_000_000, synced=0)) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _ = await dao.check( meter=_meter(delta=1_000_000), quota=Quota(limit=None), @@ -234,12 +224,10 @@ def _extract_where_sql(statement) -> str: class TestAdjustStrictVsSoft: @pytest.mark.asyncio - async def test_strict_emits_value_plus_delta_predicate(self, monkeypatch): + async def test_strict_emits_value_plus_delta_predicate(self): """Strict mode must gate on `value + delta <= limit`.""" session = _Session(row=(7,)) # post-update value - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) await dao.adjust( meter=_meter(delta=2), quota=Quota(limit=10, strict=True), @@ -255,7 +243,7 @@ async def test_strict_emits_value_plus_delta_predicate(self, monkeypatch): @pytest.mark.asyncio async def test_nonstrict_emits_value_strictly_less_than_limit_predicate( - self, monkeypatch + self, ): """Non-strict mode gates the SQL predicate on `value < limit`. @@ -266,9 +254,7 @@ async def test_nonstrict_emits_value_strictly_less_than_limit_predicate( already at-or-over limit. """ session = _Session(row=(7,)) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) await dao.adjust( meter=_meter(delta=2), quota=Quota(limit=10, strict=False), @@ -301,15 +287,13 @@ async def test_nonstrict_emits_value_strictly_less_than_limit_predicate( # --------------------------------------------------------------------- @pytest.mark.asyncio - async def test_huge_delta_denied_in_strict(self, monkeypatch): + async def test_huge_delta_denied_in_strict(self): """0 + 12 with limit=10 → predictable self-overshoot → deny. Python-side fast-path; no DB call should land. """ session = _Session(row=None) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _, _ = await dao.adjust( meter=_meter(delta=12), quota=Quota(limit=10, strict=True), @@ -321,15 +305,13 @@ async def test_huge_delta_denied_in_strict(self, monkeypatch): ) @pytest.mark.asyncio - async def test_huge_delta_denied_in_nonstrict(self, monkeypatch): + async def test_huge_delta_denied_in_nonstrict(self): """0 + 12 with limit=10 → predictable self-overshoot → deny. Non-strict shares the same `delta <= limit` rule as strict. """ session = _Session(row=None) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _, _ = await dao.adjust( meter=_meter(delta=12), quota=Quota(limit=10, strict=False), @@ -339,13 +321,11 @@ async def test_huge_delta_denied_in_nonstrict(self, monkeypatch): assert session.executed_statements == [] @pytest.mark.asyncio - async def test_at_limit_denied_in_strict(self, monkeypatch): + async def test_at_limit_denied_in_strict(self): """10 + 2 with limit=10 → strict SQL predicate filters the row out (greatest(10+2, 0) > 10) → RETURNING empty → deny.""" session = _Session(row=None) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _, _ = await dao.adjust( meter=_meter(delta=2), quota=Quota(limit=10, strict=True), @@ -357,13 +337,11 @@ async def test_at_limit_denied_in_strict(self, monkeypatch): assert len(session.executed_statements) == 1 @pytest.mark.asyncio - async def test_at_limit_denied_in_nonstrict(self, monkeypatch): + async def test_at_limit_denied_in_nonstrict(self): """10 + 2 with limit=10 → non-strict SQL predicate `value < 10` rejects current=10 → RETURNING empty → deny.""" session = _Session(row=None) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _, _ = await dao.adjust( meter=_meter(delta=2), quota=Quota(limit=10, strict=False), @@ -373,13 +351,11 @@ async def test_at_limit_denied_in_nonstrict(self, monkeypatch): assert len(session.executed_statements) == 1 @pytest.mark.asyncio - async def test_one_over_denied_in_strict(self, monkeypatch): + async def test_one_over_denied_in_strict(self): """9 + 2 with limit=10 → strict SQL predicate `greatest(9+2, 0) > 10` rejects → deny.""" session = _Session(row=None) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _, _ = await dao.adjust( meter=_meter(delta=2), quota=Quota(limit=10, strict=True), @@ -388,7 +364,7 @@ async def test_one_over_denied_in_strict(self, monkeypatch): assert allowed is False @pytest.mark.asyncio - async def test_one_over_allowed_in_nonstrict(self, monkeypatch): + async def test_one_over_allowed_in_nonstrict(self): """9 + 2 with limit=10 → non-strict SQL predicate `9 < 10` → allow. This is the cross-the-line-once case: the request itself crosses @@ -396,9 +372,7 @@ async def test_one_over_allowed_in_nonstrict(self, monkeypatch): request would then be denied by the same predicate. """ session = _Session(row=(11,)) # post-update value - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _, _ = await dao.adjust( meter=_meter(delta=2), quota=Quota(limit=10, strict=False), @@ -407,13 +381,11 @@ async def test_one_over_allowed_in_nonstrict(self, monkeypatch): assert allowed is True @pytest.mark.asyncio - async def test_fills_exactly_allowed_in_both_modes(self, monkeypatch): + async def test_fills_exactly_allowed_in_both_modes(self): """8 + 2 with limit=10 → equal to limit → allowed in both modes.""" for strict in (True, False): session = _Session(row=(10,)) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _, _ = await dao.adjust( meter=_meter(delta=2), quota=Quota(limit=10, strict=strict), @@ -421,16 +393,14 @@ async def test_fills_exactly_allowed_in_both_modes(self, monkeypatch): assert allowed is True, f"strict={strict}: 8+2 should allow" @pytest.mark.asyncio - async def test_strict_blocks_when_proposed_value_exceeds_limit(self, monkeypatch): + async def test_strict_blocks_when_proposed_value_exceeds_limit(self): """`adjust` early-returns False when caller-set value > limit. This is the absolute-value pre-check at the top of `adjust` (delta-mode callers hit the SQL predicate instead). """ session = _Session(row=None) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, _, _ = await dao.adjust( meter=_meter(value=20), # explicit value, no delta quota=Quota(limit=10, strict=True), @@ -441,13 +411,11 @@ async def test_strict_blocks_when_proposed_value_exceeds_limit(self, monkeypatch assert session.executed_statements == [] @pytest.mark.asyncio - async def test_returns_false_when_predicate_blocks(self, monkeypatch): + async def test_returns_false_when_predicate_blocks(self): """When the WHERE clause filters the row out, RETURNING is empty → upsert "succeeded" with zero rows → DAO returns allowed=False.""" session = _Session(row=None) # RETURNING produced nothing - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, dto, _ = await dao.adjust( meter=_meter(delta=5), quota=Quota(limit=10, strict=True), @@ -459,13 +427,11 @@ async def test_returns_false_when_predicate_blocks(self, monkeypatch): assert dto.value == 5 @pytest.mark.asyncio - async def test_returns_true_when_returning_row_present(self, monkeypatch): + async def test_returns_true_when_returning_row_present(self): """RETURNING produced a row → upsert succeeded → allowed=True with the post-update value coming from RETURNING.""" session = _Session(row=(7,)) - _patch_session(monkeypatch, session) - - dao = MetersDAO() + dao = MetersDAO(engine=_mock_engine(session)) allowed, dto, _ = await dao.adjust( meter=_meter(delta=2), quota=Quota(limit=10, strict=True), diff --git a/api/ee/tests/pytest/unit/test_meters_types.py b/api/ee/tests/pytest/unit/test_meters_types.py index ec2ef97c94..c78988055c 100644 --- a/api/ee/tests/pytest/unit/test_meters_types.py +++ b/api/ee/tests/pytest/unit/test_meters_types.py @@ -1,4 +1,4 @@ -from ee.src.core.entitlements.types import Counter, Gauge +from ee.src.core.access.entitlements.types import Counter, Gauge from ee.src.core.meters.types import Meters diff --git a/api/ee/tests/pytest/unit/test_period_from.py b/api/ee/tests/pytest/unit/test_period_from.py index e8f762a6dc..8c74aee4de 100644 --- a/api/ee/tests/pytest/unit/test_period_from.py +++ b/api/ee/tests/pytest/unit/test_period_from.py @@ -18,9 +18,9 @@ import pytest -from ee.src.core.entitlements.types import Period +from ee.src.core.access.entitlements.types import Period from ee.src.core.meters.types import MeterPeriod -from ee.src.utils.entitlements import period_from +from ee.src.core.access.entitlements.service import period_from # --------------------------------------------------------------------------- @@ -47,7 +47,7 @@ def test_period_yearly_sets_only_year(): """YEARLY buckets the meter by year. Month/day must stay None so every row in the same year shares a `meter_id`.""" fake_now = datetime(2026, 5, 18, 12, 0, 0, tzinfo=timezone.utc) - with patch("ee.src.utils.entitlements.datetime") as mock_dt: + with patch("ee.src.core.access.entitlements.service.datetime") as mock_dt: mock_dt.now.return_value = fake_now mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) p = period_from(period=Period.YEARLY) @@ -65,7 +65,7 @@ def test_period_yearly_sets_only_year(): def test_period_monthly_sets_year_and_month_without_anchor(): """MONTHLY without an anchor returns the calendar (year, month).""" fake_now = datetime(2026, 5, 18, 12, 0, 0, tzinfo=timezone.utc) - with patch("ee.src.utils.entitlements.datetime") as mock_dt: + with patch("ee.src.core.access.entitlements.service.datetime") as mock_dt: mock_dt.now.return_value = fake_now mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) p = period_from(period=Period.MONTHLY) @@ -78,7 +78,7 @@ def test_period_monthly_advances_when_day_meets_anchor(): """MONTHLY with `anchor=N` and `now.day >= N` rolls into next month — Stripe-style anchor semantics. Day=18, anchor=15 advances to June.""" fake_now = datetime(2026, 5, 18, 12, 0, 0, tzinfo=timezone.utc) - with patch("ee.src.utils.entitlements.datetime") as mock_dt: + with patch("ee.src.core.access.entitlements.service.datetime") as mock_dt: mock_dt.now.return_value = fake_now mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) p = period_from(period=Period.MONTHLY, anchor=15) @@ -90,7 +90,7 @@ def test_period_monthly_advances_when_day_meets_anchor(): def test_period_monthly_stays_when_day_before_anchor(): """MONTHLY with `now.day < anchor` stays in the current period.""" fake_now = datetime(2026, 5, 10, 12, 0, 0, tzinfo=timezone.utc) - with patch("ee.src.utils.entitlements.datetime") as mock_dt: + with patch("ee.src.core.access.entitlements.service.datetime") as mock_dt: mock_dt.now.return_value = fake_now mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) p = period_from(period=Period.MONTHLY, anchor=15) @@ -102,7 +102,7 @@ def test_period_monthly_stays_when_day_before_anchor(): def test_period_monthly_year_rollover_with_anchor(): """Dec 20 + anchor=15 → next period is (2027, 1).""" fake_now = datetime(2026, 12, 20, 12, 0, 0, tzinfo=timezone.utc) - with patch("ee.src.utils.entitlements.datetime") as mock_dt: + with patch("ee.src.core.access.entitlements.service.datetime") as mock_dt: mock_dt.now.return_value = fake_now mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) p = period_from(period=Period.MONTHLY, anchor=15) @@ -118,7 +118,7 @@ def test_period_monthly_year_rollover_with_anchor(): def test_period_daily_sets_full_calendar_date(): """DAILY buckets the meter by full calendar day.""" fake_now = datetime(2026, 5, 18, 12, 0, 0, tzinfo=timezone.utc) - with patch("ee.src.utils.entitlements.datetime") as mock_dt: + with patch("ee.src.core.access.entitlements.service.datetime") as mock_dt: mock_dt.now.return_value = fake_now mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) p = period_from(period=Period.DAILY) @@ -130,7 +130,7 @@ def test_period_daily_ignores_anchor(): """DAILY is calendar-day-aligned; anchor only applies to MONTHLY. Same input with and without an anchor must produce the same bucket.""" fake_now = datetime(2026, 5, 18, 12, 0, 0, tzinfo=timezone.utc) - with patch("ee.src.utils.entitlements.datetime") as mock_dt: + with patch("ee.src.core.access.entitlements.service.datetime") as mock_dt: mock_dt.now.return_value = fake_now mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) p_without = period_from(period=Period.DAILY) diff --git a/api/ee/tests/pytest/unit/test_scope_from.py b/api/ee/tests/pytest/unit/test_scope_from.py index 7b15bf21e9..934cbb238d 100644 --- a/api/ee/tests/pytest/unit/test_scope_from.py +++ b/api/ee/tests/pytest/unit/test_scope_from.py @@ -23,9 +23,9 @@ import pytest -from ee.src.core.entitlements.types import Scope +from ee.src.core.access.entitlements.types import Scope from ee.src.core.meters.types import MeterScope -from ee.src.utils.entitlements import scope_from +from ee.src.core.access.entitlements.service import scope_from from oss.src.utils.context import ( AuthScope, AuthContext, diff --git a/api/ee/tests/requirements.txt b/api/ee/tests/requirements.txt deleted file mode 100644 index 510e3b3b6f..0000000000 --- a/api/ee/tests/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --r ../../oss/tests/requirements.txt \ No newline at end of file diff --git a/api/entrypoints/routers.py b/api/entrypoints/routers.py index d02d91f11e..f60566b1fe 100644 --- a/api/entrypoints/routers.py +++ b/api/entrypoints/routers.py @@ -1,5 +1,7 @@ from contextlib import asynccontextmanager +import time +import agenta as ag from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.gzip import GZipMiddleware @@ -13,6 +15,16 @@ from oss.src.utils.logging import get_module_logger from oss.src.utils.helpers import warn_deprecated_env_vars, validate_required_env_vars +# Engines +from oss.src.dbs.postgres.shared.engine import ( + get_transactions_engine, + get_analytics_engine, +) +from oss.src.dbs.redis.shared.engine import ( + get_cache_engine, + get_streams_engine, +) + from oss.databases.postgres.migrations.core.utils import ( check_for_new_migrations as check_for_new_core_migrations, ) @@ -20,8 +32,8 @@ check_for_new_migrations as check_for_new_tracing_migrations, ) -from oss.src.services.auth_service import authentication_middleware -from oss.src.services.analytics_service import analytics_middleware +from oss.src.middlewares.auth import auth_middleware +from oss.src.middlewares.analytics import analytics_middleware from oss.src.core.auth.supertokens.config import init_supertokens @@ -132,34 +144,48 @@ from oss.src.routers import ( user_profile, - health_router, - permissions_router, projects_router, api_key_router, organization_router, workspace_router, ) +from oss.src.apis.fastapi.access.router import AccessRouter from oss.src.utils.env import env from entrypoints.worker_evaluations import evaluations_worker -import oss.src.core.evaluations.tasks.live # noqa: F401 -import oss.src.core.evaluations.tasks.legacy # noqa: F401 -import oss.src.core.evaluations.tasks.batch # noqa: F401 +import oss.src.core.evaluations.tasks.run # noqa: F401 +import oss.src.core.evaluations.tasks.processor # noqa: F401 -import agenta as ag +print("[STARTUP] About to import agenta SDK") +_t_ag_import = time.perf_counter() +# ag already imported at top +print(f"[STARTUP] agenta SDK imported (+{time.perf_counter() - _t_ag_import:.3f}s)") + +_startup_t0 = time.perf_counter() +print("[STARTUP] imports completed, beginning initialization") +print("[STARTUP] ag.init() starting") +_t_ag_init = time.perf_counter() ag.init( api_url=env.agenta.api_url, ) +print(f"[STARTUP] ag.init() completed (+{time.perf_counter() - _t_ag_init:.3f}s)") ee = None +_t_before_ee = time.perf_counter() if is_ee(): + print("[STARTUP] EE module import starting (Stripe init happens here)") import ee.src.main as ee # type: ignore + _ee_elapsed = time.perf_counter() - _t_before_ee + print(f"[STARTUP] EE module import completed (+{_ee_elapsed:.3f}s)") + log = get_module_logger(__name__) init_supertokens() +_st_elapsed = time.perf_counter() - _startup_t0 +print(f"[STARTUP] init_supertokens completed (+{_st_elapsed:.3f}s)") @asynccontextmanager @@ -183,6 +209,10 @@ async def lifespan(*args, **kwargs): for adapter in _composio_adapters.values(): await adapter.close() + await _transactions_engine.close() + await _analytics_engine.close() + await _streams_engine.close() + _OPENAPI_TAGS = [ { @@ -326,11 +356,11 @@ async def lifespan(*args, **kwargs): app.add_middleware(SupportHeadersMiddleware) if is_ee(): - from ee.src.services.throttling_service import throttling_middleware + from ee.src.middlewares.throttling import throttling_middleware app.middleware("http")(throttling_middleware) -app.middleware("http")(authentication_middleware) +app.middleware("http")(auth_middleware) app.middleware("http")(analytics_middleware) app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=5) @@ -358,47 +388,65 @@ async def lifespan(*args, **kwargs): # DAOS ------------------------------------------------------------------------- -secrets_dao = SecretsDAO() -webhooks_dao = WebhooksDAO() +_t_daos = time.perf_counter() +print("[STARTUP] DAO initialization starting") + +# Instantiate engines at startup (lazy — they don't connect until first use) +_transactions_engine = get_transactions_engine() +_analytics_engine = get_analytics_engine() +_streams_engine = get_streams_engine() +_cache_engine = get_cache_engine() -tracing_dao = TracingDAO() -events_dao = EventsDAO() +secrets_dao = SecretsDAO(engine=_transactions_engine) +webhooks_dao = WebhooksDAO(engine=_transactions_engine) + +tracing_dao = TracingDAO(engine=_analytics_engine) +events_dao = EventsDAO(engine=_analytics_engine) testcases_dao = BlobsDAO( + engine=_transactions_engine, BlobDBE=TestcaseBlobDBE, ) testsets_dao = GitDAO( + engine=_transactions_engine, ArtifactDBE=TestsetArtifactDBE, VariantDBE=TestsetVariantDBE, RevisionDBE=TestsetRevisionDBE, ) queries_dao = GitDAO( + engine=_transactions_engine, ArtifactDBE=QueryArtifactDBE, VariantDBE=QueryVariantDBE, RevisionDBE=QueryRevisionDBE, ) workflows_dao = GitDAO( + engine=_transactions_engine, ArtifactDBE=WorkflowArtifactDBE, VariantDBE=WorkflowVariantDBE, RevisionDBE=WorkflowRevisionDBE, ) environments_dao = GitDAO( + engine=_transactions_engine, ArtifactDBE=EnvironmentArtifactDBE, VariantDBE=EnvironmentVariantDBE, RevisionDBE=EnvironmentRevisionDBE, ) -evaluations_dao = EvaluationsDAO() -folders_dao = FoldersDAO() +evaluations_dao = EvaluationsDAO(engine=_transactions_engine) +folders_dao = FoldersDAO(engine=_transactions_engine) -tools_dao = ToolsDAO() +tools_dao = ToolsDAO(engine=_transactions_engine) # SERVICES --------------------------------------------------------------------- +_t_daos_done = time.perf_counter() - _t_daos +print(f"[STARTUP] DAO initialization completed (+{_t_daos_done:.3f}s)") +_t_services = time.perf_counter() + vault_service = VaultService( secrets_dao=secrets_dao, ) @@ -504,6 +552,11 @@ async def lifespan(*args, **kwargs): testsets_service=testsets_service, evaluators_service=evaluators_service, evaluations_worker=evaluations_worker, + # Sub-services the tensor slice processor needs; passing them lets the + # service build its own TensorSliceOperations (probe/populate/process). + testcases_service=testcases_service, + workflows_service=workflows_service, + applications_service=applications_service, ) simple_evaluations_service = SimpleEvaluationsService( @@ -540,6 +593,10 @@ async def lifespan(*args, **kwargs): adapter_registry=tools_adapter_registry, ) +_t_services_done = time.perf_counter() - _t_services +print(f"[STARTUP] Service initialization completed (+{_t_services_done:.3f}s)") +_t_routers = time.perf_counter() + # ROUTERS ---------------------------------------------------------------------- secrets = VaultRouter( @@ -688,6 +745,8 @@ async def lifespan(*args, **kwargs): # MOUNTING ROUTERS TO APP ROUTES ----------------------------------------------- +_t_mount_routers = time.perf_counter() + app.include_router( router=secrets.router, tags=["Secrets"], @@ -1067,15 +1126,16 @@ async def lifespan(*args, **kwargs): tags=["Admin"], ) -app.include_router( - health_router.router, - prefix="/health", - tags=["Status"], -) +@app.get("/health", operation_id="health_check", tags=["Status"]) +async def health_check(): + return {"status": "ok"} + + +access_router = AccessRouter() app.include_router( - permissions_router.router, - prefix="/permissions", + access_router.router, + prefix="/access", tags=["Access"], ) @@ -1109,5 +1169,14 @@ async def lifespan(*args, **kwargs): ) # ------------------------------------------------------------------------------ +_t_routers_done = time.perf_counter() - _t_routers +print(f"[STARTUP] Router initialization completed (+{_t_routers_done:.3f}s)") + +_t_mount_routers_done = time.perf_counter() - _t_mount_routers +print(f"[STARTUP] Router mounting completed (+{_t_mount_routers_done:.3f}s)") + if ee and is_ee(): app = ee.extend_app_schema(app) + +_total_startup = time.perf_counter() - _startup_t0 +print(f"[STARTUP] module initialization completed in {_total_startup:.3f}s") diff --git a/api/entrypoints/worker_evaluations.py b/api/entrypoints/worker_evaluations.py index 8624da4a62..99d0b8ea14 100644 --- a/api/entrypoints/worker_evaluations.py +++ b/api/entrypoints/worker_evaluations.py @@ -15,7 +15,7 @@ # Guard EE imports — see worker_tracing.py for the rationale. if is_ee(): - from ee.src.utils.entitlements import bootstrap_entitlements_services + from ee.src.core.access.entitlements.service import bootstrap_entitlements_services from oss.src.core.evaluations.runtime.locks import run_worker_heartbeat diff --git a/api/entrypoints/worker_events.py b/api/entrypoints/worker_events.py index 76966657e9..ac65d3246b 100644 --- a/api/entrypoints/worker_events.py +++ b/api/entrypoints/worker_events.py @@ -19,7 +19,7 @@ # Guard EE imports — see worker_tracing.py for the rationale. if is_ee(): - from ee.src.utils.entitlements import bootstrap_entitlements_services + from ee.src.core.access.entitlements.service import bootstrap_entitlements_services log = get_module_logger(__name__) diff --git a/api/entrypoints/worker_tracing.py b/api/entrypoints/worker_tracing.py index e87f532bd9..c4c2ef936f 100644 --- a/api/entrypoints/worker_tracing.py +++ b/api/entrypoints/worker_tracing.py @@ -44,7 +44,7 @@ # the `ee.*` package to be importable. The matching `is_ee()` branch in # `main_async` calls `bootstrap_entitlements_services()`. if is_ee(): - from ee.src.utils.entitlements import bootstrap_entitlements_services + from ee.src.core.access.entitlements.service import bootstrap_entitlements_services log = get_module_logger(__name__) diff --git a/api/entrypoints/worker_webhooks.py b/api/entrypoints/worker_webhooks.py index 33a56d98ba..0419429db4 100644 --- a/api/entrypoints/worker_webhooks.py +++ b/api/entrypoints/worker_webhooks.py @@ -14,7 +14,7 @@ # Guard EE imports — see worker_tracing.py for the rationale. if is_ee(): - from ee.src.utils.entitlements import bootstrap_entitlements_services + from ee.src.core.access.entitlements.service import bootstrap_entitlements_services import agenta as ag diff --git a/api/oss/databases/postgres/migrations/core/versions/a1d2e3f4a5b6_add_default_evaluation_queues.py b/api/oss/databases/postgres/migrations/core/versions/a1d2e3f4a5b6_add_default_evaluation_queues.py new file mode 100644 index 0000000000..6256e9b898 --- /dev/null +++ b/api/oss/databases/postgres/migrations/core/versions/a1d2e3f4a5b6_add_default_evaluation_queues.py @@ -0,0 +1,39 @@ +"""add default evaluation queues + +Revision ID: a1d2e3f4a5b6 +Revises: f7a8b9c0d1e2 +Create Date: 2026-05-15 00:00:00 + +Previously shared revision id `a1b2c3d4e5f6` with +`drop_corrupted_metrics_for_some_runs`, so alembic skipped it and the index +below never ran. Renamed to `a1d2e3f4a5b6` and chained after main's head +`e6f7a8b9c0d1` to keep the branch a single linear chain. + +The partial unique index covers ALL default queues (active or archived), so +there is at most ONE default queue row per (project_id, run_id) for the lifetime +of the run. Archiving a default does NOT free the slot — the single row is +archived/unarchived in place by reconcile, and user-facing archive of a default +is forbidden in the service layer. +""" + +from typing import Sequence, Union + +from alembic import op + +revision: str = "a1d2e3f4a5b6" +down_revision: Union[str, None] = "f7a8b9c0d1e2" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("DROP INDEX IF EXISTS ux_evaluation_queues_default_per_run") + op.execute(""" + CREATE UNIQUE INDEX ux_evaluation_queues_default_per_run + ON evaluation_queues (project_id, run_id) + WHERE (flags ->> 'is_default')::boolean = true + """) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ux_evaluation_queues_default_per_run") diff --git a/api/oss/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py b/api/oss/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py new file mode 100644 index 0000000000..30da5d2a86 --- /dev/null +++ b/api/oss/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py @@ -0,0 +1,294 @@ +"""backfill default evaluation queues + +Revision ID: a2b3c4d5e6f8 +Revises: a1d2e3f4a5b6 +Create Date: 2026-05-15 00:10:00 + +Backfills source-family flags (`has_traces` / `has_testcases` / `has_queries` / +`has_testsets`) to match the runtime derivation rule, then mass-creates default +queues per the runtime policy and recomputes `is_queue`. The query/testset +recompute keys on exact reference-key presence, not a substring match. + +Performance shape: runs are processed in keyset-paginated chunks (by +`evaluation_runs.id`). For each chunk a single COMPUTE select does the heavy +`jsonb_array_elements` step scan and the per-run default-queue existence check +ONCE (no correlated `EXISTS`/`JOIN` re-evaluated per written row). Python then +decides per run what to do, and the writes are cheap set-based `unnest` updates +keyed by id — no subqueries in the mutation path. Every step is idempotent (the +flag rebuild is deterministic; runs that already have a default queue are not +re-created), so a re-run is safe. +""" + +import json +from typing import List, Sequence, Union + +import click +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision: str = "a2b3c4d5e6f8" +down_revision: Union[str, None] = "a1d2e3f4a5b6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +BATCH_SIZE = 100 + + +# Keyset pagination over run ids. id > cursor with ORDER BY id is index-friendly +# (id is part of the PK) and stable; LIMIT bounds each chunk. Compared as text so +# the cursor is a plain string seeded as "" below every UUID. +_NEXT_RUN_IDS = sa.text(""" + SELECT id::text + FROM evaluation_runs + WHERE id::text > :cursor + ORDER BY id::text + LIMIT :batch +""") + +# One heavy pass per chunk. Derives the source-family flags from the run's steps +# (the `jsonb_array_elements` scan that mirrors dbs/postgres/evaluations/utils.py) +# and, via a single LEFT JOIN over the chunk's runs, reports whether each run +# already has a default queue (any state) and an ACTIVE default queue. Direct +# sources (`has_traces`/`has_testcases`) come from the exact step key on a +# reference-less input; reference-backed sources (`has_queries`/`has_testsets`) +# from exact-key presence (JSONB `?`), not a substring match that would misfire +# on `query_anchor` / `testset_metadata`. +_COMPUTE_CHUNK = sa.text(""" + SELECT + r.id::text AS run_id, + r.project_id::text AS project_id, + r.created_by_id::text AS created_by_id, + COALESCE(r.status, 'running') AS status, + COALESCE((r.flags ->> 'has_human')::boolean, false) AS has_human, + EXISTS ( + SELECT 1 + FROM jsonb_array_elements(COALESCE(r.data::jsonb -> 'steps', '[]'::jsonb)) AS step + WHERE step ->> 'type' = 'input' + AND COALESCE(step -> 'references', '{}'::jsonb) = '{}'::jsonb + AND lower(COALESCE(step ->> 'key', '')) IN ('traces', 'query-direct') + ) AS has_traces, + EXISTS ( + SELECT 1 + FROM jsonb_array_elements(COALESCE(r.data::jsonb -> 'steps', '[]'::jsonb)) AS step + WHERE step ->> 'type' = 'input' + AND COALESCE(step -> 'references', '{}'::jsonb) = '{}'::jsonb + AND lower(COALESCE(step ->> 'key', '')) IN ('testcases', 'testset-direct') + ) AS has_testcases, + EXISTS ( + SELECT 1 + FROM jsonb_array_elements(COALESCE(r.data::jsonb -> 'steps', '[]'::jsonb)) AS step + WHERE step ->> 'type' = 'input' + AND COALESCE(step -> 'references', '{}'::jsonb) ? 'query_revision' + ) AS has_queries, + EXISTS ( + SELECT 1 + FROM jsonb_array_elements(COALESCE(r.data::jsonb -> 'steps', '[]'::jsonb)) AS step + WHERE step ->> 'type' = 'input' + AND COALESCE(step -> 'references', '{}'::jsonb) ? 'testset_revision' + ) AS has_testsets, + bool_or((q.flags ->> 'is_default')::boolean) AS has_any_default, + bool_or( + (q.flags ->> 'is_default')::boolean AND q.deleted_at IS NULL + ) AS has_active_default + FROM evaluation_runs r + LEFT JOIN evaluation_queues q + ON q.project_id = r.project_id + AND q.run_id = r.id + WHERE r.id = ANY(CAST(:ids AS uuid[])) + GROUP BY r.id, r.project_id, r.created_by_id, r.status, r.flags +""") + +# Cheap set-based flag write: merge the precomputed flag object onto each run by +# id. No subqueries; `unnest` pairs each id with its jsonb patch. +_APPLY_FLAGS = sa.text(""" + UPDATE evaluation_runs r + SET flags = COALESCE(r.flags, '{}'::jsonb) || patch.flags::jsonb + FROM unnest( + CAST(:ids AS uuid[]), + CAST(:flags AS jsonb[]) + ) AS patch(id, flags) + WHERE r.id = patch.id +""") + +# Cheap create: one row per run id that needs a default queue (computed in +# Python — has_human and no existing default). No NOT EXISTS in the write path. +_CREATE_DEFAULT_QUEUES = sa.text(""" + INSERT INTO evaluation_queues ( + project_id, + id, + created_at, + created_by_id, + flags, + data, + status, + run_id + ) + SELECT + CAST(src.project_id AS uuid), + gen_random_uuid(), + CURRENT_TIMESTAMP, + CAST(src.created_by_id AS uuid), + jsonb_build_object('is_default', true, 'is_sequential', false), + '{}'::json, + src.status, + CAST(src.run_id AS uuid) + FROM unnest( + CAST(:project_ids AS text[]), + CAST(:run_ids AS text[]), + CAST(:created_by_ids AS text[]), + CAST(:statuses AS text[]) + ) AS src(project_id, run_id, created_by_id, status) +""") + +# Cheap archive: flip deleted_at on the active default queue of each run id that +# should no longer have one (computed in Python). Keyed by (project_id, run_id). +_ARCHIVE_STALE_DEFAULT_QUEUES = sa.text(""" + UPDATE evaluation_queues q + SET deleted_at = CURRENT_TIMESTAMP, + deleted_by_id = CAST(src.created_by_id AS uuid) + FROM unnest( + CAST(:project_ids AS text[]), + CAST(:run_ids AS text[]), + CAST(:created_by_ids AS text[]) + ) AS src(project_id, run_id, created_by_id) + WHERE q.project_id = CAST(src.project_id AS uuid) + AND q.run_id = CAST(src.run_id AS uuid) + AND (q.flags ->> 'is_default')::boolean = true + AND q.deleted_at IS NULL +""") + + +def _next_run_ids(connection: Connection, *, cursor: str) -> List[str]: + result = connection.execute( + _NEXT_RUN_IDS, + {"cursor": cursor, "batch": BATCH_SIZE}, + ) + return [row[0] for row in result.fetchall()] + + +def _flags_patch(row) -> str: + # Build the jsonb patch for a run: the four source-family flags plus the + # recomputed is_queue, matching the queue state this chunk produces. + # + # is_queue is true iff the run has a human step AND ends this chunk with an + # ACTIVE default queue. Mirroring _reconcile_default_queue and the create + # guard below: + # - human + already-active default -> active (is_queue=True) + # - human + no default at all -> created (is_queue=True) + # - human + only an ARCHIVED default -> not unarchived here -> False + # (same as the prior migration) + # - non-human + active default -> archived (is_queue=False) + # - non-human -> no active default (False) + if row.has_human: + ends_with_active_default = row.has_active_default or not row.has_any_default + else: + ends_with_active_default = False + + return json.dumps( + { + "has_traces": bool(row.has_traces), + "has_testcases": bool(row.has_testcases), + "has_queries": bool(row.has_queries), + "has_testsets": bool(row.has_testsets), + "is_queue": bool(ends_with_active_default), + } + ) + + +def upgrade() -> None: + connection = op.get_bind() + + cursor = "" + processed = 0 + created = 0 + archived = 0 + + while True: + ids = _next_run_ids(connection, cursor=cursor) + if not ids: + break + + rows = connection.execute(_COMPUTE_CHUNK, {"ids": ids}).fetchall() + + flag_ids: List[str] = [] + flag_patches: List[str] = [] + + create_projects: List[str] = [] + create_runs: List[str] = [] + create_creators: List[str] = [] + create_statuses: List[str] = [] + + archive_projects: List[str] = [] + archive_runs: List[str] = [] + archive_creators: List[str] = [] + + for row in rows: + flag_ids.append(row.run_id) + flag_patches.append(_flags_patch(row)) + + # Mirror _reconcile_default_queue: has_human runs should have a + # default; create one only when none exists yet. + if row.has_human and not row.has_any_default: + create_projects.append(row.project_id) + create_runs.append(row.run_id) + create_creators.append(row.created_by_id) + create_statuses.append(row.status) + + # Non-human runs with a stale active default get it archived. + if not row.has_human and row.has_active_default: + archive_projects.append(row.project_id) + archive_runs.append(row.run_id) + archive_creators.append(row.created_by_id) + + # Create/archive first so the flag write's is_queue (computed in Python + # from the same decisions) matches the resulting queue state. + if create_runs: + connection.execute( + _CREATE_DEFAULT_QUEUES, + { + "project_ids": create_projects, + "run_ids": create_runs, + "created_by_ids": create_creators, + "statuses": create_statuses, + }, + ) + created += len(create_runs) + + if archive_runs: + connection.execute( + _ARCHIVE_STALE_DEFAULT_QUEUES, + { + "project_ids": archive_projects, + "run_ids": archive_runs, + "created_by_ids": archive_creators, + }, + ) + archived += len(archive_runs) + + connection.execute( + _APPLY_FLAGS, + {"ids": flag_ids, "flags": flag_patches}, + ) + + processed += len(ids) + cursor = ids[-1] + + click.echo( + f"[default-queue backfill] processed_runs={processed} " + f"batch={len(ids)} created={created} archived={archived}" + ) + + click.echo( + f"[default-queue backfill] done processed_runs={processed} " + f"created={created} archived={archived}" + ) + + +def downgrade() -> None: + # Keep generated queues/results intact on downgrade. Remove only the newly + # inferred flags; old is_queue semantics cannot be reconstructed safely. + op.execute(""" + UPDATE evaluation_runs + SET flags = COALESCE(flags, '{}'::jsonb) - 'has_traces' - 'has_testcases' + """) diff --git a/api/oss/src/apis/fastapi/access/__init__.py b/api/oss/src/apis/fastapi/access/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/oss/src/apis/fastapi/access/router.py b/api/oss/src/apis/fastapi/access/router.py new file mode 100644 index 0000000000..a383aef158 --- /dev/null +++ b/api/oss/src/apis/fastapi/access/router.py @@ -0,0 +1,234 @@ +from typing import Optional, Union +from uuid import UUID + +from fastapi import APIRouter, Query, HTTPException +from fastapi.responses import JSONResponse + +from oss.src.utils.logging import get_module_logger +from oss.src.utils.caching import get_cache, set_cache +from oss.src.utils.context import get_auth_context, get_auth_scope +from oss.src.utils.common import is_ee, is_oss + +if is_ee(): + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import check_action_access + from ee.src.core.access.entitlements.service import check_entitlements, Counter + + +log = get_module_logger(__name__) + + +class Allow(JSONResponse): + def __init__( + self, + credentials: Optional[str] = None, + ) -> None: + super().__init__( + status_code=200, + content={ + "effect": "allow", + "credentials": credentials, + }, + ) + + +class Deny(HTTPException): + def __init__(self) -> None: + super().__init__( + status_code=403, + detail="Forbidden", + ) + + +async def _check_scope_access( + scope_type: Optional[str] = None, + scope_id: Optional[UUID] = None, +) -> bool: + auth_scope = get_auth_scope() + + allow_scope = False + + if scope_type == "project": + allow_scope = str(auth_scope.project_id) == str(scope_id) + elif scope_type == "workspace": + allow_scope = str(auth_scope.workspace_id) == str(scope_id) + elif not scope_type and not scope_id: + allow_scope = True + + return allow_scope + + +async def _check_resource_access( + resource_type: Optional[str] = None, +) -> Union[bool, int]: + allow_resource = False + + if resource_type == "service": + allow_resource = True + + if resource_type == "local_secrets": + check, meter, _ = await check_entitlements( # type: ignore + key=Counter.CREDITS_CONSUMED, # type: ignore + delta=1, + ) + + if not check: + return False + + if not meter or not meter.value: + return False + + return meter.value + + return allow_resource + + +class AccessRouter: + def __init__(self) -> None: + self.router = APIRouter() + + self.router.add_api_route( + "/permissions/check", + self.check_permissions, + methods=["GET"], + operation_id="check_permissions", + ) + + async def check_permissions( + self, + action: Optional[str] = Query(None), + scope_type: Optional[str] = Query(None), + scope_id: Optional[UUID] = Query(None), + resource_type: Optional[str] = Query(None), + resource_id: Optional[UUID] = Query(None), + ): + ctx = get_auth_context() + project_id = str(ctx.scope.project_id) + user_id = str(ctx.scope.user_id) + credentials_header = ctx.credentials.header()[1] + + cache_key = { + "action": action, + "scope_type": scope_type, + "scope_id": scope_id, + "resource_type": resource_type, + "resource_id": resource_id, + } + + try: + if is_oss(): + return Allow(credentials_header) + + if not action or not resource_type: + log.warn("Missing required parameters: action, resource_type") + raise Deny() + + allow = await get_cache( + project_id=project_id, + user_id=user_id, + namespace="check_permissions", + key=cache_key, + ) + + if allow == "allow": + return Allow(credentials_header) + if allow == "deny": + log.warn("Permission denied") + raise Deny() + + # CHECK PERMISSION 1/3: SCOPE + allow_scope = await _check_scope_access( + scope_type=scope_type, + scope_id=scope_id, + ) + + if not allow_scope: + log.warn("Scope access denied") + await set_cache( + project_id=project_id, + user_id=user_id, + namespace="check_permissions", + key=cache_key, + value="deny", + ) + raise Deny() + + # CHECK PERMISSION 2/3: ACTION + allow_action = await check_action_access( + project_id=project_id, + user_uid=user_id, + permission=Permission(action), + ) + + if not allow_action: + log.warn("Action access denied") + await set_cache( + project_id=project_id, + user_id=user_id, + namespace="check_permissions", + key=cache_key, + value="deny", + ) + raise Deny() + + # CHECK PERMISSION 3/3: RESOURCE + allow_resource = await _check_resource_access( + resource_type=resource_type, + ) + + if isinstance(allow_resource, bool): + if allow_resource is False: + log.warn("Resource access denied") + await set_cache( + project_id=project_id, + user_id=user_id, + namespace="check_permissions", + key=cache_key, + value="deny", + ) + raise Deny() + + if allow_resource is True: + await set_cache( + project_id=project_id, + user_id=user_id, + namespace="check_permissions", + key=cache_key, + value="allow", + ) + return Allow(credentials_header) + + elif isinstance(allow_resource, int): + if allow_resource <= 0: + log.warn("Resource access denied") + await set_cache( + project_id=project_id, + user_id=user_id, + namespace="check_permissions", + key=cache_key, + value="deny", + ) + raise Deny() + else: + return Allow(credentials_header) + + log.warn("Resource access denied") + await set_cache( + project_id=project_id, + user_id=user_id, + namespace="check_permissions", + key=cache_key, + value="deny", + ) + raise Deny() + + except Exception as exc: # pylint: disable=broad-except + log.warn(exc) + await set_cache( + project_id=project_id, + user_id=user_id, + namespace="check_permissions", + key=cache_key, + value="deny", + ) + raise Deny() from exc diff --git a/api/oss/src/apis/fastapi/annotations/router.py b/api/oss/src/apis/fastapi/annotations/router.py index 32a2b7821a..e6afa8e90a 100644 --- a/api/oss/src/apis/fastapi/annotations/router.py +++ b/api/oss/src/apis/fastapi/annotations/router.py @@ -24,8 +24,11 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/apis/fastapi/applications/router.py b/api/oss/src/apis/fastapi/applications/router.py index 9d94a64ed4..c02f2e1ff6 100644 --- a/api/oss/src/apis/fastapi/applications/router.py +++ b/api/oss/src/apis/fastapi/applications/router.py @@ -88,23 +88,14 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) -# TEMPORARY: Disabling name editing -RENAME_APPS_DISABLED_MESSAGE = "Renaming applications is temporarily disabled." - - -def _build_rename_apps_disabled_detail(*, existing_name: Optional[str]) -> str: - if existing_name: - return ( - f"{RENAME_APPS_DISABLED_MESSAGE} " - f"Current application name is '{existing_name}'." - ) - - return RENAME_APPS_DISABLED_MESSAGE class ApplicationsRouter: @@ -670,27 +661,6 @@ async def edit_application( detail="Application ID in path does not match application ID in request body.", ) - # TEMPORARY: Disabling name editing - existing_application = await self.applications_service.fetch_application( - project_id=UUID(request.state.project_id), - application_ref=Reference(id=application_id), - ) - if existing_application is None: - return ApplicationResponse() - - edit_model = application_edit_request.application - if ( - "name" in edit_model.model_fields_set - and edit_model.name is not None - and edit_model.name != existing_application.name - ): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=_build_rename_apps_disabled_detail( - existing_name=existing_application.name - ), - ) - application = await self.applications_service.edit_application( project_id=UUID(request.state.project_id), user_id=UUID(request.state.user_id), @@ -2014,26 +1984,6 @@ async def edit_simple_application( ) # TEMPORARY: Disabling name editing - existing_application = await self.simple_applications_service.applications_service.fetch_application( - project_id=UUID(request.state.project_id), - application_ref=Reference(id=application_id), - ) - if existing_application is None: - return SimpleApplicationResponse() - - edit_model = simple_application_edit_request.application - if ( - "name" in edit_model.model_fields_set - and edit_model.name is not None - and edit_model.name != existing_application.name - ): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=_build_rename_apps_disabled_detail( - existing_name=existing_application.name - ), - ) - simple_application = await self.simple_applications_service.edit( project_id=UUID(request.state.project_id), user_id=UUID(request.state.user_id), diff --git a/api/oss/src/apis/fastapi/environments/router.py b/api/oss/src/apis/fastapi/environments/router.py index 222424dcf9..457dda4c8b 100644 --- a/api/oss/src/apis/fastapi/environments/router.py +++ b/api/oss/src/apis/fastapi/environments/router.py @@ -74,8 +74,11 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/apis/fastapi/environments/utils.py b/api/oss/src/apis/fastapi/environments/utils.py index 3dc4c403a4..b834270864 100644 --- a/api/oss/src/apis/fastapi/environments/utils.py +++ b/api/oss/src/apis/fastapi/environments/utils.py @@ -36,8 +36,11 @@ if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) async def ensure_environment_deploy_allowed( diff --git a/api/oss/src/apis/fastapi/evaluations/models.py b/api/oss/src/apis/fastapi/evaluations/models.py index 8e87909942..86d8348657 100644 --- a/api/oss/src/apis/fastapi/evaluations/models.py +++ b/api/oss/src/apis/fastapi/evaluations/models.py @@ -1,5 +1,6 @@ from typing import Optional, List from uuid import UUID +from datetime import datetime from pydantic import BaseModel, model_validator from fastapi import HTTPException @@ -12,6 +13,7 @@ EvaluationRunCreate, EvaluationRunEdit, EvaluationRunQuery, + EvaluationRunDataStep, # EvaluationScenario, EvaluationScenarioCreate, @@ -20,12 +22,10 @@ # EvaluationResult, EvaluationResultCreate, - EvaluationResultEdit, EvaluationResultQuery, # EvaluationMetrics, EvaluationMetricsCreate, - EvaluationMetricsEdit, EvaluationMetricsQuery, EvaluationMetricsRefresh, # @@ -81,6 +81,51 @@ def __init__( self.queue_id = queue_id +class DefaultQueueDataInvalidException(HTTPException): + """Raised when a default queue carries scenario/step/assignment/batch filters.""" + + def __init__(self, message: str, queue_id: Optional[UUID] = None): + details = dict(message=message) + if queue_id: + details["queue_id"] = str(queue_id) + super().__init__(status_code=422, detail=details) + self.queue_id = queue_id + + +class EvaluationMetricsInvalidException(HTTPException): + """Raised when a metric does not match a valid global/variational/temporal shape.""" + + def __init__( + self, + message: str, + run_id: Optional[UUID] = None, + scenario_id: Optional[UUID] = None, + timestamp: Optional[datetime] = None, + ): + details = dict(message=message) + if run_id: + details["run_id"] = str(run_id) + if scenario_id: + details["scenario_id"] = str(scenario_id) + if timestamp: + details["timestamp"] = str(timestamp) + super().__init__(status_code=422, detail=details) + self.run_id = run_id + self.scenario_id = scenario_id + self.timestamp = timestamp + + +class DefaultQueueEditingForbiddenException(HTTPException): + """Raised when demoting or hard-deleting a default queue is attempted.""" + + def __init__(self, message: str, queue_id: Optional[UUID] = None): + details = dict(message=message) + if queue_id: + details["queue_id"] = str(queue_id) + super().__init__(status_code=409, detail=details) + self.queue_id = queue_id + + # EVALUATION RUNS -------------------------------------------------------------- @@ -177,18 +222,10 @@ class EvaluationScenarioIdsResponse(BaseModel): # - EVALUATION RESULTS --------------------------------------------------------- -class EvaluationResultsCreateRequest(BaseModel): +class EvaluationResultsSetRequest(BaseModel): results: List[EvaluationResultCreate] -class EvaluationResultEditRequest(BaseModel): - result: EvaluationResultEdit - - -class EvaluationResultsEditRequest(BaseModel): - results: List[EvaluationResultEdit] - - class EvaluationResultQueryRequest(BaseModel): result: Optional[EvaluationResultQuery] = None # @@ -219,15 +256,95 @@ class EvaluationResultIdsResponse(BaseModel): result_ids: List[UUID] = [] -# - EVALUATION METRICS --------------------------------------------------------- +# - EVALUATION TENSOR SLICE ---------------------------------------------------- -class EvaluationMetricsCreateRequest(BaseModel): - metrics: List[EvaluationMetricsCreate] +class TensorSliceRequest(BaseModel): + # Coordinate projection over EXISTING scenarios. Any omitted dimension is + # "all" for that axis; an explicit empty list means "none addressed". + scenario_ids: Optional[List[UUID]] = None + step_keys: Optional[List[str]] = None + repeat_idxs: Optional[List[int]] = None + + +class PopulateSliceRequest(EvaluationResultsSetRequest): + # Write-side slice op: same `results` payload as the generic results setter, + # but the populate endpoint owns its own request type so its run-scoped + # validation and OpenAPI surface can diverge from `set_results`. + pass + + +class ProcessSliceRequest(TensorSliceRequest): + # `overwrite=False` (default) runs only cells without a result (fill-missing); + # `overwrite=True` re-runs every addressed cell (force). + overwrite: bool = False + + +class ProbeSliceRequest(TensorSliceRequest): + # Read-side slice op: coordinate projection over existing cells. Its own + # request type keeps the probe OpenAPI surface independent of the other + # tensor ops. + pass + + +class PruneSliceRequest(TensorSliceRequest): + # Delete-side slice op: clears the addressed cells (and refreshes metrics). + # Owns its request type for the same reason. + pass + + +class RefreshSliceRequest(TensorSliceRequest): + # Metrics-side slice op: recompute the metric rows (variational + aggregate) + # over the addressed scope without writing or executing cells. Owns its + # request type for the same reason as the others. + pass + + +# `process` returns 202 (async dispatch acknowledged) and `prune`/`refresh` +# return 204 — none carries a body, so there is no response model for them. + +# - EVALUATION GRAPH-SHAPE OPS ------------------------------------------------- +# +# Reshape the tensor (scenarios x steps x repeats). Paired with the tensor ops +# above: shape ops add/remove coordinates, tensor ops fill/clear cells within a +# shape. One request model per axis op; responses reuse the scenario/run/result +# envelopes already defined above. -class EvaluationMetricsEditRequest(BaseModel): - metrics: List[EvaluationMetricsEdit] + +class AddScenariosRequest(BaseModel): + # Height: append `count` empty scenario rows; `populate`/`process` fill them. + count: int + # Optional temporal bucket. Floored to the minute server-side (interval is + # fixed at 1 minute); omit to leave the new scenarios off the temporal axis. + timestamp: Optional[datetime] = None + + +class RemoveScenariosRequest(BaseModel): + # Height: drop these scenario rows (and all their cells). + scenario_ids: List[UUID] + + +class AddStepsRequest(BaseModel): + # Width: append graph step columns (idempotent on key). + steps: List[EvaluationRunDataStep] + + +class RemoveStepsRequest(BaseModel): + # Width: drop graph step columns by key. + step_keys: List[str] + + +class SetRepeatsRequest(BaseModel): + # Depth: set the run's repeat count. + repeats: int + + +# - EVALUATION METRICS --------------------------------------------------------- + + +class EvaluationMetricsSetRequest(BaseModel): + metrics: List[EvaluationMetricsCreate] class EvaluationMetricsQueryRequest(BaseModel): diff --git a/api/oss/src/apis/fastapi/evaluations/router.py b/api/oss/src/apis/fastapi/evaluations/router.py index 3410b8d852..0004f3ad6f 100644 --- a/api/oss/src/apis/fastapi/evaluations/router.py +++ b/api/oss/src/apis/fastapi/evaluations/router.py @@ -46,18 +46,27 @@ EvaluationScenarioIdResponse, EvaluationScenarioIdsResponse, # EVALUATION RESULTS - EvaluationResultsCreateRequest, - EvaluationResultEditRequest, - EvaluationResultsEditRequest, + EvaluationResultsSetRequest, EvaluationResultQueryRequest, EvaluationResultIdsRequest, EvaluationResultResponse, EvaluationResultsResponse, EvaluationResultIdResponse, EvaluationResultIdsResponse, + # EVALUATION TENSOR SLICE + ProcessSliceRequest, + PopulateSliceRequest, + ProbeSliceRequest, + PruneSliceRequest, + RefreshSliceRequest, + # EVALUATION GRAPH-SHAPE OPS + AddScenariosRequest, + RemoveScenariosRequest, + AddStepsRequest, + RemoveStepsRequest, + SetRepeatsRequest, # EVALUATION METRICS - EvaluationMetricsCreateRequest, - EvaluationMetricsEditRequest, + EvaluationMetricsSetRequest, EvaluationMetricsQueryRequest, EvaluationMetricsIdsRequest, EvaluationMetricsResponse, @@ -99,12 +108,16 @@ from oss.src.core.evaluations.types import ( SimpleQueueScenariosQuery, EvaluationQueueScenariosQuery, + EvaluationScenarioQuery, ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION - from ee.src.utils.entitlements import check_entitlements, Counter + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) + from ee.src.core.access.entitlements.service import check_entitlements, Counter log = get_module_logger(__name__) @@ -124,7 +137,7 @@ def __init__( self.admin_router = APIRouter() - # EVALUATION RUNS ------------------------------------------------------ + # ADMIN EVALUATION REFRESH --------------------------------------------- # POST /api/evaluations/runs/refresh self.admin_router.add_api_route( @@ -134,6 +147,8 @@ def __init__( response_model_exclude_none=True, ) + # EVALUATION RUNS ------------------------------------------------------ + # POST /api/evaluations/runs/ self.router.add_api_route( path="/runs/", @@ -234,16 +249,6 @@ def __init__( operation_id="close_run", ) - # POST /api/evaluations/runs/{run_id}/close/{status} - self.router.add_api_route( - path="/runs/{run_id}/close/{status}", - methods=["POST"], - endpoint=self.close_run, - response_model=EvaluationRunResponse, - response_model_exclude_none=True, - operation_id="close_run_with_status", - ) - # POST /api/evaluations/runs/{run_id}/open self.router.add_api_route( path="/runs/{run_id}/open", @@ -254,6 +259,16 @@ def __init__( operation_id="open_run", ) + # GET /api/evaluations/runs/{run_id}/queues/default + self.router.add_api_route( + path="/runs/{run_id}/queues/default", + methods=["GET"], + endpoint=self.fetch_default_queue, + response_model=EvaluationQueueResponse, + response_model_exclude_none=True, + operation_id="fetch_default_queue", + ) + # EVALUATION SCENARIOS ------------------------------------------------- # POST /api/evaluations/scenarios/ @@ -332,20 +347,10 @@ def __init__( self.router.add_api_route( path="/results/", methods=["POST"], - endpoint=self.create_results, - response_model=EvaluationResultsResponse, - response_model_exclude_none=True, - operation_id="create_results", - ) - - # PATCH /api/evaluations/results/ - self.router.add_api_route( - path="/results/", - methods=["PATCH"], - endpoint=self.edit_results, + endpoint=self.set_results, response_model=EvaluationResultsResponse, response_model_exclude_none=True, - operation_id="edit_results", + operation_id="set_results", ) # DELETE /api/evaluations/results/ @@ -378,16 +383,6 @@ def __init__( operation_id="fetch_result", ) - # PATCH /api/evaluations/results/{result_id} - self.router.add_api_route( - path="/results/{result_id}", - methods=["PATCH"], - endpoint=self.edit_result, - response_model=EvaluationResultResponse, - response_model_exclude_none=True, - operation_id="edit_result", - ) - # DELETE /api/evaluations/results/{result_id} self.router.add_api_route( path="/results/{result_id}", @@ -410,24 +405,15 @@ def __init__( operation_id="refresh_metrics", ) + # TODO: deprecate once web uses /mretrics/refresh # POST /api/evaluations/metrics/ self.router.add_api_route( path="/metrics/", methods=["POST"], - endpoint=self.create_metrics, - response_model=EvaluationMetricsResponse, - response_model_exclude_none=True, - operation_id="create_metrics", - ) - - # PATCH /api/evaluations/metrics/ - self.router.add_api_route( - path="/metrics/", - methods=["PATCH"], - endpoint=self.edit_metrics, + endpoint=self.set_metrics, response_model=EvaluationMetricsResponse, response_model_exclude_none=True, - operation_id="edit_metrics", + operation_id="set_metrics", ) # DELETE /api/evaluations/metrics/ @@ -450,6 +436,26 @@ def __init__( operation_id="query_metrics", ) + # GET /api/evaluations/metrics/{metrics_id} + self.router.add_api_route( + path="/metrics/{metrics_id}", + methods=["GET"], + endpoint=self.fetch_metric, + response_model=EvaluationMetricsResponse, + response_model_exclude_none=True, + operation_id="fetch_metric", + ) + + # DELETE /api/evaluations/metrics/{metrics_id} + self.router.add_api_route( + path="/metrics/{metrics_id}", + methods=["DELETE"], + endpoint=self.delete_metric, + response_model=EvaluationMetricsIdsResponse, + response_model_exclude_none=True, + operation_id="delete_metric", + ) + # EVALUATION QUEUES ---------------------------------------------------- # POST /api/evaluations/queues/ @@ -627,6 +633,7 @@ async def create_runs( # PATCH /evaluations/runs/ @intercept_exceptions() + @handle_evaluation_closed_exception() async def edit_runs( self, request: Request, @@ -813,8 +820,32 @@ async def fetch_run( return run_response + # GET /evaluations/runs/{run_id}/queues/default + @intercept_exceptions() + @suppress_exceptions(default=EvaluationQueueResponse(), exclude=[HTTPException]) + async def fetch_default_queue( + self, + request: Request, + *, + run_id: UUID, + ) -> EvaluationQueueResponse: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_EVALUATION_QUEUES, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + queue = await self.evaluations_service.fetch_default_queue( + project_id=UUID(request.state.project_id), + run_id=run_id, + ) + return EvaluationQueueResponse(count=1 if queue else 0, queue=queue) + # PATCH /evaluations/runs/{run_id} @intercept_exceptions() + @handle_evaluation_closed_exception() async def edit_run( self, request: Request, @@ -878,7 +909,7 @@ async def delete_run( return run_id_response # POST /evaluations/runs/{run_id}/close - # POST /evaluations/runs/{run_id}/close/{status} + # POST /evaluations/runs/{run_id}/close?status= @intercept_exceptions() async def close_run( self, @@ -886,7 +917,7 @@ async def close_run( *, run_id: UUID, # - status: Optional[EvaluationStatus] = None, + status: Optional[EvaluationStatus] = Query(None), ) -> EvaluationRunResponse: if is_ee(): if not await check_action_access( # type: ignore @@ -1168,42 +1199,11 @@ async def delete_scenario( # POST /evaluations/results/ @intercept_exceptions() @handle_evaluation_closed_exception() - async def create_results( - self, - request: Request, - *, - results_create_request: EvaluationResultsCreateRequest, - ) -> EvaluationResultsResponse: - if is_ee(): - if not await check_action_access( # type: ignore - user_uid=request.state.user_id, - project_id=request.state.project_id, - permission=Permission.EDIT_EVALUATION_RESULTS, # type: ignore - ): - raise FORBIDDEN_EXCEPTION # type: ignore - - results = await self.evaluations_service.create_results( - project_id=UUID(request.state.project_id), - user_id=UUID(request.state.user_id), - # - results=results_create_request.results, - ) - - results_response = EvaluationResultsResponse( - count=len(results), - results=results, - ) - - return results_response - - # PATCH /evaluations/results/ - @intercept_exceptions() - @handle_evaluation_closed_exception() - async def edit_results( + async def set_results( self, request: Request, *, - results_edit_request: EvaluationResultsEditRequest, + results_set_request: EvaluationResultsSetRequest, ) -> EvaluationResultsResponse: if is_ee(): if not await check_action_access( # type: ignore @@ -1213,11 +1213,11 @@ async def edit_results( ): raise FORBIDDEN_EXCEPTION # type: ignore - results = await self.evaluations_service.edit_results( + results = await self.evaluations_service.set_results( project_id=UUID(request.state.project_id), user_id=UUID(request.state.user_id), # - results=results_edit_request.results, + results=results_set_request.results, ) results_response = EvaluationResultsResponse( @@ -1319,42 +1319,6 @@ async def fetch_result( return result_response - # PATCH /evaluations/results/{result_id} - @intercept_exceptions() - @handle_evaluation_closed_exception() - async def edit_result( - self, - request: Request, - *, - result_id: UUID, - # - result_edit_request: EvaluationResultEditRequest, - ) -> EvaluationResultResponse: - if is_ee(): - if not await check_action_access( # type: ignore - user_uid=request.state.user_id, - project_id=request.state.project_id, - permission=Permission.EDIT_EVALUATION_RESULTS, # type: ignore - ): - raise FORBIDDEN_EXCEPTION # type: ignore - - if str(result_id) != str(result_edit_request.result.id): - return EvaluationResultResponse() - - result = await self.evaluations_service.edit_result( - project_id=UUID(request.state.project_id), - user_id=UUID(request.state.user_id), - # - result=result_edit_request.result, - ) - - result_response = EvaluationResultResponse( - count=1 if result else 0, - result=result, - ) - - return result_response - # DELETE /evaluations/results/{result_id} @intercept_exceptions() @handle_evaluation_closed_exception() @@ -1421,11 +1385,11 @@ async def refresh_metrics( # POST /evaluations/metrics/ @intercept_exceptions() @handle_evaluation_closed_exception() - async def create_metrics( + async def set_metrics( self, request: Request, *, - metrics_create_request: EvaluationMetricsCreateRequest, + metrics_set_request: EvaluationMetricsSetRequest, ) -> EvaluationMetricsResponse: if is_ee(): if not await check_action_access( # type: ignore @@ -1435,11 +1399,11 @@ async def create_metrics( ): raise FORBIDDEN_EXCEPTION # type: ignore - metrics = await self.evaluations_service.create_metrics( + metrics = await self.evaluations_service.set_metrics( project_id=UUID(request.state.project_id), user_id=UUID(request.state.user_id), # - metrics=metrics_create_request.metrics, + metrics=metrics_set_request.metrics, ) metrics_response = EvaluationMetricsResponse( @@ -1449,15 +1413,15 @@ async def create_metrics( return metrics_response - # PATCH /evaluations/metrics/ + # DELETE /evaluations/metrics/ @intercept_exceptions() @handle_evaluation_closed_exception() - async def edit_metrics( + async def delete_metrics( self, request: Request, *, - metrics_edit_request: EvaluationMetricsEditRequest, - ) -> EvaluationMetricsResponse: + metrics_ids_request: EvaluationMetricsIdsRequest, + ) -> EvaluationMetricsIdsResponse: if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, @@ -1466,58 +1430,59 @@ async def edit_metrics( ): raise FORBIDDEN_EXCEPTION # type: ignore - metrics = await self.evaluations_service.edit_metrics( + metrics_ids = await self.evaluations_service.delete_metrics( project_id=UUID(request.state.project_id), - user_id=UUID(request.state.user_id), # - metrics=metrics_edit_request.metrics, + metrics_ids=metrics_ids_request.metrics_ids, ) - metrics_response = EvaluationMetricsResponse( - count=len(metrics), - metrics=metrics, + metrics_ids_response = EvaluationMetricsIdsResponse( + count=len(metrics_ids), + metrics_ids=metrics_ids, ) - return metrics_response + return metrics_ids_response - # DELETE /evaluations/metrics/ + # POST /evaluations/metrics/query @intercept_exceptions() - @handle_evaluation_closed_exception() - async def delete_metrics( + @suppress_exceptions(default=EvaluationMetricsResponse(), exclude=[HTTPException]) + async def query_metrics( self, request: Request, *, - metrics_ids_request: EvaluationMetricsIdsRequest, - ) -> EvaluationMetricsIdsResponse: + metric_query_request: EvaluationMetricsQueryRequest, + ) -> EvaluationMetricsResponse: if is_ee(): if not await check_action_access( # type: ignore user_uid=request.state.user_id, project_id=request.state.project_id, - permission=Permission.EDIT_EVALUATION_METRICS, # type: ignore + permission=Permission.VIEW_EVALUATION_METRICS, # type: ignore ): raise FORBIDDEN_EXCEPTION # type: ignore - metrics_ids = await self.evaluations_service.delete_metrics( + metrics = await self.evaluations_service.query_metrics( project_id=UUID(request.state.project_id), # - metrics_ids=metrics_ids_request.metrics_ids, + metric=metric_query_request.metrics, + # + windowing=metric_query_request.windowing, ) - metrics_ids_response = EvaluationMetricsIdsResponse( - count=len(metrics_ids), - metrics_ids=metrics_ids, + metrics_response = EvaluationMetricsResponse( + count=len(metrics), + metrics=metrics, ) - return metrics_ids_response + return metrics_response - # POST /evaluations/metrics/query + # GET /evaluations/metrics/{metrics_id} @intercept_exceptions() @suppress_exceptions(default=EvaluationMetricsResponse(), exclude=[HTTPException]) - async def query_metrics( + async def fetch_metric( self, request: Request, *, - metric_query_request: EvaluationMetricsQueryRequest, + metrics_id: UUID, ) -> EvaluationMetricsResponse: if is_ee(): if not await check_action_access( # type: ignore @@ -1527,12 +1492,10 @@ async def query_metrics( ): raise FORBIDDEN_EXCEPTION # type: ignore - metrics = await self.evaluations_service.query_metrics( + metrics = await self.evaluations_service.fetch_metrics( project_id=UUID(request.state.project_id), # - metric=metric_query_request.metrics, - # - windowing=metric_query_request.windowing, + metrics_ids=[metrics_id], ) metrics_response = EvaluationMetricsResponse( @@ -1542,6 +1505,36 @@ async def query_metrics( return metrics_response + # DELETE /evaluations/metrics/{metrics_id} + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def delete_metric( + self, + request: Request, + *, + metrics_id: UUID, + ) -> EvaluationMetricsIdsResponse: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_METRICS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + metrics_ids = await self.evaluations_service.delete_metrics( + project_id=UUID(request.state.project_id), + # + metrics_ids=[metrics_id], + ) + + metrics_ids_response = EvaluationMetricsIdsResponse( + count=len(metrics_ids), + metrics_ids=metrics_ids, + ) + + return metrics_ids_response + # EVALUATION QUEUES -------------------------------------------------------- # POST /evaluations/queues/ @@ -1734,6 +1727,54 @@ async def edit_queue( return queue_response + # POST /evaluations/queues/{queue_id}/archive + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def archive_queue( + self, + request: Request, + *, + queue_id: UUID, + ) -> EvaluationQueueResponse: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_QUEUES, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + queue = await self.evaluations_service.archive_queue( + project_id=UUID(request.state.project_id), + user_id=UUID(request.state.user_id), + queue_id=queue_id, + ) + return EvaluationQueueResponse(count=1 if queue else 0, queue=queue) + + # POST /evaluations/queues/{queue_id}/unarchive + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def unarchive_queue( + self, + request: Request, + *, + queue_id: UUID, + ) -> EvaluationQueueResponse: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_QUEUES, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + queue = await self.evaluations_service.unarchive_queue( + project_id=UUID(request.state.project_id), + user_id=UUID(request.state.user_id), + queue_id=queue_id, + ) + return EvaluationQueueResponse(count=1 if queue else 0, queue=queue) + # DELETE /evaluations/queues/{queue_id} @intercept_exceptions() @handle_evaluation_closed_exception() @@ -1837,6 +1878,16 @@ def __init__( operation_id="create_simple_evaluation", ) + # POST /api/simple/evaluations/query + self.router.add_api_route( + path="/query", + methods=["POST"], + endpoint=self.query_evaluations, + response_model=SimpleEvaluationsResponse, + response_model_exclude_none=True, + operation_id="query_simple_evaluations", + ) + # GET /api/simple/evaluations/{evaluation_id} self.router.add_api_route( path="/{evaluation_id}", @@ -1867,19 +1918,9 @@ def __init__( operation_id="delete_simple_evaluation", ) - # POST /api/simple/evaluations/query + # POST /api/simple/evaluations/{evaluation_id}/start self.router.add_api_route( - path="/query", - methods=["POST"], - endpoint=self.query_evaluations, - response_model=SimpleEvaluationsResponse, - response_model_exclude_none=True, - operation_id="query_simple_evaluations", - ) - - # POST /api/simple/evaluations/{evaluation_id}/start - self.router.add_api_route( - path="/{evaluation_id}/start", + path="/{evaluation_id}/start", methods=["POST"], endpoint=self.start_evaluation, response_model=SimpleEvaluationResponse, @@ -1917,6 +1958,104 @@ def __init__( operation_id="open_simple_evaluation", ) + # POST /api/simple/evaluations/{evaluation_id}/populate + self.router.add_api_route( + path="/{evaluation_id}/populate", + methods=["POST"], + endpoint=self.populate_evaluation_slice, + response_model=EvaluationResultsResponse, + response_model_exclude_none=True, + operation_id="populate_slice", + ) + + # POST /api/simple/evaluations/{evaluation_id}/process + self.router.add_api_route( + path="/{evaluation_id}/process", + methods=["POST"], + endpoint=self.process_evaluation_slice, + status_code=http_status.HTTP_202_ACCEPTED, + response_model=None, + responses={http_status.HTTP_202_ACCEPTED: {"description": "Accepted."}}, + operation_id="process_slice", + ) + + # POST /api/simple/evaluations/{evaluation_id}/probe + self.router.add_api_route( + path="/{evaluation_id}/probe", + methods=["POST"], + endpoint=self.probe_evaluation_slice, + response_model=EvaluationResultsResponse, + response_model_exclude_none=True, + operation_id="probe_slice", + ) + + # POST /api/simple/evaluations/{evaluation_id}/prune + self.router.add_api_route( + path="/{evaluation_id}/prune", + methods=["POST"], + endpoint=self.prune_evaluation_slice, + status_code=http_status.HTTP_204_NO_CONTENT, + operation_id="prune_slice", + ) + + # POST /api/simple/evaluations/{evaluation_id}/refresh + self.router.add_api_route( + path="/{evaluation_id}/refresh", + methods=["POST"], + endpoint=self.refresh_evaluation_slice, + status_code=http_status.HTTP_204_NO_CONTENT, + operation_id="refresh_slice", + ) + + # POST /api/simple/evaluations/{evaluation_id}/scenarios/add + self.router.add_api_route( + path="/{evaluation_id}/scenarios/add", + methods=["POST"], + endpoint=self.add_evaluation_scenarios, + response_model=EvaluationScenariosResponse, + response_model_exclude_none=True, + operation_id="add_scenarios", + ) + + # POST /api/simple/evaluations/{evaluation_id}/scenarios/remove + self.router.add_api_route( + path="/{evaluation_id}/scenarios/remove", + methods=["POST"], + endpoint=self.remove_evaluation_scenarios, + status_code=204, # rows removed — no body + operation_id="remove_scenarios", + ) + + # POST /api/simple/evaluations/{evaluation_id}/steps/add + self.router.add_api_route( + path="/{evaluation_id}/steps/add", + methods=["POST"], + endpoint=self.add_evaluation_steps, + response_model=EvaluationRunResponse, + response_model_exclude_none=True, + operation_id="add_steps", + ) + + # POST /api/simple/evaluations/{evaluation_id}/steps/remove + self.router.add_api_route( + path="/{evaluation_id}/steps/remove", + methods=["POST"], + endpoint=self.remove_evaluation_steps, + response_model=EvaluationRunResponse, + response_model_exclude_none=True, + operation_id="remove_steps", + ) + + # POST /api/simple/evaluations/{evaluation_id}/repeats/set + self.router.add_api_route( + path="/{evaluation_id}/repeats/set", + methods=["POST"], + endpoint=self.set_evaluation_repeats, + response_model=EvaluationRunResponse, + response_model_exclude_none=True, + operation_id="set_repeats", + ) + # SIMPLE EVALUATIONS ------------------------------------------------------- # POST /api/simple/evaluations/ @@ -2225,6 +2364,355 @@ async def open_evaluation( return response + # TENSOR SLICE OPS --------------------------------------------------------- + + # POST /api/simple/evaluations/{evaluation_id}/populate + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def populate_evaluation_slice( + self, + request: Request, + *, + evaluation_id: UUID, + populate_slice_request: PopulateSliceRequest, + ) -> EvaluationResultsResponse: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_RUNS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + # The path addresses ONE run; each result carries its own run_id. Reject + # any result whose run_id does not match the path so a caller cannot + # write cells into a different run via the body (the service/DAO only + # scope by project + per-result run_id, not the path). + mismatched = [ + result + for result in populate_slice_request.results + if result.run_id != evaluation_id + ] + if mismatched: + raise HTTPException( + status_code=http_status.HTTP_400_BAD_REQUEST, + detail=( + "All results must target the evaluation in the path " + f"({evaluation_id})." + ), + ) + + results = await self.simple_evaluations_service.populate_slice( + project_id=UUID(request.state.project_id), + user_id=UUID(request.state.user_id), + # + results=populate_slice_request.results, + ) + + return EvaluationResultsResponse( + count=len(results), + results=results, + ) + + # POST /api/simple/evaluations/{evaluation_id}/process + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def process_evaluation_slice( + self, + request: Request, + *, + evaluation_id: UUID, + preocess_slice_request: ProcessSliceRequest, + ) -> None: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_RUNS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + # Async dispatch via taskiq — the 202 acknowledges acceptance; the work + # finishes on the worker. No body to return. `overwrite` maps to the + # internal process mode (force = re-run all; else fill-missing). + await self.simple_evaluations_service.dispatch_tensor_slice( + project_id=UUID(request.state.project_id), + user_id=UUID(request.state.user_id), + # + run_id=evaluation_id, + scenario_ids=preocess_slice_request.scenario_ids, + step_keys=preocess_slice_request.step_keys, + repeat_idxs=preocess_slice_request.repeat_idxs, + process_mode="force" + if preocess_slice_request.overwrite + else "fill-missing", + ) + + # POST /api/simple/evaluations/{evaluation_id}/probe + @intercept_exceptions() + async def probe_evaluation_slice( + self, + request: Request, + *, + evaluation_id: UUID, + probe_slice_request: ProbeSliceRequest, + ) -> EvaluationResultsResponse: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_EVALUATION_RUNS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + results = await self.simple_evaluations_service.probe_slice( + project_id=UUID(request.state.project_id), + # + run_id=evaluation_id, + scenario_ids=probe_slice_request.scenario_ids, + step_keys=probe_slice_request.step_keys, + repeat_idxs=probe_slice_request.repeat_idxs, + ) + + return EvaluationResultsResponse( + count=len(results), + results=results, + ) + + # POST /api/simple/evaluations/{evaluation_id}/prune + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def prune_evaluation_slice( + self, + request: Request, + *, + evaluation_id: UUID, + prune_slice_request: PruneSliceRequest, + ) -> None: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_RUNS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + # Removes the addressed result cells (and refreshes metrics over the + # scope). 204 — no body. + await self.simple_evaluations_service.prune_slice( + project_id=UUID(request.state.project_id), + user_id=UUID(request.state.user_id), + # + run_id=evaluation_id, + scenario_ids=prune_slice_request.scenario_ids, + step_keys=prune_slice_request.step_keys, + repeat_idxs=prune_slice_request.repeat_idxs, + ) + + # POST /api/simple/evaluations/{evaluation_id}/refresh + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def refresh_evaluation_slice( + self, + request: Request, + *, + evaluation_id: UUID, + refresh_slice_request: RefreshSliceRequest, + ) -> None: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_RUNS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + # Recomputes the metric rows (variational + aggregate) over the addressed + # scope without writing or executing cells. Used by callers that populated + # finished cells themselves (e.g. the SDK). 204 — no body. + await self.simple_evaluations_service.refresh_slice( + project_id=UUID(request.state.project_id), + user_id=UUID(request.state.user_id), + # + run_id=evaluation_id, + scenario_ids=refresh_slice_request.scenario_ids, + step_keys=refresh_slice_request.step_keys, + repeat_idxs=refresh_slice_request.repeat_idxs, + ) + + # POST /api/simple/evaluations/{evaluation_id}/scenarios/add + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def add_evaluation_scenarios( + self, + request: Request, + *, + evaluation_id: UUID, + add_request: AddScenariosRequest, + ) -> EvaluationScenariosResponse: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_RUNS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + scenarios = await self.simple_evaluations_service.add_scenarios( + project_id=UUID(request.state.project_id), + user_id=UUID(request.state.user_id), + # + run_id=evaluation_id, + count=add_request.count, + timestamp=add_request.timestamp, + ) + + return EvaluationScenariosResponse( + count=len(scenarios), + scenarios=scenarios, + ) + + # POST /api/simple/evaluations/{evaluation_id}/scenarios/remove + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def remove_evaluation_scenarios( + self, + request: Request, + *, + evaluation_id: UUID, + remove_request: RemoveScenariosRequest, + ) -> None: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_RUNS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + # Scope to the run in the path: only delete scenarios that actually + # belong to `evaluation_id`, so a caller cannot delete another run's + # scenarios by id. Resolve the owned subset first; reject if any + # requested id is not in this run. + owned = set( + await self.simple_evaluations_service.evaluations_service.query_scenario_ids( + project_id=UUID(request.state.project_id), + scenario=EvaluationScenarioQuery( + run_id=evaluation_id, + ids=remove_request.scenario_ids, + ), + ) + ) + not_owned = [s for s in remove_request.scenario_ids if s not in owned] + if not_owned: + raise HTTPException( + status_code=http_status.HTTP_400_BAD_REQUEST, + detail=( + "All scenario_ids must belong to the evaluation in the path " + f"({evaluation_id})." + ), + ) + + # Drops the scenario rows (and their cells). 204 — no body. + await self.simple_evaluations_service.remove_scenarios( + project_id=UUID(request.state.project_id), + # + scenario_ids=remove_request.scenario_ids, + ) + + # POST /api/simple/evaluations/{evaluation_id}/steps/add + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def add_evaluation_steps( + self, + request: Request, + *, + evaluation_id: UUID, + add_request: AddStepsRequest, + ) -> EvaluationRunResponse: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_RUNS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + run = await self.simple_evaluations_service.add_steps( + project_id=UUID(request.state.project_id), + user_id=UUID(request.state.user_id), + # + run_id=evaluation_id, + steps=add_request.steps, + ) + + return EvaluationRunResponse( + count=1 if run else 0, + run=run, + ) + + # POST /api/simple/evaluations/{evaluation_id}/steps/remove + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def remove_evaluation_steps( + self, + request: Request, + *, + evaluation_id: UUID, + remove_request: RemoveStepsRequest, + ) -> EvaluationRunResponse: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_RUNS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + run = await self.simple_evaluations_service.remove_steps( + project_id=UUID(request.state.project_id), + user_id=UUID(request.state.user_id), + # + run_id=evaluation_id, + step_keys=remove_request.step_keys, + ) + + return EvaluationRunResponse( + count=1 if run else 0, + run=run, + ) + + # POST /api/simple/evaluations/{evaluation_id}/repeats/set + @intercept_exceptions() + @handle_evaluation_closed_exception() + async def set_evaluation_repeats( + self, + request: Request, + *, + evaluation_id: UUID, + repeats_request: SetRepeatsRequest, + ) -> EvaluationRunResponse: + if is_ee(): + if not await check_action_access( # type: ignore + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.EDIT_EVALUATION_RUNS, # type: ignore + ): + raise FORBIDDEN_EXCEPTION # type: ignore + + run = await self.simple_evaluations_service.set_repeats( + project_id=UUID(request.state.project_id), + user_id=UUID(request.state.user_id), + # + run_id=evaluation_id, + repeats=repeats_request.repeats, + ) + + return EvaluationRunResponse( + count=1 if run else 0, + run=run, + ) + class SimpleQueuesRouter: def __init__( diff --git a/api/oss/src/apis/fastapi/evaluations/utils.py b/api/oss/src/apis/fastapi/evaluations/utils.py index 3810092b92..63263d679e 100644 --- a/api/oss/src/apis/fastapi/evaluations/utils.py +++ b/api/oss/src/apis/fastapi/evaluations/utils.py @@ -16,6 +16,11 @@ EvaluationQueueQueryFlags, # EvaluationClosedConflict, + EvaluationMetricsInvalid, + DefaultQueueDataInvalid, + DefaultQueueDemotionForbidden, + DefaultQueueDeletionForbidden, + DefaultQueueArchiveForbidden, ) from oss.src.apis.fastapi.shared.utils import ( @@ -38,6 +43,9 @@ EvaluationQueueQueryRequest, # EvaluationClosedException, + EvaluationMetricsInvalidException, + DefaultQueueDataInvalidException, + DefaultQueueEditingForbiddenException, ) log = get_module_logger(__name__) @@ -57,6 +65,27 @@ async def wrapper(*args, **kwargs): result_id=e.result_id, metrics_id=e.metrics_id, ) from e + except EvaluationMetricsInvalid as e: + raise EvaluationMetricsInvalidException( + message=e.message, + run_id=e.run_id, + scenario_id=e.scenario_id, + timestamp=e.timestamp, + ) from e + except DefaultQueueDataInvalid as e: + raise DefaultQueueDataInvalidException( + message=e.message, + queue_id=e.queue_id, + ) from e + except ( + DefaultQueueDemotionForbidden, + DefaultQueueDeletionForbidden, + DefaultQueueArchiveForbidden, + ) as e: + raise DefaultQueueEditingForbiddenException( + message=e.message, + queue_id=e.queue_id, + ) from e except Exception as e: raise e diff --git a/api/oss/src/apis/fastapi/evaluators/router.py b/api/oss/src/apis/fastapi/evaluators/router.py index b181f65a8e..21a18a5386 100644 --- a/api/oss/src/apis/fastapi/evaluators/router.py +++ b/api/oss/src/apis/fastapi/evaluators/router.py @@ -99,8 +99,11 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/apis/fastapi/folders/router.py b/api/oss/src/apis/fastapi/folders/router.py index fcefb0a611..54875685cb 100644 --- a/api/oss/src/apis/fastapi/folders/router.py +++ b/api/oss/src/apis/fastapi/folders/router.py @@ -31,8 +31,11 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/apis/fastapi/invocations/router.py b/api/oss/src/apis/fastapi/invocations/router.py index 50ab6b8713..c95e1a306c 100644 --- a/api/oss/src/apis/fastapi/invocations/router.py +++ b/api/oss/src/apis/fastapi/invocations/router.py @@ -24,8 +24,11 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/apis/fastapi/legacy_variants/router.py b/api/oss/src/apis/fastapi/legacy_variants/router.py index bec66a2046..4398c5af67 100644 --- a/api/oss/src/apis/fastapi/legacy_variants/router.py +++ b/api/oss/src/apis/fastapi/legacy_variants/router.py @@ -18,8 +18,11 @@ from oss.src.utils.exceptions import intercept_exceptions if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import FORBIDDEN_EXCEPTION, check_action_access + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + FORBIDDEN_EXCEPTION, + check_action_access, + ) def _as_reference(ref: Optional[ReferenceRequestModel]) -> Optional[Reference]: diff --git a/api/oss/src/apis/fastapi/otlp/router.py b/api/oss/src/apis/fastapi/otlp/router.py index c3ac194fac..d1dba97e4d 100644 --- a/api/oss/src/apis/fastapi/otlp/router.py +++ b/api/oss/src/apis/fastapi/otlp/router.py @@ -19,9 +19,12 @@ from oss.src.apis.fastapi.otlp.utils.processing import parse_from_otel_span_dto if is_ee(): - from ee.src.utils.entitlements import check_entitlements, Counter - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.entitlements.service import check_entitlements, Counter + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) if TYPE_CHECKING: from oss.src.core.tracing.service import TracingService diff --git a/api/oss/src/apis/fastapi/queries/router.py b/api/oss/src/apis/fastapi/queries/router.py index 91b9db0148..ff241102fa 100644 --- a/api/oss/src/apis/fastapi/queries/router.py +++ b/api/oss/src/apis/fastapi/queries/router.py @@ -56,8 +56,11 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/apis/fastapi/testcases/router.py b/api/oss/src/apis/fastapi/testcases/router.py index be309f68bc..66977ea2a8 100644 --- a/api/oss/src/apis/fastapi/testcases/router.py +++ b/api/oss/src/apis/fastapi/testcases/router.py @@ -28,8 +28,11 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/apis/fastapi/testsets/router.py b/api/oss/src/apis/fastapi/testsets/router.py index b6f59fa440..7a5f44d7a2 100644 --- a/api/oss/src/apis/fastapi/testsets/router.py +++ b/api/oss/src/apis/fastapi/testsets/router.py @@ -91,8 +91,11 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/apis/fastapi/tools/router.py b/api/oss/src/apis/fastapi/tools/router.py index d066c0b8d7..043d114fa7 100644 --- a/api/oss/src/apis/fastapi/tools/router.py +++ b/api/oss/src/apis/fastapi/tools/router.py @@ -56,8 +56,11 @@ _SLUG_SEGMENT_RE = re.compile(r"^[a-zA-Z0-9_-]+$") if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/apis/fastapi/traces/router.py b/api/oss/src/apis/fastapi/traces/router.py index faee284645..388d9ae6c9 100644 --- a/api/oss/src/apis/fastapi/traces/router.py +++ b/api/oss/src/apis/fastapi/traces/router.py @@ -20,8 +20,11 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/apis/fastapi/tracing/router.py b/api/oss/src/apis/fastapi/tracing/router.py index 9f4ca68cd3..75610f36e9 100644 --- a/api/oss/src/apis/fastapi/tracing/router.py +++ b/api/oss/src/apis/fastapi/tracing/router.py @@ -64,9 +64,12 @@ log = get_module_logger(__name__) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION - from ee.src.utils.entitlements import check_entitlements, Counter + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) + from ee.src.core.access.entitlements.service import check_entitlements, Counter class TracingRouter: diff --git a/api/oss/src/apis/fastapi/vault/router.py b/api/oss/src/apis/fastapi/vault/router.py index 7f05e8bd0f..a78fd3ece0 100644 --- a/api/oss/src/apis/fastapi/vault/router.py +++ b/api/oss/src/apis/fastapi/vault/router.py @@ -17,8 +17,8 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import check_action_access log = get_module_logger(__name__) @@ -185,6 +185,7 @@ async def update_secret( project_id=UUID(request.state.project_id), secret_id=UUID(secret_id), update_secret_dto=body, + user_id=UUID(request.state.user_id), ) if secrets_dto is None: raise HTTPException( diff --git a/api/oss/src/apis/fastapi/webhooks/router.py b/api/oss/src/apis/fastapi/webhooks/router.py index ff344a74e8..6ab1173d2a 100644 --- a/api/oss/src/apis/fastapi/webhooks/router.py +++ b/api/oss/src/apis/fastapi/webhooks/router.py @@ -32,8 +32,11 @@ ) if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) class WebhooksRouter: diff --git a/api/oss/src/apis/fastapi/workflows/router.py b/api/oss/src/apis/fastapi/workflows/router.py index 8062ecf9d1..85828cef1d 100644 --- a/api/oss/src/apis/fastapi/workflows/router.py +++ b/api/oss/src/apis/fastapi/workflows/router.py @@ -97,8 +97,11 @@ if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access, FORBIDDEN_EXCEPTION + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) log = get_module_logger(__name__) diff --git a/api/oss/src/core/accounts/service.py b/api/oss/src/core/accounts/service.py index 4113f5d646..e30cffcdb6 100644 --- a/api/oss/src/core/accounts/service.py +++ b/api/oss/src/core/accounts/service.py @@ -87,15 +87,12 @@ from ee.src.core.subscriptions.types import ( # type: ignore[import] get_default_plan as _ee_get_default_plan, ) - from ee.src.core.entitlements.controls import ( # type: ignore[import] + from ee.src.core.access.controls import ( # type: ignore[import] get_plans as _ee_get_plans, ) - from ee.src.core.meters.service import MetersService as _EeMetersService # type: ignore[import] - from ee.src.dbs.postgres.meters.dao import MetersDAO as _EeMetersDAO # type: ignore[import] _ee_subscription_service = _EeSubscriptionsService( subscriptions_dao=_EeSubscriptionsDAO(), - meters_service=_EeMetersService(meters_dao=_EeMetersDAO()), ) except ImportError: pass diff --git a/api/oss/src/core/auth/helper.py b/api/oss/src/core/auth/helper.py index a0ad8eef95..bdd4b34a69 100644 --- a/api/oss/src/core/auth/helper.py +++ b/api/oss/src/core/auth/helper.py @@ -1,12 +1,11 @@ from dataclasses import dataclass from typing import Any, Optional, Set -import posthog - -from oss.src.services.exceptions import UnauthorizedException +from oss.src.utils.exceptions import UnauthorizedException from oss.src.utils.caching import get_cache, set_cache from oss.src.utils.common import is_ee from oss.src.utils.env import env +from oss.src.utils.lazy import _load_posthog from oss.src.utils.logging import get_module_logger @@ -61,10 +60,27 @@ async def _get_posthog_string_entries(feature_flag: str) -> Set[str]: if cached_entries is not None: return _normalize_string_set(cached_entries) - flag_entries = posthog.get_feature_flag_payload( - feature_flag, - "user distinct id", - ) + posthog = _load_posthog() + if posthog is None: + log.warning( + "[AUTH] PostHog feature flag lookup skipped", + feature_flag=feature_flag, + reason="unavailable", + ) + return set() + + try: + flag_entries = posthog.get_feature_flag_payload( + feature_flag, + "user distinct id", + ) + except Exception as exc: + log.warning( + "[AUTH] PostHog feature flag lookup skipped", + feature_flag=feature_flag, + reason=str(exc), + ) + return set() normalized_entries = _normalize_string_set(flag_entries) @@ -147,6 +163,6 @@ async def ensure_auth_info_not_blocked( auth_info: Optional[AuthInfo], ) -> Optional[AuthInfo]: if auth_info and await is_auth_info_blocked(auth_info): - raise UnauthorizedException(detail="Access Denied.") + raise UnauthorizedException(message="Access Denied.") return auth_info diff --git a/api/oss/src/core/auth/service.py b/api/oss/src/core/auth/service.py index a39310a8c4..2cf5ffadfa 100644 --- a/api/oss/src/core/auth/service.py +++ b/api/oss/src/core/auth/service.py @@ -11,7 +11,7 @@ from oss.src.models.db_models import InvitationDB, ProjectDB, OrganizationDB from oss.src.services import db_manager -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import get_transactions_engine from oss.src.dbs.postgres.users.dao import IdentitiesDAO if is_ee(): @@ -130,7 +130,9 @@ async def discover(self, email: str) -> Dict[str, Any]: # 2. Organizations with pending project invitations if email: try: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: # Query project_invitations for this email, join with projects to get organization_id stmt = ( select(ProjectDB.organization_id) @@ -480,7 +482,9 @@ async def enforce_domain_policies(self, email: str, user_id: UUID) -> None: "Auto-join requires organization, user, and at least one workspace" ) - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: existing_org_member = await session.execute( select(OrganizationMemberDB).filter_by( user_id=user.id, organization_id=organization.id @@ -791,7 +795,9 @@ async def _get_organization_flags( if not is_ee(): return None - async with db_manager.engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: stmt = select(OrganizationDB.flags).where( OrganizationDB.id == organization_id ) @@ -803,7 +809,9 @@ async def _get_organization_slug(self, organization_id: UUID) -> Optional[str]: if not is_ee(): return None - async with db_manager.engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: stmt = select(OrganizationDB.slug).where( OrganizationDB.id == organization_id ) @@ -820,7 +828,9 @@ async def _is_organization_member( if not is_ee(): return False - async with db_manager.engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: stmt = select(OrganizationMemberDB).where( OrganizationMemberDB.user_id == user_id, OrganizationMemberDB.organization_id == organization_id, @@ -837,7 +847,9 @@ async def _is_organization_owner( if not is_ee(): return False - async with db_manager.engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: stmt = select(OrganizationMemberDB.role).where( OrganizationMemberDB.user_id == user_id, OrganizationMemberDB.organization_id == organization_id, diff --git a/api/oss/src/core/auth/supertokens/overrides.py b/api/oss/src/core/auth/supertokens/overrides.py index 0fc3ab1bda..ac05eb3255 100644 --- a/api/oss/src/core/auth/supertokens/overrides.py +++ b/api/oss/src/core/auth/supertokens/overrides.py @@ -1,9 +1,8 @@ from typing import Dict, Any, List, Optional, Union from urllib.parse import urlparse -import posthog - from oss.src.utils.logging import get_module_logger +from oss.src.utils.lazy import _load_posthog from supertokens_python.recipe.thirdparty.provider import ( ProviderInput, @@ -65,7 +64,7 @@ ) from oss.src.services import db_manager -from oss.src.services.exceptions import UnauthorizedException +from oss.src.utils.exceptions import UnauthorizedException from oss.src.services.db_manager import ( get_user_with_email, check_if_user_invitation_exists, @@ -234,7 +233,7 @@ async def _create_account(email: str, uid: str) -> bool: organization_db = await get_oss_organization() if not organization_db: raise UnauthorizedException( - detail="No organization found. Please contact the administrator." + message="No organization found. Please contact the administrator." ) # Verify user can join (invitation check) @@ -244,13 +243,14 @@ async def _create_account(email: str, uid: str) -> bool: ) if not user_invitation_exists: raise UnauthorizedException( - detail="You need to be invited by the organization owner to gain access." + message="You need to be invited by the organization owner to gain access." ) payload["organization_id"] = str(organization_db.id) await create_accounts(payload) - if env.posthog.enabled and env.posthog.api_key: + posthog = _load_posthog() + if posthog is not None: try: posthog.capture( distinct_id=auth_info.email, @@ -263,8 +263,11 @@ async def _create_account(email: str, uid: str) -> bool: "$set": {"email": auth_info.email}, }, ) - except Exception: - log.error("[AUTH] Failed to capture PostHog signup event", exc_info=True) + except Exception as exc: + log.warning( + "[AUTH] PostHog signup event capture skipped", + reason=str(exc), + ) log.info("[AUTH] _create_account done", email=auth_info.email, uid=uid) return True diff --git a/api/oss/src/core/auth/turnstile.py b/api/oss/src/core/auth/turnstile.py index 8a90a05cc9..a51440173a 100644 --- a/api/oss/src/core/auth/turnstile.py +++ b/api/oss/src/core/auth/turnstile.py @@ -2,7 +2,7 @@ from supertokens_python.framework.request import BaseRequest -from oss.src.services.exceptions import UnauthorizedException +from oss.src.utils.exceptions import UnauthorizedException from oss.src.utils.common import is_ee from oss.src.utils.env import env from oss.src.utils.logging import get_module_logger @@ -70,7 +70,7 @@ async def verify_turnstile_or_raise( auth_flow, get_client_ip(request), ) - raise UnauthorizedException(detail="Please complete the security check.") + raise UnauthorizedException(message="Please complete the security check.") payload = { "secret": env.cloudflare.turnstile.secret_key or "", @@ -93,7 +93,7 @@ async def verify_turnstile_or_raise( remote_ip, exc_info=True, ) - raise UnauthorizedException(detail=TURNSTILE_FAILURE_MESSAGE) from None + raise UnauthorizedException(message=TURNSTILE_FAILURE_MESSAGE) from None if verification_result.get("success") is True: actual_hostname = _normalize_hostname(verification_result.get("hostname")) @@ -107,7 +107,7 @@ async def verify_turnstile_or_raise( sorted(expected_hostnames), remote_ip, ) - raise UnauthorizedException(detail=TURNSTILE_FAILURE_MESSAGE) + raise UnauthorizedException(message=TURNSTILE_FAILURE_MESSAGE) log.info( "[AUTH] Turnstile verification succeeded auth_flow=%s hostname=%s action=%s client_ip=%s", @@ -126,4 +126,4 @@ async def verify_turnstile_or_raise( verification_result.get("action"), remote_ip, ) - raise UnauthorizedException(detail=TURNSTILE_FAILURE_MESSAGE) + raise UnauthorizedException(message=TURNSTILE_FAILURE_MESSAGE) diff --git a/api/oss/src/core/evaluations/interfaces.py b/api/oss/src/core/evaluations/interfaces.py index 5858682f64..658aacca3e 100644 --- a/api/oss/src/core/evaluations/interfaces.py +++ b/api/oss/src/core/evaluations/interfaces.py @@ -15,11 +15,9 @@ EvaluationScenarioQuery, EvaluationResult, EvaluationResultCreate, - EvaluationResultEdit, EvaluationResultQuery, EvaluationMetrics, EvaluationMetricsCreate, - EvaluationMetricsEdit, EvaluationMetricsQuery, EvaluationQueue, EvaluationQueueCreate, @@ -308,7 +306,7 @@ async def create_result( raise NotImplementedError @abstractmethod - async def create_results( + async def set_results( self, *, project_id: UUID, @@ -338,28 +336,6 @@ async def fetch_results( ) -> List[EvaluationResult]: raise NotImplementedError - @abstractmethod - async def edit_result( - self, - *, - project_id: UUID, - user_id: UUID, - # - result: EvaluationResultEdit, - ) -> Optional[EvaluationResult]: - raise NotImplementedError - - @abstractmethod - async def edit_results( - self, - *, - project_id: UUID, - user_id: UUID, - # - results: List[EvaluationResultEdit], - ) -> List[EvaluationResult]: - raise NotImplementedError - @abstractmethod async def delete_result( self, @@ -392,37 +368,36 @@ async def query_results( ) -> List[EvaluationResult]: raise NotImplementedError - # - EVALUATION METRICS ----------------------------------------------------- - @abstractmethod - async def create_metrics( + async def query_result_ids( self, *, project_id: UUID, - user_id: UUID, # - metrics: List[EvaluationMetricsCreate], - ) -> List[EvaluationMetrics]: + result: Optional[EvaluationResultQuery] = None, + ) -> List[UUID]: raise NotImplementedError + # - EVALUATION METRICS ----------------------------------------------------- + @abstractmethod - async def fetch_metrics( + async def set_metrics( self, *, project_id: UUID, + user_id: UUID, # - metrics_ids: List[UUID], + metrics: List[EvaluationMetricsCreate], ) -> List[EvaluationMetrics]: raise NotImplementedError @abstractmethod - async def edit_metrics( + async def fetch_metrics( self, *, project_id: UUID, - user_id: UUID, # - metrics: List[EvaluationMetricsEdit], + metrics_ids: List[UUID], ) -> List[EvaluationMetrics]: raise NotImplementedError @@ -514,6 +489,26 @@ async def edit_queues( ) -> List[EvaluationQueue]: raise NotImplementedError + @abstractmethod + async def archive_queue( + self, + *, + project_id: UUID, + user_id: UUID, + queue_id: UUID, + ) -> Optional[EvaluationQueue]: + raise NotImplementedError + + @abstractmethod + async def unarchive_queue( + self, + *, + project_id: UUID, + user_id: UUID, + queue_id: UUID, + ) -> Optional[EvaluationQueue]: + raise NotImplementedError + @abstractmethod async def delete_queue( self, diff --git a/api/oss/src/core/evaluations/runtime/adapters.py b/api/oss/src/core/evaluations/runtime/adapters.py new file mode 100644 index 0000000000..b8a5eb9778 --- /dev/null +++ b/api/oss/src/core/evaluations/runtime/adapters.py @@ -0,0 +1,535 @@ +from asyncio import Semaphore, gather +from typing import Any, Callable, Dict, List, Optional +from uuid import UUID + +from agenta.sdk.evaluations.runtime.models import ( + ResultLogRequest, + WorkflowExecutionRequest, + WorkflowExecutionResult, +) +from agenta.sdk.models.evaluations import EvaluationStatus as SdkEvaluationStatus + +from oss.src.core.evaluations.runtime.cache import RunnableCacheResolver +from oss.src.core.evaluations.types import ( + EvaluationClosedConflict, + EvaluationMetricsRefresh, + EvaluationMetricsInvalid, + EvaluationResultCreate, + EvaluationScenarioCreate, + EvaluationScenarioEdit, + EvaluationStatus, +) +from oss.src.core.evaluations.utils import fetch_trace +from oss.src.core.workflows.dtos import ( + WorkflowServiceRequest, + WorkflowServiceRequestData, +) + + +def _status(status: Any) -> EvaluationStatus: + value = getattr(status, "value", status) + return EvaluationStatus(value) + + +def _read_field(source: Any, field: str) -> Any: + if isinstance(source, dict): + return source.get(field) + return getattr(source, field, None) + + +def _project_inputs(inputs: Any, data: Any) -> Any: + """Project source inputs onto the revision's declared input schema. + + A source row (e.g. a testcase) carries every column — input columns plus + ground-truth/bookkeeping keys like ``correct_answer`` or ``testcase_id``. + The invoked workflow should only receive the inputs its revision declares, + so filter ``inputs`` down to the keys present in + ``data.schemas.inputs.properties``. + + If the revision declares no input schema (no ``properties``), inputs pass + through unchanged so untyped/legacy revisions are not broken. + """ + if not isinstance(inputs, dict): + return inputs + + schemas = _read_field(data, "schemas") if data is not None else None + inputs_schema = _read_field(schemas, "inputs") if schemas is not None else None + properties = ( + _read_field(inputs_schema, "properties") if inputs_schema is not None else None + ) + if not isinstance(properties, dict) or not properties: + return inputs + + return {key: value for key, value in inputs.items() if key in properties} + + +def _dump_model(source: Any, **kwargs: Any) -> Any: + if hasattr(source, "model_dump"): + return source.model_dump(**kwargs) + return source + + +def _dump_json(source: Any) -> Any: + if hasattr(source, "model_dump"): + return source.model_dump(mode="json", exclude_none=True) + if isinstance(source, dict): + return {key: _dump_json(value) for key, value in source.items()} + if isinstance(source, list): + return [_dump_json(value) for value in source] + return source + + +class APIWorkflowServiceRunner: + """API adapter from SDK runtime requests to the backend workflow service.""" + + def __init__( + self, + *, + workflows_service: Any, + request_builder: Optional[ + Callable[[WorkflowExecutionRequest], Dict[str, Any]] + ] = None, + ): + self.workflows_service = workflows_service + self.request_builder = request_builder + + async def execute( + self, + request: WorkflowExecutionRequest, + ) -> WorkflowExecutionResult: + kwargs = ( + self.request_builder(request) + if self.request_builder + else request.model_dump(mode="python", exclude_none=True) + ) + response = await self.workflows_service.invoke_workflow(**kwargs) + status = getattr(response, "status", None) + status_code = getattr(status, "code", None) + has_error = status_code != 200 + error = None + + if has_error: + error = ( + status.model_dump(mode="json", exclude_none=True) + if hasattr(status, "model_dump") + else {"code": status_code} + ) + + return WorkflowExecutionResult( + status=( + SdkEvaluationStatus.FAILURE + if has_error + else SdkEvaluationStatus.SUCCESS + ), + trace_id=getattr(response, "trace_id", None), + span_id=getattr(response, "span_id", None), + error=error, + outputs=getattr(response, "outputs", None), + ) + + +class APIScenarioFactory: + def __init__( + self, + *, + project_id: UUID, + user_id: UUID, + evaluations_service: Any, + ): + self.project_id = project_id + self.user_id = user_id + self.evaluations_service = evaluations_service + + async def bulk_create( + self, + run_id: UUID, + *, + count: int, + timestamp: Any = None, + interval: Optional[int] = None, + ) -> List[Any]: + """Mint `count` RUNNING scenarios for a run in one DAO call. + + The bulk counterpart of the retired streaming factory: the unified + ingest flows (run/slice) mint all scenarios up front, then populate and + re-execute them. `timestamp`/`interval` are the run-wide temporal + coordinates (live query); they stay None for non-live runs. Order is + preserved by `create_scenarios`, so the returned list aligns 1:1 with + the source items the caller intends to bind. + """ + if count <= 0: + return [] + scenarios = await self.evaluations_service.create_scenarios( + project_id=self.project_id, + user_id=self.user_id, + scenarios=[ + EvaluationScenarioCreate( + run_id=run_id, + timestamp=timestamp, + interval=interval, + status=EvaluationStatus.RUNNING, + ) + for _ in range(count) + ], + ) + if len(scenarios) != count: + raise ValueError( + f"Failed to create {count} scenario(s) for run {run_id}: " + f"got {len(scenarios)}." + ) + return scenarios + + +class APIResultSetter: + def __init__( + self, + *, + project_id: UUID, + user_id: UUID, + timestamp: Any, + interval: Optional[int], + evaluations_service: Any, + ): + self.project_id = project_id + self.user_id = user_id + self.timestamp = timestamp + self.interval = interval + self.evaluations_service = evaluations_service + + async def set(self, request: ResultLogRequest) -> Any: + cell = request.cell + results = await self.evaluations_service.set_results( + project_id=self.project_id, + user_id=self.user_id, + results=[ + EvaluationResultCreate( + run_id=cell.run_id, + scenario_id=cell.scenario_id, + step_key=cell.step_key, + repeat_idx=cell.repeat_idx, + status=_status(cell.status), + trace_id=( + request.trace_id + if request.trace_id is not None + else cell.trace_id + ), + testcase_id=( + request.testcase_id + if request.testcase_id is not None + else cell.testcase_id + ), + error=request.error if request.error is not None else cell.error, + timestamp=self.timestamp, + interval=self.interval, + ) + ], + ) + return results[0] if results else None + + +class APIScenarioEditor: + """Engine `edit_scenario` adapter: write one scenario's terminal status. + + The engine computes each scenario's verdict in-loop, so the status write is + a property of `process` itself (shared by ingest + re-execute) rather than a + separate post-process. Tolerates a mid-flight run close: closing is a lock, + not a failure, so a write that loses the race is skipped, not raised. + """ + + def __init__( + self, + *, + project_id: UUID, + user_id: UUID, + evaluations_service: Any, + ): + self.project_id = project_id + self.user_id = user_id + self.evaluations_service = evaluations_service + + async def __call__(self, scenario: Any, status: Any) -> Any: + try: + return await self.evaluations_service.edit_scenario( + project_id=self.project_id, + user_id=self.user_id, + scenario=EvaluationScenarioEdit( + id=scenario.id, + tags=getattr(scenario, "tags", None), + meta=getattr(scenario, "meta", None), + status=_status(status), + ), + ) + except EvaluationClosedConflict: + return None + + +class APIMetricsRefresher: + """Single adapter for all three metric-refresh shapes. + + A metric belongs to exactly one of three kinds, keyed by which coordinates + are set (the same classification `set_metrics` enforces downstream): + - variational: scenario_id(s) set, no timestamp -> per-scenario rows + - temporal: timestamp(s)+interval, no scenario -> time-bucket rows + - global: neither -> the whole-run row + + The caller chooses the shape by which arguments it passes; passing a + scenario together with a timestamp is the both-set shape that matches no + unique index and is rejected here before it can reach the DAO. + """ + + def __init__( + self, + *, + project_id: UUID, + user_id: UUID, + evaluations_service: Any, + ): + self.project_id = project_id + self.user_id = user_id + self.evaluations_service = evaluations_service + + async def __call__( + self, + run_id: UUID, + scenario_id: Optional[UUID] = None, + *, + scenario_ids: Optional[List[UUID]] = None, + timestamps: Optional[List[Any]] = None, + interval: Optional[int] = None, + ) -> Any: + # The SDK runtime calls this positionally as (run_id, scenario_id) for + # the per-scenario variational refresh and (run_id, None) for the + # run-level global rollup. `_refresh_slice_aggregate` calls it by keyword + # for the temporal buckets and the global fallback. + has_scenario = scenario_id is not None or bool(scenario_ids) + has_temporal = bool(timestamps) + if has_scenario and has_temporal: + raise EvaluationMetricsInvalid( + run_id=run_id, + scenario_id=scenario_id, + timestamp=timestamps[0] if timestamps else None, + ) + + return await self.evaluations_service.refresh_metrics( + project_id=self.project_id, + user_id=self.user_id, + metrics=EvaluationMetricsRefresh( + run_id=run_id, + scenario_id=scenario_id, + scenario_ids=scenario_ids, + timestamps=timestamps, + interval=interval, + ), + ) + + +class APITraceFetcher: + """Callable trace loader: `await loader(trace_id) -> trace`. + + The engine's `fetch_trace` seam is a plain async callable; this named class + carries the bound `project_id`/`tracing_service` and is invoked directly + (its instance IS the callable), so the API keeps a readable named adapter + while satisfying the callable contract. + """ + + def __init__( + self, + *, + project_id: UUID, + tracing_service: Any, + ): + self.project_id = project_id + self.tracing_service = tracing_service + + async def __call__(self, trace_id: str) -> Any: + return await fetch_trace( + tracing_service=self.tracing_service, + project_id=self.project_id, + trace_id=trace_id, + ) + + +class APIWorkflowRunner: + def __init__( + self, + *, + project_id: UUID, + user_id: UUID, + workflows_service: Any, + ): + self.project_id = project_id + self.user_id = user_id + self.workflows_service = workflows_service + + async def execute( + self, + request: WorkflowExecutionRequest, + ) -> WorkflowExecutionResult: + return (await self.execute_batch([request]))[0] + + async def execute_batch( + self, + requests: List[WorkflowExecutionRequest], + semaphore: Optional[Semaphore] = None, + ) -> List[WorkflowExecutionResult]: + async def _guarded( + request: WorkflowExecutionRequest, + ) -> WorkflowExecutionResult: + if semaphore is not None: + async with semaphore: + return await self._execute_one(request) + return await self._execute_one(request) + + return list(await gather(*(_guarded(r) for r in requests))) + + async def _execute_one( + self, + request: WorkflowExecutionRequest, + ) -> WorkflowExecutionResult: + revision = request.revision + data = _read_field(revision, "data") + if isinstance(revision, dict): + revision_dump = revision + elif hasattr(revision, "model_dump"): + revision_dump = revision.model_dump(mode="json", exclude_none=True) + else: + revision_dump = revision + + parameters = _read_field(data, "parameters") if data else None + flags = _read_field(revision, "flags") + flags = ( + _dump_model( + flags, + mode="json", + exclude_none=True, + exclude_unset=True, + ) + if flags + else None + ) + response = await self.workflows_service.invoke_workflow( + project_id=self.project_id, + user_id=self.user_id, + request=WorkflowServiceRequest( + version="2025.07.14", + flags=flags, + data=WorkflowServiceRequestData( + revision=revision_dump, + parameters=parameters, + testcase=( + request.source.testcase.model_dump( + mode="json", + exclude_none=True, + ) + if hasattr(request.source.testcase, "model_dump") + else request.source.testcase + ), + inputs=_project_inputs(request.source.inputs, data), + trace=( + request.upstream_trace.model_dump( + mode="json", + exclude_none=True, + ) + if hasattr(request.upstream_trace, "model_dump") + else request.upstream_trace + ), + outputs=request.upstream_outputs or request.source.outputs, + ), + references=_dump_json(request.references), + links=request.links or {}, + ), + ) + status = getattr(response, "status", None) + status_code = getattr(status, "code", None) + has_error = status_code != 200 + return WorkflowExecutionResult( + status=( + SdkEvaluationStatus.FAILURE + if has_error + else SdkEvaluationStatus.SUCCESS + ), + trace_id=getattr(response, "trace_id", None), + span_id=getattr(response, "span_id", None), + error=( + status.model_dump(mode="json", exclude_none=True) + if has_error and hasattr(status, "model_dump") + else {"code": status_code} + if has_error + else None + ), + outputs=getattr(response, "outputs", None), + ) + + +class APIEvaluatorRunner(APIWorkflowRunner): + def __init__( + self, + *, + project_id: UUID, + user_id: UUID, + workflows_service: Any, + ): + super().__init__( + project_id=project_id, + user_id=user_id, + workflows_service=workflows_service, + ) + + +class APICachedRunner: + def __init__( + self, + *, + runner: Any, + tracing_service: Any, + project_id: UUID, + enabled: bool, + ): + self.runner = runner + self.tracing_service = tracing_service + self.project_id = project_id + self.enabled = enabled + self.cache_resolver = RunnableCacheResolver() + + async def execute( + self, + request: WorkflowExecutionRequest, + ) -> WorkflowExecutionResult: + return (await self.execute_batch([request]))[0] + + async def execute_batch( + self, + requests: List[WorkflowExecutionRequest], + semaphore: Optional[Semaphore] = None, + ) -> List[WorkflowExecutionResult]: + results: List[Optional[WorkflowExecutionResult]] = [None] * len(requests) + missing: List[WorkflowExecutionRequest] = [] + missing_positions: List[int] = [] + + for idx, request in enumerate(requests): + cache = await self.cache_resolver.resolve( + tracing_service=self.tracing_service, + project_id=self.project_id, + enabled=self.enabled and self.tracing_service is not None, + references=request.references, + links=request.links, + required_count=1, + ) + reusable = cache.reusable_traces[0] if cache.reusable_traces else None + if reusable and getattr(reusable, "trace_id", None): + results[idx] = WorkflowExecutionResult( + status=SdkEvaluationStatus.SUCCESS, + trace_id=str(reusable.trace_id), + trace=reusable, + ) + continue + + missing.append(request) + missing_positions.append(idx) + + if missing: + executed = await self.runner.execute_batch(missing, semaphore=semaphore) + for idx, execution in zip(missing_positions, executed): + results[idx] = execution + + return [result for result in results if result is not None] diff --git a/api/oss/src/core/evaluations/runtime/cache.py b/api/oss/src/core/evaluations/runtime/cache.py new file mode 100644 index 0000000000..805e4564f5 --- /dev/null +++ b/api/oss/src/core/evaluations/runtime/cache.py @@ -0,0 +1,60 @@ +from typing import Any, Dict, List, Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + +from oss.src.core.evaluations.utils import ( + fetch_traces_by_hash, + make_hash, + plan_missing_traces, + select_traces_for_reuse, +) + + +class CacheResolution(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + hash_id: Optional[str] + reusable_traces: List[Any] + missing_count: int + + +class RunnableCacheResolver: + async def resolve( + self, + *, + tracing_service: Any, + project_id: UUID, + enabled: bool, + references: Optional[Dict[str, Any]] = None, + links: Optional[Dict[str, Any]] = None, + required_count: int = 1, + ) -> CacheResolution: + hash_id = make_hash(references=references, links=links) + + if not enabled or not hash_id or required_count <= 0: + return CacheResolution( + hash_id=hash_id, + reusable_traces=[], + missing_count=max(0, required_count), + ) + + cached_traces = await fetch_traces_by_hash( + tracing_service, + project_id, + hash_id=hash_id, + limit=required_count, + ) + reusable_traces = select_traces_for_reuse( + traces=cached_traces, + required_count=required_count, + ) + + return CacheResolution( + hash_id=hash_id, + reusable_traces=reusable_traces, + missing_count=plan_missing_traces( + required_count=required_count, + reusable_count=len(reusable_traces), + ), + ) diff --git a/api/oss/src/core/evaluations/runtime/locks.py b/api/oss/src/core/evaluations/runtime/locks.py index 33cff2b4f0..3309d791e6 100644 --- a/api/oss/src/core/evaluations/runtime/locks.py +++ b/api/oss/src/core/evaluations/runtime/locks.py @@ -21,6 +21,7 @@ from pydantic import BaseModel import oss.src.utils.caching as caching +import oss.src.utils.locking as locking from oss.src.utils.logging import get_module_logger log = get_module_logger(__name__) @@ -121,10 +122,10 @@ async def _write_meta( payload: LockPayload, ttl: int, ) -> None: - await caching.r_lock.set( + await locking.set_key( _actual_meta_name(lock_key), orjson.dumps(payload.model_dump(mode="json")), - ex=ttl, + ttl=ttl, ) @@ -134,7 +135,7 @@ async def _touch_meta( ttl: int, ) -> None: meta_key = _actual_meta_name(lock_key) - raw = await caching.r_lock.get(meta_key) + raw = await locking.get_key(meta_key) if not raw: return @@ -145,10 +146,10 @@ async def _touch_meta( return payload.updated_at = _now_iso() - await caching.r_lock.set( + await locking.set_key( meta_key, orjson.dumps(payload.model_dump(mode="json")), - ex=ttl, + ttl=ttl, ) @@ -157,11 +158,11 @@ async def _read_meta_if_lock_exists( lock_key: str, ) -> Optional[LockPayload]: actual_lock_key = _actual_lock_name(lock_key) - if not await caching.r_lock.exists(actual_lock_key): - await caching.r_lock.delete(_actual_meta_name(lock_key)) + if not await locking.has_key(actual_lock_key): + await locking.delete_key(_actual_meta_name(lock_key)) return None - raw = await caching.r_lock.get(_actual_meta_name(lock_key)) + raw = await locking.get_key(_actual_meta_name(lock_key)) if not raw: return None @@ -180,7 +181,7 @@ async def _acquire_lock( ttl: int, ) -> Optional[LockPayload]: namespace, key = _lock_args(lock_key) - job_token = await caching.acquire_lock( + job_token = await locking.acquire_lock( namespace=namespace, key=key, ttl=ttl, @@ -207,7 +208,7 @@ async def _acquire_lock( job_id=job_id, exc_info=True, ) - await caching.release_lock( + await locking.release_lock( namespace=namespace, key=key, owner=job_token, @@ -224,7 +225,7 @@ async def _renew_lock( ttl: int, ) -> bool: namespace, key = _lock_args(lock_key) - renewed = await caching.renew_lock( + renewed = await locking.renew_lock( namespace=namespace, key=key, ttl=ttl, @@ -254,7 +255,7 @@ async def _release_lock( job_token: str, ) -> bool: namespace, key = _lock_args(lock_key) - released = await caching.release_lock( + released = await locking.release_lock( namespace=namespace, key=key, owner=job_token, @@ -263,7 +264,7 @@ async def _release_lock( return False try: - await caching.r_lock.delete(_actual_meta_name(lock_key)) + await locking.delete_key(_actual_meta_name(lock_key)) except Exception: log.warning( "[LOCK] Released lock but failed to delete metadata", @@ -367,7 +368,7 @@ async def list_active_job_locks( Wildcard discovery must use SCAN, never KEYS. """ payloads: list[LockPayload] = [] - async for raw_lock_key in caching.r_lock.scan_iter( + async for raw_lock_key in locking.scan_keys( match=_actual_lock_name(job_lock_pattern(run_id)) ): meta_key = ( @@ -375,7 +376,7 @@ async def list_active_job_locks( if isinstance(raw_lock_key, bytes) else f"{raw_lock_key}:meta" ) - raw_payload = await caching.r_lock.get(meta_key) + raw_payload = await locking.get_key(meta_key) if not raw_payload: continue @@ -403,9 +404,7 @@ async def is_run_executing( *, run_id: str, ) -> bool: - async for _ in caching.r_lock.scan_iter( - match=_actual_lock_name(job_lock_pattern(run_id)) - ): + async for _ in locking.scan_keys(match=_actual_lock_name(job_lock_pattern(run_id))): return True return False @@ -414,7 +413,7 @@ async def has_mutation_lock( *, run_id: str, ) -> bool: - return bool(await caching.r_lock.exists(_actual_lock_name(run_lock_key(run_id)))) + return await locking.has_key(_actual_lock_name(run_lock_key(run_id))) async def refresh_worker_heartbeat( @@ -424,7 +423,7 @@ async def refresh_worker_heartbeat( ) -> WorkerHeartbeatPayload: now = _now_iso() hb_key = _actual_lock_name(worker_heartbeat_key(worker_id)) - raw = await caching.r_lock.get(hb_key) + raw = await locking.get_key(hb_key) created_at = now if raw: @@ -442,10 +441,10 @@ async def refresh_worker_heartbeat( created_at=created_at, updated_at=now, ) - await caching.r_lock.set( + await locking.set_key( hb_key, orjson.dumps(payload.model_dump(mode="json")), - ex=ttl, + ttl=ttl, ) return payload diff --git a/api/oss/src/core/evaluations/runtime/models.py b/api/oss/src/core/evaluations/runtime/models.py new file mode 100644 index 0000000000..ba92d4ea1e --- /dev/null +++ b/api/oss/src/core/evaluations/runtime/models.py @@ -0,0 +1,131 @@ +from typing import Any, Dict, List, Literal, Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from oss.src.core.evaluations.types import EvaluationStatus, Origin, Type + +InputSourceKind = Literal["query", "testset", "trace", "testcase", "direct"] +SourceBatchKind = Literal["traces", "testcases"] +TopologyStatus = Literal["supported", "potential", "not_planned", "unsupported"] +DispatchKind = Literal[ + "batch_query", + "batch_testset", + "batch_invocation", + "queue_traces", + "queue_testcases", + "live_query", +] +SliceProcessMode = Literal["fill-missing", "force"] + + +class RuntimeModel(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + +class InputSourceSpec(RuntimeModel): + kind: InputSourceKind + step_key: str + references: Dict[str, Any] = Field(default_factory=dict) + + +class ResolvedSourceItem(RuntimeModel): + kind: InputSourceKind + step_key: str + references: Dict[str, Any] = Field(default_factory=dict) + trace_id: Optional[str] = None + span_id: Optional[str] = None + testcase_id: Optional[UUID] = None + testcase: Optional[Any] = None + trace: Optional[Any] = None + inputs: Optional[Any] = None + outputs: Optional[Any] = None + + +class ResolvedSourceBatch(RuntimeModel): + kind: SourceBatchKind + step_key: str + trace_ids: List[str] = Field(default_factory=list) + testcase_ids: List[UUID] = Field(default_factory=list) + + +class ResolvedTestsetInputSpec(RuntimeModel): + step_key: str + testset: Any + testset_revision: Any + testcases: List[Any] = Field(default_factory=list) + testcases_data: List[Dict[str, Any]] = Field(default_factory=list) + + +class ScenarioBinding(RuntimeModel): + scenario_id: UUID + source: ResolvedSourceItem + interval: Optional[int] = None + timestamp: Optional[Any] = None + + +class EvaluationStep(RuntimeModel): + key: str + type: Type + origin: Origin + references: Dict[str, Any] = Field(default_factory=dict) + inputs: List[str] = Field(default_factory=list) + + +class TensorSlice(RuntimeModel): + run_id: UUID + scenario_ids: Optional[List[UUID]] = None + step_keys: Optional[List[str]] = None + repeat_idxs: Optional[List[int]] = None + process_mode: SliceProcessMode = "fill-missing" + + +class TensorProbeSummary(RuntimeModel): + existing_count: int = 0 + missing_count: int = 0 + success_count: int = 0 + failure_count: int = 0 + pending_count: int = 0 + any_count: int = 0 + + +class PlannedCell(RuntimeModel): + run_id: UUID + scenario_id: UUID + step_key: str + step_type: Type + origin: Origin + repeat_idx: int + status: EvaluationStatus + should_execute: bool = False + trace_id: Optional[str] = None + span_id: Optional[str] = None + testcase_id: Optional[UUID] = None + error: Optional[Dict[str, Any]] = None + + +class ExecutionPlan(RuntimeModel): + run_id: UUID + cells: List[PlannedCell] + + @property + def executable_cells(self) -> List[PlannedCell]: + return [cell for cell in self.cells if cell.should_execute] + + +class ProcessSummary(RuntimeModel): + created: int = 0 + reused: int = 0 + pending: int = 0 + failed: int = 0 + # A coordinate whose input was never populated (no trace_id/testcase_id and + # no internal reference): there is nothing to run, so the line is SKIPPED — + # distinct from `failed`, which means execution was attempted and errored. + skipped: int = 0 + + +class TopologyDecision(RuntimeModel): + status: TopologyStatus + label: str + reason: str + dispatch: Optional[DispatchKind] = None diff --git a/api/oss/src/core/evaluations/runtime/planner.py b/api/oss/src/core/evaluations/runtime/planner.py new file mode 100644 index 0000000000..46de7bd0ca --- /dev/null +++ b/api/oss/src/core/evaluations/runtime/planner.py @@ -0,0 +1,146 @@ +from typing import Dict, Iterable, List, Optional +from uuid import UUID + +from oss.src.core.evaluations.runtime.models import ( + EvaluationStep, + ExecutionPlan, + PlannedCell, + ResolvedSourceItem, + ScenarioBinding, +) +from oss.src.core.evaluations.types import ( + EvaluationResultCreate, + EvaluationRun, + EvaluationRunDataStep, + EvaluationStatus, +) +from agenta.sdk.evaluations.runtime.planner import ( + EvaluationPlanner as SdkEvaluationPlanner, +) + + +def _step_inputs(step: EvaluationRunDataStep) -> List[str]: + return [step_input.key for step_input in (step.inputs or []) if step_input.key] + + +def normalize_steps( + steps: Optional[Iterable[EvaluationRunDataStep]], +) -> List[EvaluationStep]: + return [ + EvaluationStep( + key=step.key, + type=step.type, + origin=step.origin, + references=step.references or {}, + inputs=_step_inputs(step), + ) + for step in (steps or []) + ] + + +def make_scenario_bindings( + *, + scenario_ids: List[UUID], + source_items: List[ResolvedSourceItem], +) -> List[ScenarioBinding]: + if len(scenario_ids) != len(source_items): + raise ValueError("scenario_ids and source_items must have the same length") + + return [ + ScenarioBinding(scenario_id=scenario_id, source=source_item) + for scenario_id, source_item in zip(scenario_ids, source_items) + ] + + +class EvaluationPlanner: + """Backend DTO adapter around the SDK-owned runtime planner.""" + + def plan( + self, + *, + run: EvaluationRun, + bindings: List[ScenarioBinding], + ) -> ExecutionPlan: + if not run.id: + raise ValueError("run.id is required") + + steps = normalize_steps(run.data.steps if run.data else None) + flags = run.flags + + sdk_plan = SdkEvaluationPlanner().plan_bindings( + run_id=run.id, + bindings=bindings, # type: ignore[arg-type] + steps=steps, # type: ignore[arg-type] + repeats=run.data.repeats if run.data else None, + is_split=bool(flags and flags.is_split), + is_live=bool(flags and flags.is_live), + has_traces=bool(flags and flags.has_traces), + has_testcases=bool(flags and flags.has_testcases), + ) + + return ExecutionPlan( + run_id=sdk_plan.run_id, + cells=[ + PlannedCell( + run_id=cell.run_id, + scenario_id=cell.scenario_id, + step_key=cell.step_key, + step_type=cell.step_type, + origin=cell.origin, + repeat_idx=cell.repeat_idx, + status=EvaluationStatus(cell.status.value), + should_execute=cell.should_execute, + trace_id=cell.trace_id, + span_id=cell.span_id, + testcase_id=cell.testcase_id, + error=cell.error, + ) + for cell in sdk_plan.cells + ], + ) + + +def index_cells_by_slot( + plan: ExecutionPlan, +) -> Dict[tuple[UUID, str, int], PlannedCell]: + return { + (cell.scenario_id, cell.step_key, cell.repeat_idx): cell for cell in plan.cells + } + + +def planned_cells_to_result_creates( + cells: Iterable[PlannedCell], +) -> List[EvaluationResultCreate]: + return [ + EvaluationResultCreate( + run_id=cell.run_id, + scenario_id=cell.scenario_id, + step_key=cell.step_key, + repeat_idx=cell.repeat_idx, + status=cell.status, + trace_id=cell.trace_id, + testcase_id=cell.testcase_id, + error=cell.error, + ) + for cell in cells + ] + + +def plan_source_input_result_creates( + *, + run: EvaluationRun, + scenario_id: UUID, + source_item: ResolvedSourceItem, +) -> List[EvaluationResultCreate]: + plan = EvaluationPlanner().plan( + run=run, + bindings=make_scenario_bindings( + scenario_ids=[scenario_id], + source_items=[source_item], + ), + ) + return planned_cells_to_result_creates( + cell + for cell in plan.cells + if cell.step_type == "input" and cell.step_key == source_item.step_key + ) diff --git a/api/oss/src/core/evaluations/runtime/runner.py b/api/oss/src/core/evaluations/runtime/runner.py new file mode 100644 index 0000000000..60cd4313cd --- /dev/null +++ b/api/oss/src/core/evaluations/runtime/runner.py @@ -0,0 +1,91 @@ +from datetime import datetime +from typing import Any, List, Optional +from uuid import UUID + +from agenta.sdk.evaluations.runtime.executor import EvaluationTaskRunner + + +class TaskiqEvaluationTaskRunner(EvaluationTaskRunner): + """API adapter from generic evaluation dispatch to Taskiq tasks.""" + + def __init__(self, *, worker: Any): + self.worker = worker + + async def process_run_from_source( + self, + *, + project_id: UUID, + user_id: UUID, + run_id: UUID, + newest: Optional[datetime] = None, + oldest: Optional[datetime] = None, + ) -> Any: + kwargs = dict( + project_id=project_id, + user_id=user_id, + run_id=run_id, + ) + if newest is not None: + kwargs["newest"] = newest + if oldest is not None: + kwargs["oldest"] = oldest + + return await self.worker.process_run_from_source.kiq(**kwargs) + + async def process_run_from_batch( + self, + *, + project_id: UUID, + user_id: UUID, + run_id: UUID, + source_kind: str, + trace_ids: Optional[List[str]] = None, + testcase_ids: Optional[List[UUID]] = None, + input_step_key: Optional[str] = None, + ) -> Any: + kwargs = dict( + project_id=project_id, + user_id=user_id, + run_id=run_id, + source_kind=source_kind, + ) + if trace_ids is not None: + kwargs["trace_ids"] = trace_ids + if testcase_ids is not None: + kwargs["testcase_ids"] = testcase_ids + if input_step_key is not None: + kwargs["input_step_key"] = input_step_key + + return await self.worker.process_run_from_batch.kiq(**kwargs) + + async def process_rerun( + self, + *, + project_id: UUID, + user_id: UUID, + run_id: UUID, + scenario_ids: Optional[List[UUID]] = None, + step_keys: Optional[List[str]] = None, + repeat_idxs: Optional[List[int]] = None, + process_mode: Optional[str] = None, + ) -> Any: + # Re-execute EXISTING scenarios by coordinate (the tensor process(slice) + # op): retry transiently-failed scenarios, run a newly-added evaluator + # over existing scenarios, re-run a specific repeat, etc. Distinct verb + # from process_run_from_batch, which ingests NEW source items into NEW + # scenarios. + kwargs: dict = dict( + project_id=project_id, + user_id=user_id, + run_id=run_id, + ) + if scenario_ids is not None: + kwargs["scenario_ids"] = scenario_ids + if step_keys is not None: + kwargs["step_keys"] = step_keys + if repeat_idxs is not None: + kwargs["repeat_idxs"] = repeat_idxs + if process_mode is not None: + kwargs["process_mode"] = process_mode + + return await self.worker.process_rerun.kiq(**kwargs) diff --git a/api/oss/src/core/evaluations/runtime/sources.py b/api/oss/src/core/evaluations/runtime/sources.py new file mode 100644 index 0000000000..b3f25b4994 --- /dev/null +++ b/api/oss/src/core/evaluations/runtime/sources.py @@ -0,0 +1,481 @@ +from datetime import datetime +from typing import Any, Dict, List, Optional +from uuid import UUID + +from oss.src.core.evaluations.runtime.models import ( + ResolvedSourceBatch, + ResolvedSourceItem, + ResolvedTestsetInputSpec, +) +from oss.src.core.evaluations.types import EvaluationRun, EvaluationRunDataStep +from oss.src.core.evaluations.utils import fetch_trace +from oss.src.core.shared.dtos import Reference +from oss.src.core.tracing.dtos import ( + Filtering, + Windowing, + Formatting, + Format, + Focus, + TracingQuery, + LogicalOperator, +) + + +def _extract_root_span(trace: Any) -> Optional[Any]: + spans = ( + trace.get("spans") if isinstance(trace, dict) else getattr(trace, "spans", None) + ) + if not isinstance(spans, dict) or not spans: + return None + + for span in spans.values(): + if isinstance(span, list): + continue + if _extract_span_id(span): + return span + + return None + + +def _extract_span_id(span: Any) -> Optional[str]: + span_id = ( + span.get("span_id") + if isinstance(span, dict) + else getattr(span, "span_id", None) + ) + return str(span_id) if span_id else None + + +def _extract_ag_data(trace: Any) -> Dict[str, Any]: + root_span = _extract_root_span(trace) + if root_span is None: + return {} + + attributes = ( + root_span.get("attributes", {}) + if isinstance(root_span, dict) + else getattr(root_span, "attributes", {}) + ) + if hasattr(attributes, "model_dump"): + attributes = attributes.model_dump(mode="json", exclude_none=True) + if not isinstance(attributes, dict): + return {} + + ag = attributes.get("ag") or {} + data = ag.get("data") if isinstance(ag, dict) else {} + return data if isinstance(data, dict) else {} + + +class SourceResolutionError(Exception): + """An input step does not carry exactly one recognized source reference.""" + + pass + + +class SourceResolver: + # The exact reference key this resolver handles. The dispatch loop selects a + # resolver by which key the step carries, not by which one happens to return + # a non-empty batch. + source_reference_key: str = "" + + def applies(self, step: EvaluationRunDataStep) -> bool: + refs = step.references or {} + return self.source_reference_key in refs + + async def resolve( + self, + *, + project_id: UUID, + step: EvaluationRunDataStep, + ) -> Optional[ResolvedSourceBatch]: + raise NotImplementedError + + +class QueryRevisionTraceResolver(SourceResolver): + source_reference_key = "query_revision" + + def __init__(self, *, queries_service: Any): + self.queries_service = queries_service + + async def resolve( + self, + *, + project_id: UUID, + step: EvaluationRunDataStep, + ) -> Optional[ResolvedSourceBatch]: + refs = step.references or {} + query_revision_ref = refs.get("query_revision") + + if not step.key or not query_revision_ref or not query_revision_ref.id: + return None + + query_revision = await self.queries_service.fetch_query_revision( + project_id=project_id, + query_revision_ref=query_revision_ref, + include_trace_ids=True, + ) + trace_ids = ( + query_revision.data.trace_ids + if query_revision and query_revision.data and query_revision.data.trace_ids + else [] + ) + + if not trace_ids: + return None + + return ResolvedSourceBatch( + kind="traces", + step_key=step.key, + trace_ids=trace_ids, + ) + + +class TestsetRevisionTestcaseResolver(SourceResolver): + source_reference_key = "testset_revision" + + def __init__(self, *, testsets_service: Any): + self.testsets_service = testsets_service + + async def resolve( + self, + *, + project_id: UUID, + step: EvaluationRunDataStep, + ) -> Optional[ResolvedSourceBatch]: + refs = step.references or {} + testset_revision_ref = refs.get("testset_revision") + + if not step.key or not testset_revision_ref or not testset_revision_ref.id: + return None + + testset_revision = await self.testsets_service.fetch_testset_revision( + project_id=project_id, + testset_revision_ref=testset_revision_ref, + include_testcase_ids=True, + ) + testcase_ids = ( + testset_revision.data.testcase_ids + if testset_revision + and testset_revision.data + and testset_revision.data.testcase_ids + else [] + ) + + if not testcase_ids: + return None + + return ResolvedSourceBatch( + kind="testcases", + step_key=step.key, + testcase_ids=testcase_ids, + ) + + +class TestsetRevisionPayloadResolver: + def __init__(self, *, testsets_service: Any): + self.testsets_service = testsets_service + + async def resolve( + self, + *, + project_id: UUID, + step: EvaluationRunDataStep, + ) -> ResolvedTestsetInputSpec: + refs = step.references or {} + testset_revision_ref = refs.get("testset_revision") + + if not testset_revision_ref or not isinstance(testset_revision_ref.id, UUID): + raise ValueError( + f"Evaluation input step {step.key} missing testset_revision reference." + ) + + testset_revision = await self.testsets_service.fetch_testset_revision( + project_id=project_id, + testset_revision_ref=testset_revision_ref, + ) + if not testset_revision: + raise ValueError( + f"Testset revision with id {testset_revision_ref.id} not found!" + ) + if not testset_revision.data or not testset_revision.data.testcases: + raise ValueError( + f"Testset revision with id {testset_revision_ref.id} has no testcases!" + ) + + testset_variant = await self.testsets_service.fetch_testset_variant( + project_id=project_id, + testset_variant_ref=Reference(id=testset_revision.variant_id), + ) + if not testset_variant: + raise ValueError( + f"Testset variant with id {testset_revision.variant_id} not found!" + ) + + testset = await self.testsets_service.fetch_testset( + project_id=project_id, + testset_ref=Reference(id=testset_variant.testset_id), + ) + if not testset: + raise ValueError(f"Testset with id {testset_variant.testset_id} not found!") + + testcases = testset_revision.data.testcases + return ResolvedTestsetInputSpec( + step_key=step.key, + testset=testset, + testset_revision=testset_revision, + testcases=testcases, + testcases_data=[ + {**testcase.data, "testcase_id": str(testcase.id)} + for testcase in testcases + ], + ) + + +async def resolve_queue_source_batches( + *, + project_id: UUID, + run: EvaluationRun, + queries_service: Any, + testsets_service: Any, +) -> List[ResolvedSourceBatch]: + if not run.data or not run.data.steps: + return [] + + resolvers: List[SourceResolver] = [ + QueryRevisionTraceResolver(queries_service=queries_service), + TestsetRevisionTestcaseResolver(testsets_service=testsets_service), + ] + batches: List[ResolvedSourceBatch] = [] + + for step in run.data.steps: + if step.type != "input" or not step.key: + continue + + # Exactly one recognized source reference per input step. Selecting by + # the applicable key (not by first non-empty result) means an empty + # result — a query with zero traces — stays an empty batch instead of + # falling through to the wrong resolver. + applicable = [resolver for resolver in resolvers if resolver.applies(step)] + + if not applicable: + continue + + if len(applicable) > 1: + raise SourceResolutionError( + f"Input step '{step.key}' carries multiple source references " + f"({', '.join(r.source_reference_key for r in applicable)}); " + "exactly one is allowed." + ) + + batch = await applicable[0].resolve( + project_id=project_id, + step=step, + ) + if batch: + batches.append(batch) + + return batches + + +async def resolve_testset_input_specs( + *, + project_id: UUID, + input_steps: List[EvaluationRunDataStep], + testsets_service: Any, +) -> List[ResolvedTestsetInputSpec]: + resolver = TestsetRevisionPayloadResolver(testsets_service=testsets_service) + return [ + await resolver.resolve( + project_id=project_id, + step=input_step, + ) + for input_step in input_steps + ] + + +async def resolve_direct_source_items( + *, + project_id: UUID, + trace_ids: Optional[List[str]] = None, + testcase_ids: Optional[List[UUID]] = None, + testcases_service: Any = None, + tracing_service: Any = None, +) -> List[ResolvedSourceItem]: + source_items: List[ResolvedSourceItem] = [] + testcase_ids = testcase_ids or [] + trace_ids = trace_ids or [] + + testcases = ( + await testcases_service.fetch_testcases( + project_id=project_id, + testcase_ids=testcase_ids, + ) + if testcase_ids and testcases_service is not None + else [] + ) + testcases_by_id = { + testcase.id: testcase for testcase in testcases if getattr(testcase, "id", None) + } + traces_by_id: Dict[str, Any] = {} + + if trace_ids and tracing_service is not None: + for trace_id in trace_ids: + trace = await fetch_trace( + tracing_service=tracing_service, + project_id=project_id, + trace_id=trace_id, + max_retries=1, + delay=0, + ) + if trace is not None: + traces_by_id[trace_id] = trace + + source_items.extend( + ResolvedSourceItem( + kind="testcase", + step_key="", + testcase=testcases_by_id.get(testcase_id), + testcase_id=testcase_id, + ) + for testcase_id in testcase_ids + ) + for trace_id in trace_ids: + trace = traces_by_id.get(trace_id) + ag_data = _extract_ag_data(trace) if trace is not None else {} + root_span = _extract_root_span(trace) if trace is not None else None + source_items.append( + ResolvedSourceItem( + kind="trace", + step_key="", + trace_id=trace_id, + span_id=_extract_span_id(root_span), + trace=trace, + inputs=ag_data.get("inputs"), + outputs=ag_data.get("outputs"), + ) + ) + + return source_items + + +async def resolve_live_query_traces( + *, + project_id: UUID, + query_revisions: Dict[str, Any], + tracing_service: Any, + newest: Optional[datetime] = None, + oldest: Optional[datetime] = None, + use_windowing: bool = False, +) -> Dict[str, List[Any]]: + query_traces: Dict[str, List[Any]] = {} + + for query_step_key, query_revision in query_revisions.items(): + formatting = Formatting( + focus=Focus.TRACE, + format=Format.AGENTA, + ) + filtering = Filtering( + operator=LogicalOperator.AND, + conditions=[], + ) + windowing = Windowing( + oldest=oldest, + newest=newest, + next=None, + limit=None, + order="ascending", + interval=None, + rate=None, + ) + + query_revision_data = getattr(query_revision, "data", None) + if query_revision_data: + query_filtering = getattr(query_revision_data, "filtering", None) + query_windowing = getattr(query_revision_data, "windowing", None) + + if query_filtering: + filtering = query_filtering + + if query_windowing and use_windowing: + windowing = Windowing( + oldest=query_windowing.oldest, + newest=query_windowing.newest, + limit=query_windowing.limit, + order=query_windowing.order, + rate=query_windowing.rate, + ) + elif query_windowing: + windowing.rate = query_windowing.rate + + query_traces[query_step_key] = ( + await tracing_service.query_traces( + project_id=project_id, + query=TracingQuery( + formatting=formatting, + filtering=filtering, + windowing=windowing, + ), + ) + or [] + ) + + return query_traces + + +async def resolve_query_source_items( + *, + project_id: UUID, + run: EvaluationRun, + queries_service: Any, + tracing_service: Any, + newest: Optional[datetime] = None, + oldest: Optional[datetime] = None, + use_windowing: bool = False, +) -> Dict[str, List[ResolvedSourceItem]]: + if not run.data or not run.data.steps: + return {} + + query_revisions: Dict[str, Any] = {} + for step in run.data.steps: + if step.type != "input" or not step.key: + continue + + query_revision_ref = (step.references or {}).get("query_revision") + if not query_revision_ref: + continue + + query_revision = await queries_service.fetch_query_revision( + project_id=project_id, + query_revision_ref=query_revision_ref, + ) + if ( + not query_revision + or not getattr(query_revision, "id", None) + or not getattr(query_revision, "slug", None) + ): + continue + + query_revisions[step.key] = query_revision + + query_traces = await resolve_live_query_traces( + project_id=project_id, + query_revisions=query_revisions, + tracing_service=tracing_service, + newest=newest, + oldest=oldest, + use_windowing=use_windowing, + ) + + return { + query_step_key: [ + ResolvedSourceItem( + kind="trace", + step_key=query_step_key, + trace_id=trace.trace_id, + trace=trace, + ) + for trace in traces + if trace and trace.trace_id + ] + for query_step_key, traces in query_traces.items() + } diff --git a/api/oss/src/core/evaluations/runtime/tensor.py b/api/oss/src/core/evaluations/runtime/tensor.py new file mode 100644 index 0000000000..ec995fee0b --- /dev/null +++ b/api/oss/src/core/evaluations/runtime/tensor.py @@ -0,0 +1,313 @@ +from typing import List, Optional, Protocol +from uuid import UUID + +from oss.src.core.evaluations.runtime.models import ( + ProcessSummary, + TensorProbeSummary, + TensorSlice, +) +from oss.src.core.evaluations.types import ( + EvaluationMetricsRefresh, + EvaluationResult, + EvaluationResultCreate, + EvaluationResultQuery, + EvaluationScenarioQuery, + EvaluationStatus, +) + + +def _empty_dimension(values: Optional[List[object]]) -> bool: + return values == [] + + +def _slice_is_empty(tensor_slice: TensorSlice) -> bool: + return any( + _empty_dimension(values) + for values in ( + tensor_slice.scenario_ids, + tensor_slice.step_keys, + tensor_slice.repeat_idxs, + ) + ) + + +def _query_from_slice(tensor_slice: TensorSlice) -> EvaluationResultQuery: + return EvaluationResultQuery( + run_id=tensor_slice.run_id, + scenario_ids=tensor_slice.scenario_ids, + step_keys=tensor_slice.step_keys, + repeat_idxs=tensor_slice.repeat_idxs, + ) + + +class SliceProcessor(Protocol): + """Execution boundary for `process(slice)`. + + A slice processor takes the canonical output coordinate (existing + scenarios x steps x repeats) and re-executes the runnable cells in that + scope: it plans from the scenarios' existing source bindings, restricts to + the requested `step_keys`/`repeat_idxs`, invokes only missing work, and + populates the result cells. It does NOT refresh metrics — that is the + separate `refresh` op, invoked by the caller on the right boundary. + + It is deliberately adapter-free at this seam so `runtime/` does not depend on + `tasks/`: the concrete implementation (which closes over the tracing / + testcases / workflows / applications services and the run's revisions) is + wired at the composition root and injected here. The same shape lets the SDK + or an in-memory test supply its own processor without changing tensor ops. + """ + + async def process( + self, + *, + project_id: UUID, + user_id: UUID, + tensor_slice: TensorSlice, + ) -> ProcessSummary: ... + + +class TensorSliceOperations: + def __init__( + self, + *, + evaluations_service, + slice_processor: Optional[SliceProcessor] = None, + ): + self.evaluations_service = evaluations_service + self.slice_processor = slice_processor + + async def probe( + self, + *, + project_id: UUID, + tensor_slice: TensorSlice, + ) -> List[EvaluationResult]: + if _slice_is_empty(tensor_slice): + return [] + + return await self.evaluations_service.query_results( + project_id=project_id, + result=_query_from_slice(tensor_slice), + ) + + async def populate( + self, + *, + project_id: UUID, + user_id: UUID, + results: List[EvaluationResultCreate], + ) -> List[EvaluationResult]: + # `populate` only writes result cells. Metrics are a separate operation + # (`refresh`): the three metric kinds refresh on different boundaries + # (scenario-complete / interval / run), so the caller decides when. + if not results: + return [] + + return await self.evaluations_service.set_results( + project_id=project_id, + user_id=user_id, + results=results, + ) + + async def probe_summary( + self, + *, + project_id: UUID, + tensor_slice: TensorSlice, + expected_count: Optional[int] = None, + ) -> TensorProbeSummary: + results = await self.probe( + project_id=project_id, + tensor_slice=tensor_slice, + ) + existing_count = len(results) + expected = expected_count if expected_count is not None else existing_count + + return TensorProbeSummary( + existing_count=existing_count, + missing_count=max(0, expected - existing_count), + success_count=sum( + 1 for result in results if result.status == EvaluationStatus.SUCCESS + ), + failure_count=sum( + 1 + for result in results + if result.status + in { + EvaluationStatus.FAILURE, + EvaluationStatus.ERRORS, + } + ), + pending_count=sum( + 1 for result in results if result.status == EvaluationStatus.PENDING + ), + any_count=existing_count, + ) + + async def prune( + self, + *, + project_id: UUID, + user_id: UUID, + tensor_slice: TensorSlice, + ) -> List[UUID]: + # `prune` removes result cells, then re-triggers a metrics refresh over + # the affected scope so aggregates recompute over the now-smaller cell + # set. Every tensor-write op (populate / process / prune) re-triggers + # refresh after touching cells — prune leaves nothing stale. It does NOT + # touch steps or scenarios; those are the graph-shape ops on the service + # (add_steps/remove_steps, add_scenarios/remove_scenarios). + if _slice_is_empty(tensor_slice): + return [] + # prune only needs the ids to delete, so use the ID-only query rather + # than hydrating full result DTOs via `probe`. + result_ids = await self.evaluations_service.query_result_ids( + project_id=project_id, + result=_query_from_slice(tensor_slice), + ) + if not result_ids: + return [] + + deleted = await self.evaluations_service.delete_results( + project_id=project_id, + result_ids=result_ids, + ) + await self.refresh( + project_id=project_id, + user_id=user_id, + tensor_slice=tensor_slice, + ) + return deleted + + async def process( + self, + *, + project_id: UUID, + user_id: UUID, + tensor_slice: TensorSlice, + ) -> ProcessSummary: + """Execute the runnable cells in the slice and return what changed. + + This is the plan->execute->populate loop of the design's `process(slice)` + operation — results only. Metrics are refreshed by the separate + `refresh` op, not here. The actual execution is delegated to the + injected `slice_processor`; this method owns only the slice-level + guard (an empty dimension means "nothing addressed", so there is + nothing to execute). + + Execution requires a wired `slice_processor`. Earlier this method + silently refreshed metrics and returned an empty summary, which read as + "executed the slice" while doing almost nothing (UEL-015) — so when no + processor is wired we now fail loudly rather than masquerade. + """ + if _slice_is_empty(tensor_slice): + return ProcessSummary() + + if self.slice_processor is None: + raise NotImplementedError( + "process(slice) requires a wired slice_processor; " + "TensorSliceOperations was constructed without one. " + "Use probe/populate/prune for read/write/delete, or wire a " + "SliceProcessor at the composition root to execute the slice." + ) + + return await self.slice_processor.process( + project_id=project_id, + user_id=user_id, + tensor_slice=tensor_slice, + ) + + async def refresh( + self, + *, + project_id: UUID, + user_id: UUID, + tensor_slice: TensorSlice, + ) -> None: + """Recompute metrics over the slice's scope — variational AND aggregate. + + First-class peer of probe/process/populate/prune. The three metric kinds + refresh on different boundaries (scenario-complete, interval, run), so a + complete refresh does both layers: + + 1. variational — the per-scenario rows for the slice's scenarios; + 2. aggregate — temporal buckets (live runs, which aggregate over time) + or the single global row (non-live runs). + + Callers that already refreshed variational inside `process` can still + call this; recomputing the per-scenario rows is idempotent. Kept separate + from process/populate so writes do not implicitly recompute — the caller + invokes `refresh` once, at the right boundary. + """ + if _slice_is_empty(tensor_slice): + return + + run_id = tensor_slice.run_id + + # 1. Variational — per-scenario rows for the addressed scenarios. + await self.evaluations_service.refresh_metrics( + project_id=project_id, + user_id=user_id, + metrics=EvaluationMetricsRefresh( + run_id=run_id, + scenario_ids=tensor_slice.scenario_ids, + ), + ) + + # 2. Aggregate — temporal (live) or global (non-live). + run = await self.evaluations_service.fetch_run( + project_id=project_id, + run_id=run_id, + ) + if not run: + return + + is_live = bool(run.flags and run.flags.is_live) + + if not is_live: + # Non-live: one global aggregate (no scenario, no timestamp). + await self.evaluations_service.refresh_metrics( + project_id=project_id, + user_id=user_id, + metrics=EvaluationMetricsRefresh(run_id=run_id), + ) + return + + # Live: recompute the temporal buckets the slice touched. Group the + # slice's scenarios' timestamps by interval so each affected + # (interval, timestamp) bucket is recomputed. + scenarios = await self.evaluations_service.query_scenarios( + project_id=project_id, + scenario=EvaluationScenarioQuery( + run_id=run_id, + ids=tensor_slice.scenario_ids, + ), + ) + timestamps_by_interval: dict[int, set] = {} + for scenario in scenarios: + if scenario.timestamp is None or scenario.interval is None: + continue + timestamps_by_interval.setdefault(scenario.interval, set()).add( + scenario.timestamp + ) + + if not timestamps_by_interval: + # No temporal buckets on the slice's scenarios — fall back to the + # global aggregate so the run is not left with stale metrics. + await self.evaluations_service.refresh_metrics( + project_id=project_id, + user_id=user_id, + metrics=EvaluationMetricsRefresh(run_id=run_id), + ) + return + + for interval, timestamps in timestamps_by_interval.items(): + await self.evaluations_service.refresh_metrics( + project_id=project_id, + user_id=user_id, + metrics=EvaluationMetricsRefresh( + run_id=run_id, + timestamps=sorted(timestamps), + interval=interval, + ), + ) diff --git a/api/oss/src/core/evaluations/runtime/topology.py b/api/oss/src/core/evaluations/runtime/topology.py new file mode 100644 index 0000000000..f7950b6e0d --- /dev/null +++ b/api/oss/src/core/evaluations/runtime/topology.py @@ -0,0 +1,34 @@ +from agenta.sdk.evaluations.runtime.topology import classify_steps_topology + +from oss.src.core.evaluations.runtime.planner import normalize_steps +from oss.src.core.evaluations.runtime.models import TopologyDecision +from oss.src.core.evaluations.types import EvaluationRun + + +def classify_run_topology(run: EvaluationRun) -> TopologyDecision: + """Classify the current evaluation graph for worker dispatch. + + This is intentionally conservative. It mirrors the currently supported + worker-dispatched topologies while naming future-interest and not-planned + shapes explicitly. + """ + + steps = run.data.steps if run.data and run.data.steps else [] + flags = run.flags + + decision = classify_steps_topology( + steps=normalize_steps(steps), + is_live=bool(flags and flags.is_live), + has_queries=bool(flags and flags.has_queries), + has_testsets=bool(flags and flags.has_testsets), + has_traces=bool(flags and flags.has_traces), + has_testcases=bool(flags and flags.has_testcases), + has_evaluators=bool(flags and flags.has_evaluators), + ) + + return TopologyDecision( + status=decision.status, + label=decision.label, + reason=decision.reason, + dispatch=decision.dispatch, + ) diff --git a/api/oss/src/core/evaluations/service.py b/api/oss/src/core/evaluations/service.py index 6869e78e52..ab9066a7fc 100644 --- a/api/oss/src/core/evaluations/service.py +++ b/api/oss/src/core/evaluations/service.py @@ -20,6 +20,7 @@ EvaluationRunDataMapping, EvaluationRunDataStepInput, EvaluationRunDataStep, + EvaluationRunDataConcurrency, EvaluationRunData, EvaluationRun, EvaluationRunCreate, @@ -33,12 +34,10 @@ # EVALUATION RESULT EvaluationResult, EvaluationResultCreate, - EvaluationResultEdit, EvaluationResultQuery, # EVALUATION METRICS EvaluationMetrics, EvaluationMetricsCreate, - EvaluationMetricsEdit, EvaluationMetricsQuery, EvaluationMetricsRefresh, # EVALUATION QUEUE @@ -49,6 +48,11 @@ EvaluationQueueData, EvaluationQueueEdit, EvaluationQueueQuery, + # DEFAULT QUEUE EXCEPTIONS + DefaultQueueDataInvalid, + DefaultQueueDemotionForbidden, + DefaultQueueDeletionForbidden, + DefaultQueueArchiveForbidden, ) from oss.src.core.evaluations.types import ( Target, @@ -72,6 +76,7 @@ SimpleQueueSettings, ) from oss.src.core.evaluations.types import CURRENT_VERSION +from oss.src.core.evaluations.types import EvaluationClosedConflict from oss.src.core.tracing.dtos import ( TracingQuery, Filtering, @@ -90,6 +95,8 @@ from oss.src.core.evaluators.dtos import EvaluatorRevision from oss.src.core.queries.service import QueriesService from oss.src.core.testsets.service import TestsetsService +from oss.src.core.testcases.service import TestcasesService +from oss.src.core.workflows.service import WorkflowsService from oss.src.core.applications.service import ApplicationsService from oss.src.core.evaluations.utils import ( @@ -100,10 +107,20 @@ ) from oss.src.core.evaluations.utils import get_metrics_keys_from_schema +from oss.src.core.evaluations.runtime.topology import classify_run_topology +from oss.src.core.evaluations.runtime.sources import resolve_queue_source_batches +from oss.src.core.evaluations.runtime.runner import TaskiqEvaluationTaskRunner +from oss.src.core.evaluations.runtime.models import SliceProcessMode, TensorSlice +from oss.src.core.evaluations.runtime.tensor import TensorSliceOperations log = get_module_logger(__name__) +# Product policy toggle: when True, every evaluation run keeps a default queue +# even when it has no human evaluators. Keep this as a global until the product +# decision is finalized. +EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS = False + if TYPE_CHECKING: from oss.src.tasks.taskiq.evaluations.worker import EvaluationsWorker @@ -191,6 +208,9 @@ def __init__( testsets_service: TestsetsService, evaluators_service: EvaluatorsService, evaluations_worker: Optional["EvaluationsWorker"] = None, + testcases_service: Optional[TestcasesService] = None, + workflows_service: Optional[WorkflowsService] = None, + applications_service: Optional[ApplicationsService] = None, ): self.evaluations_dao = evaluations_dao @@ -199,6 +219,40 @@ def __init__( self.testsets_service = testsets_service self.evaluators_service = evaluators_service self.evaluations_worker = evaluations_worker + self.testcases_service = testcases_service + self.workflows_service = workflows_service + self.applications_service = applications_service + self.evaluations_task_runner = ( + TaskiqEvaluationTaskRunner(worker=evaluations_worker) + if evaluations_worker is not None + else None + ) + + # Tensor slice ops (probe/populate run in-process; process dispatches + # async via taskiq). Built here so the service owns its runtime + # collaborator, like `evaluations_task_runner` above. `APISliceProcessor` + # lives in `tasks/processor.py` which imports this module, so it is + # imported locally to avoid a circular import. Requires the sub-services + # the SDK engine needs; absent those (e.g. worker/parser contexts) the + # ops degrade to None and probe/populate no-op. + self.tensor_slice_operations: Optional[TensorSliceOperations] = None + if ( + testcases_service is not None + and workflows_service is not None + and applications_service is not None + ): + from oss.src.core.evaluations.tasks.processor import APISliceProcessor + + self.tensor_slice_operations = TensorSliceOperations( + evaluations_service=self, + slice_processor=APISliceProcessor( + evaluations_service=self, + tracing_service=tracing_service, + testcases_service=testcases_service, + workflows_service=workflows_service, + applications_service=applications_service, + ), + ) ### CRUD @@ -225,7 +279,7 @@ async def refresh_runs( log.error(e, exc_info=True) return False - if self.evaluations_worker is None: + if self.evaluations_task_runner is None: log.warning( "[LIVE] Taskiq client is not configured; skipping live run dispatch" ) @@ -266,12 +320,10 @@ async def refresh_runs( run=run, ) - await self.evaluations_worker.evaluate_live_query.kiq( + await self.evaluations_task_runner.process_run_from_source( project_id=project_id, user_id=user_id, - # run_id=run.id, - # newest=newest, oldest=oldest, ) @@ -359,34 +411,97 @@ async def _ensure_human_annotation_queue( user_id: UUID, run: EvaluationRun, ) -> None: - """Create an EvaluationQueue for human annotation steps if none exists for this run.""" - if not run.id or not run.data or not run.data.steps: - return + await self._reconcile_default_queue( + project_id=project_id, + user_id=user_id, + run=run, + ) - human_step_keys = [ - step.key - for step in run.data.steps - if step.type == "annotation" and step.origin == "human" and step.key - ] + async def fetch_default_queue( + self, + *, + project_id: UUID, + run_id: UUID, + include_archived: bool = False, + ) -> Optional[EvaluationQueue]: + queues = await self.query_queues( + project_id=project_id, + queue=EvaluationQueueQuery( + run_id=run_id, + flags=EvaluationQueueQueryFlags(is_default=True), + include_archived=include_archived, + ), + ) + return queues[0] if queues else None - if not human_step_keys: - return + async def _reconcile_default_queue( + self, + *, + project_id: UUID, + user_id: UUID, + run: EvaluationRun, + ) -> EvaluationRun: + if not run.id: + return run - existing_queues = await self.query_queues( + has_human = bool(run.flags and run.flags.has_human) + should_exist = EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS or has_human + default_queue = await self.fetch_default_queue( project_id=project_id, - queue=EvaluationQueueQuery(run_id=run.id), + run_id=run.id, + include_archived=True, ) - if any(q.run_id == run.id for q in existing_queues): - return - await self.create_queue( - project_id=project_id, - user_id=user_id, - queue=EvaluationQueueCreate( - run_id=run.id, - status=EvaluationStatus.RUNNING, - data=EvaluationQueueData(step_keys=human_step_keys), - ), + if should_exist: + if default_queue is None: + default_queue = await self.create_queue( + project_id=project_id, + user_id=user_id, + queue=EvaluationQueueCreate( + run_id=run.id, + status=EvaluationStatus.RUNNING, + flags=EvaluationQueueFlags(is_default=True), + data=EvaluationQueueData(), + ), + ) + elif default_queue.deleted_at is not None: + default_queue = await self.unarchive_queue( + project_id=project_id, + user_id=user_id, + queue_id=default_queue.id, + ) + elif default_queue is not None and default_queue.deleted_at is None: + default_queue = await self.archive_queue( + project_id=project_id, + user_id=user_id, + queue_id=default_queue.id, + force=True, + ) + + is_queue = bool( + has_human and default_queue is not None and default_queue.deleted_at is None + ) + if run.flags and run.flags.is_queue == is_queue: + return run + + flags = run.flags.model_copy() if run.flags else EvaluationRunFlags() + flags.is_queue = is_queue + return ( + await self.evaluations_dao.edit_run( + project_id=project_id, + user_id=user_id, + run=EvaluationRunEdit( + id=run.id, + name=run.name, + description=run.description, + flags=flags, + tags=run.tags, + meta=run.meta, + status=run.status, + data=run.data, + ), + ) + or run ) async def fetch_live_runs( @@ -434,6 +549,120 @@ async def has_run_mutation_lock( return await _has_mutation_lock(run_id=str(run_id)) + @staticmethod + def _step_keys(run: Optional[EvaluationRun]) -> set: + if run is None or run.data is None or not run.data.steps: + return set() + return {step.key for step in run.data.steps} + + async def _reconcile_run( + self, + *, + project_id: UUID, + user_id: UUID, + # + run: EvaluationRun, + prior_step_keys: set, + ) -> EvaluationRun: + """Bring all run-derived state in line with the run's current graph. + + This is the single post-write reconciliation path shared by create and + edit. `create_run` is just `edit_run` starting from an empty graph: it + passes `prior_step_keys=set()`, so the prune step is a no-op (there are + no prior cells), while the default-queue reconciliation runs identically + in both cases. + + Steps: + 1. prune tensor cells (and input-only scenarios + their metrics) for + any step that existed before but is gone from the current graph, + per `docs/designs/unified-eval-loops/step-removal-semantics.md`. + 2. reconcile the default queue + `is_queue` from the current graph. + """ + await self._prune_removed_steps( + project_id=project_id, + user_id=user_id, + run=run, + removed_step_keys=prior_step_keys - self._step_keys(run), + ) + + return await self._reconcile_default_queue( + project_id=project_id, + user_id=user_id, + run=run, + ) + + async def _prune_removed_steps( + self, + *, + project_id: UUID, + user_id: UUID, + # + run: EvaluationRun, + removed_step_keys: set, + ) -> None: + """Destructively prune cells, orphan scenarios, and metrics for steps + that left the graph. Removal is destructive (Model A): stored graph and + stored tensor keep the same shape. A no-op when nothing was removed. + """ + if not removed_step_keys or not run.id: + return + + removed_results = await self.query_results( + project_id=project_id, + result=EvaluationResultQuery( + run_id=run.id, + step_keys=sorted(removed_step_keys), + ), + ) + affected_scenario_ids = sorted( + {r.scenario_id for r in removed_results if r.scenario_id}, + key=str, + ) + + removed_result_ids = [r.id for r in removed_results if r.id] + if removed_result_ids: + await self.delete_results( + project_id=project_id, + result_ids=removed_result_ids, + ) + + # Scenarios sourced only from a removed step have no remaining cells. + orphan_scenario_ids: List[UUID] = [] + for scenario_id in affected_scenario_ids: + remaining = await self.query_results( + project_id=project_id, + result=EvaluationResultQuery( + run_id=run.id, + scenario_ids=[scenario_id], + ), + ) + if not remaining: + orphan_scenario_ids.append(scenario_id) + + if orphan_scenario_ids: + await self.delete_scenarios( + project_id=project_id, + scenario_ids=orphan_scenario_ids, + ) + + # Flush metrics for surviving affected scenarios so current metrics stay + # aligned with the post-removal graph. Orphans are gone. + orphans = set(orphan_scenario_ids) + surviving_scenario_ids = [ + scenario_id + for scenario_id in affected_scenario_ids + if scenario_id not in orphans + ] + if surviving_scenario_ids: + await self.refresh_metrics( + project_id=project_id, + user_id=user_id, + metrics=EvaluationMetricsRefresh( + run_id=run.id, + scenario_ids=surviving_scenario_ids, + ), + ) + async def create_run( self, *, @@ -444,12 +673,21 @@ async def create_run( ) -> Optional[EvaluationRun]: run.version = CURRENT_VERSION - return await self.evaluations_dao.create_run( + created_run = await self.evaluations_dao.create_run( project_id=project_id, user_id=user_id, # run=run, ) + if created_run: + # Create is edit from an empty graph: no prior steps to prune. + created_run = await self._reconcile_run( + project_id=project_id, + user_id=user_id, + run=created_run, + prior_step_keys=set(), + ) + return created_run async def create_runs( self, @@ -462,12 +700,21 @@ async def create_runs( for run in runs: run.version = CURRENT_VERSION - return await self.evaluations_dao.create_runs( + created_runs = await self.evaluations_dao.create_runs( project_id=project_id, user_id=user_id, # runs=runs, ) + return [ + await self._reconcile_run( + project_id=project_id, + user_id=user_id, + run=created_run, + prior_step_keys=set(), + ) + for created_run in created_runs + ] async def fetch_run( self, @@ -505,12 +752,25 @@ async def edit_run( ) -> Optional[EvaluationRun]: run.version = CURRENT_VERSION - return await self.evaluations_dao.edit_run( + # Capture the prior graph so reconciliation can prune any step the edit + # drops. An edit that omits a step is a destructive removal. + prior_run = await self.fetch_run(project_id=project_id, run_id=run.id) + prior_step_keys = self._step_keys(prior_run) + + edited_run = await self.evaluations_dao.edit_run( project_id=project_id, user_id=user_id, # run=run, ) + if edited_run: + edited_run = await self._reconcile_run( + project_id=project_id, + user_id=user_id, + run=edited_run, + prior_step_keys=prior_step_keys, + ) + return edited_run async def edit_runs( self, @@ -523,12 +783,29 @@ async def edit_runs( for run in runs: run.version = CURRENT_VERSION - return await self.evaluations_dao.edit_runs( + prior_runs = await self.fetch_runs( + project_id=project_id, + run_ids=[run.id for run in runs], + ) + prior_step_keys_by_id = { + prior_run.id: self._step_keys(prior_run) for prior_run in prior_runs + } + + edited_runs = await self.evaluations_dao.edit_runs( project_id=project_id, user_id=user_id, # runs=runs, ) + return [ + await self._reconcile_run( + project_id=project_id, + user_id=user_id, + run=edited_run, + prior_step_keys=prior_step_keys_by_id.get(edited_run.id, set()), + ) + for edited_run in edited_runs + ] async def delete_run( self, @@ -808,7 +1085,7 @@ async def create_result( result=result, ) - async def create_results( + async def set_results( self, *, project_id: UUID, @@ -819,7 +1096,7 @@ async def create_results( for result in results: result.version = CURRENT_VERSION - return await self.evaluations_dao.create_results( + return await self.evaluations_dao.set_results( project_id=project_id, user_id=user_id, # @@ -852,41 +1129,6 @@ async def fetch_results( result_ids=result_ids, ) - async def edit_result( - self, - *, - project_id: UUID, - user_id: UUID, - # - result: EvaluationResultEdit, - ) -> Optional[EvaluationResult]: - result.version = CURRENT_VERSION - - return await self.evaluations_dao.edit_result( - project_id=project_id, - user_id=user_id, - # - result=result, - ) - - async def edit_results( - self, - *, - project_id: UUID, - user_id: UUID, - # - results: List[EvaluationResultEdit], - ) -> List[EvaluationResult]: - for result in results: - result.version = CURRENT_VERSION - - return await self.evaluations_dao.edit_results( - project_id=project_id, - user_id=user_id, - # - results=results, - ) - async def delete_result( self, *, @@ -930,9 +1172,22 @@ async def query_results( windowing=windowing, ) + async def query_result_ids( + self, + *, + project_id: UUID, + # + result: Optional[EvaluationResultQuery] = None, + ) -> List[UUID]: + return await self.evaluations_dao.query_result_ids( + project_id=project_id, + # + result=result, + ) + # - EVALUATION METRIC ------------------------------------------------------ - async def create_metrics( + async def set_metrics( self, *, project_id: UUID, @@ -943,7 +1198,7 @@ async def create_metrics( for metric in metrics: metric.version = CURRENT_VERSION - return await self.evaluations_dao.create_metrics( + return await self.evaluations_dao.set_metrics( project_id=project_id, user_id=user_id, # @@ -963,24 +1218,6 @@ async def fetch_metrics( metrics_ids=metrics_ids, ) - async def edit_metrics( - self, - *, - project_id: UUID, - user_id: UUID, - # - metrics: List[EvaluationMetricsEdit], - ) -> List[EvaluationMetrics]: - for metric in metrics: - metric.version = CURRENT_VERSION - - return await self.evaluations_dao.edit_metrics( - project_id=project_id, - user_id=user_id, - # - metrics=metrics, - ) - async def delete_metrics( self, *, @@ -1178,7 +1415,15 @@ async def _refresh_metrics( steps_trace_ids[step_key] = trace_ids if not steps_trace_ids: - log.warning("[METRICS] No trace_ids found! Cannot extract metrics.") + # A human/custom annotation is run elsewhere (web / SDK), so it has no + # trace here by design — only warn if a step we expected to trace + # (an invocation or auto annotation) failed to produce one. + expected_traces = any( + step.type != "annotation" or step.origin not in {"human", "custom"} + for step in refreshable_steps + ) + if expected_traces: + log.warning("[METRICS] No trace_ids found! Cannot extract metrics.") return [] inferred_metrics_keys_by_step: Dict[str, List[Dict[str, str]]] = {} @@ -1328,14 +1573,6 @@ async def _refresh_metrics( bucket = buckets[0] - # log.info( - # f"[METRICS] Step '{step_key}': bucket has metrics: {bool(bucket.metrics)}" - # ) - # if bucket.metrics: - # log.info( - # f"[METRICS] Step '{step_key}': metrics keys: {list(bucket.metrics.keys())}" - # ) - if not bucket.metrics: log.warning("Bucket metrics should not be empty") log.warning("Bucket:", bucket) @@ -1367,7 +1604,7 @@ async def _refresh_metrics( ) ] - metrics = await self.create_metrics( + metrics = await self.set_metrics( project_id=project_id, user_id=user_id, # @@ -1538,6 +1775,24 @@ def mapping_key( # - EVALUATION QUEUE ------------------------------------------------------- + @staticmethod + def _validate_default_queue_data( + *, flags: Optional[EvaluationQueueFlags], data: Optional[EvaluationQueueData] + ) -> None: + if not flags or not flags.is_default or not data: + return + if any( + value is not None + for value in ( + data.user_ids, + data.scenario_ids, + data.step_keys, + data.batch_size, + data.batch_offset, + ) + ): + raise DefaultQueueDataInvalid() + async def create_queue( self, *, @@ -1547,13 +1802,21 @@ async def create_queue( queue: EvaluationQueueCreate, ) -> Optional[EvaluationQueue]: queue.version = CURRENT_VERSION + self._validate_default_queue_data(flags=queue.flags, data=queue.data) - return await self.evaluations_dao.create_queue( + created_queue = await self.evaluations_dao.create_queue( project_id=project_id, user_id=user_id, # queue=queue, ) + if created_queue: + await self._sync_run_queue_flag_for_default_queue( + project_id=project_id, + user_id=user_id, + queue=created_queue, + ) + return created_queue async def create_queues( self, @@ -1565,13 +1828,21 @@ async def create_queues( ) -> List[EvaluationQueue]: for queue in queues: queue.version = CURRENT_VERSION + self._validate_default_queue_data(flags=queue.flags, data=queue.data) - return await self.evaluations_dao.create_queues( + created_queues = await self.evaluations_dao.create_queues( project_id=project_id, user_id=user_id, # queues=queues, ) + for created_queue in created_queues: + await self._sync_run_queue_flag_for_default_queue( + project_id=project_id, + user_id=user_id, + queue=created_queue, + ) + return created_queues async def fetch_queue( self, @@ -1608,13 +1879,29 @@ async def edit_queue( queue: EvaluationQueueEdit, ) -> Optional[EvaluationQueue]: queue.version = CURRENT_VERSION + existing = await self.fetch_queue(project_id=project_id, queue_id=queue.id) + if existing and existing.flags and existing.flags.is_default: + if queue.flags and not queue.flags.is_default: + raise DefaultQueueDemotionForbidden(queue_id=queue.id) + effective_flags = existing.flags + else: + effective_flags = queue.flags or (existing.flags if existing else None) + effective_data = queue.data or (existing.data if existing else None) + self._validate_default_queue_data(flags=effective_flags, data=effective_data) - return await self.evaluations_dao.edit_queue( + edited_queue = await self.evaluations_dao.edit_queue( project_id=project_id, user_id=user_id, # queue=queue, ) + if edited_queue: + await self._sync_run_queue_flag_for_default_queue( + project_id=project_id, + user_id=user_id, + queue=edited_queue, + ) + return edited_queue async def edit_queues( self, @@ -1627,12 +1914,127 @@ async def edit_queues( for queue in queues: queue.version = CURRENT_VERSION - return await self.evaluations_dao.edit_queues( + existing_queues = await self.fetch_queues( + project_id=project_id, + queue_ids=[queue.id for queue in queues], + ) + existing_by_id = {queue.id: queue for queue in existing_queues} + for queue in queues: + existing = existing_by_id.get(queue.id) + if existing and existing.flags and existing.flags.is_default: + if queue.flags and not queue.flags.is_default: + raise DefaultQueueDemotionForbidden(queue_id=queue.id) + effective_flags = existing.flags + else: + effective_flags = queue.flags or (existing.flags if existing else None) + effective_data = queue.data or (existing.data if existing else None) + self._validate_default_queue_data( + flags=effective_flags, data=effective_data + ) + + edited_queues = await self.evaluations_dao.edit_queues( project_id=project_id, user_id=user_id, # queues=queues, ) + for edited_queue in edited_queues: + await self._sync_run_queue_flag_for_default_queue( + project_id=project_id, + user_id=user_id, + queue=edited_queue, + ) + return edited_queues + + async def _sync_run_queue_flag_for_default_queue( + self, + *, + project_id: UUID, + user_id: UUID, + queue: EvaluationQueue, + ) -> None: + if not queue.flags or not queue.flags.is_default: + return + run = await self.fetch_run(project_id=project_id, run_id=queue.run_id) + if not run: + return + has_human = bool(run.flags and run.flags.has_human) + is_queue = bool(has_human and queue.deleted_at is None) + if run.flags and run.flags.is_queue == is_queue: + return + flags = run.flags.model_copy() if run.flags else EvaluationRunFlags() + flags.is_queue = is_queue + try: + await self.evaluations_dao.edit_run( + project_id=project_id, + user_id=user_id, + run=EvaluationRunEdit( + id=run.id, + name=run.name, + description=run.description, + flags=flags, + tags=run.tags, + meta=run.meta, + status=run.status, + data=run.data, + ), + ) + except EvaluationClosedConflict: + # Archiving/unarchiving a default queue is a worklist action allowed + # on a closed run, but the closed run rejects content edits. The + # derived is_queue flag is best-effort here; it reconciles on reopen. + pass + + async def archive_queue( + self, + *, + project_id: UUID, + user_id: UUID, + queue_id: UUID, + force: bool = False, + ) -> Optional[EvaluationQueue]: + # Default queues are system-managed: only reconcile (force=True) may + # archive them. Direct user-facing archive of a default is forbidden. + if not force: + existing = await self.fetch_queue( + project_id=project_id, + queue_id=queue_id, + ) + if existing and existing.flags and existing.flags.is_default: + raise DefaultQueueArchiveForbidden(queue_id=queue_id) + + queue = await self.evaluations_dao.archive_queue( + project_id=project_id, + user_id=user_id, + queue_id=queue_id, + ) + if queue: + await self._sync_run_queue_flag_for_default_queue( + project_id=project_id, + user_id=user_id, + queue=queue, + ) + return queue + + async def unarchive_queue( + self, + *, + project_id: UUID, + user_id: UUID, + queue_id: UUID, + ) -> Optional[EvaluationQueue]: + queue = await self.evaluations_dao.unarchive_queue( + project_id=project_id, + user_id=user_id, + queue_id=queue_id, + ) + if queue: + await self._sync_run_queue_flag_for_default_queue( + project_id=project_id, + user_id=user_id, + queue=queue, + ) + return queue async def delete_queue( self, @@ -1641,6 +2043,9 @@ async def delete_queue( # queue_id: UUID, ) -> Optional[UUID]: + existing = await self.fetch_queue(project_id=project_id, queue_id=queue_id) + if existing and existing.flags and existing.flags.is_default: + raise DefaultQueueDeletionForbidden(queue_id=queue_id) return await self.evaluations_dao.delete_queue( project_id=project_id, # @@ -1654,6 +2059,12 @@ async def delete_queues( # queue_ids: List[UUID], ) -> List[UUID]: + existing_queues = await self.fetch_queues( + project_id=project_id, + queue_ids=queue_ids, + ) + if any(queue.flags and queue.flags.is_default for queue in existing_queues): + raise DefaultQueueDeletionForbidden() return await self.evaluations_dao.delete_queues( project_id=project_id, # @@ -1800,6 +2211,11 @@ def __init__( self.evaluators_service = evaluators_service self.evaluations_service = evaluations_service self.evaluations_worker = evaluations_worker + self.evaluations_task_runner = ( + TaskiqEvaluationTaskRunner(worker=evaluations_worker) + if evaluations_worker is not None + else None + ) async def create( self, @@ -1856,6 +2272,7 @@ async def create( evaluator_steps=evaluation.data.evaluator_steps, # repeats=evaluation.data.repeats, + concurrency=evaluation.data.concurrency, # is_live=evaluation.flags.is_live, ) @@ -2232,50 +2649,26 @@ async def start( _evaluation = await self._parse_evaluation_run(run=run) return _evaluation - if self.evaluations_worker is None: + if self.evaluations_task_runner is None: log.warning( "[EVAL] Taskiq client missing; cannot dispatch evaluation run", ) return _evaluation - has_query_steps = bool(_evaluation.data.query_steps) - has_testset_steps = bool(_evaluation.data.testset_steps) - has_application_steps = bool(_evaluation.data.application_steps) - has_evaluator_steps = bool(_evaluation.data.evaluator_steps) + # Worker task names are API-internal, so dispatch through the + # unified run processor rather than topology-specific handlers. + topology = classify_run_topology(run) - if has_query_steps and has_evaluator_steps: - await self._ensure_human_annotation_queue( - project_id=project_id, - user_id=user_id, - run=run, - ) - await self.evaluations_worker.evaluate_batch_query.kiq( - project_id=project_id, - user_id=user_id, - # - run_id=run.id, - ) - - elif ( - has_testset_steps and has_application_steps and has_evaluator_steps - ): - await self.evaluations_worker.evaluate_batch_testset.kiq( - project_id=project_id, - user_id=user_id, - # - run_id=run.id, - ) - - elif ( - has_testset_steps - and has_application_steps - and not has_evaluator_steps - and not has_query_steps - ): - await self.evaluations_worker.evaluate_batch_invocation.kiq( + if topology.dispatch: + if topology.dispatch == "batch_query": + await self._ensure_human_annotation_queue( + project_id=project_id, + user_id=user_id, + run=run, + ) + await self.evaluations_task_runner.process_run_from_source( project_id=project_id, user_id=user_id, - # run_id=run.id, ) @@ -2283,10 +2676,9 @@ async def start( log.warning( "[EVAL] [start] [skip] unsupported non-live run topology", run_id=run.id, - has_query_steps=has_query_steps, - has_testset_steps=has_testset_steps, - has_application_steps=has_application_steps, - has_evaluator_steps=has_evaluator_steps, + topology=topology.label, + topology_status=topology.status, + reason=topology.reason, ) return _evaluation @@ -2335,7 +2727,7 @@ async def _ensure_human_annotation_queue( run=run, ) - async def evaluate_batch_traces( + async def dispatch_trace_slice( self, *, project_id: UUID, @@ -2347,7 +2739,7 @@ async def evaluate_batch_traces( ) -> bool: if not trace_ids: return False - if self.evaluations_worker is None: + if self.evaluations_task_runner is None: log.warning( "[EVAL] Taskiq client missing; cannot dispatch trace batch", run_id=run_id, @@ -2358,9 +2750,13 @@ async def evaluate_batch_traces( project_id=project_id, run_id=run_id, ) - if not run or not run.flags or not run.flags.is_queue: + if ( + not run + or not run.flags + or not (run.flags.has_traces or run.flags.has_queries) + ): log.warning( - "[EVAL] trace batch dispatch requires a queue evaluation run", + "[EVAL] trace batch dispatch requires a trace-capable evaluation run", run_id=run_id, ) return False @@ -2371,17 +2767,17 @@ async def evaluate_batch_traces( run=run, ) - await self.evaluations_worker.evaluate_batch_traces.kiq( + await self.evaluations_task_runner.process_run_from_batch( project_id=project_id, user_id=user_id, - # run_id=run_id, + source_kind="traces", trace_ids=trace_ids, input_step_key=input_step_key, ) return True - async def evaluate_batch_testcases( + async def dispatch_testcase_slice( self, *, project_id: UUID, @@ -2393,7 +2789,7 @@ async def evaluate_batch_testcases( ) -> bool: if not testcase_ids: return False - if self.evaluations_worker is None: + if self.evaluations_task_runner is None: log.warning( "[EVAL] Taskiq client missing; cannot dispatch testcase batch", run_id=run_id, @@ -2404,9 +2800,13 @@ async def evaluate_batch_testcases( project_id=project_id, run_id=run_id, ) - if not run or not run.flags or not run.flags.is_queue: + if ( + not run + or not run.flags + or not (run.flags.has_testcases or run.flags.has_testsets) + ): log.warning( - "[EVAL] testcase batch dispatch requires a queue evaluation run", + "[EVAL] testcase batch dispatch requires a testcase-capable evaluation run", run_id=run_id, ) return False @@ -2417,16 +2817,384 @@ async def evaluate_batch_testcases( run=run, ) - await self.evaluations_worker.evaluate_batch_testcases.kiq( + await self.evaluations_task_runner.process_run_from_batch( project_id=project_id, user_id=user_id, - # run_id=run_id, + source_kind="testcases", testcase_ids=testcase_ids, input_step_key=input_step_key, ) return True + # --- TENSOR SLICE OPS ----------------------------------------------------- + # + # Coordinate-addressed ops over EXISTING scenarios (scenarios x steps x + # repeats), distinct from the source-keyed dispatch_*_slice above (which + # ingests NEW source items). `process` is the re-execution verb (retry / + # fill-missing / run-added-step), dispatched async via taskiq under the job + # lock. `probe` (read) and `populate` (write) are immediate, in-process. + + async def dispatch_tensor_slice( + self, + *, + project_id: UUID, + user_id: UUID, + # + run_id: UUID, + scenario_ids: Optional[List[UUID]] = None, + step_keys: Optional[List[str]] = None, + repeat_idxs: Optional[List[int]] = None, + process_mode: SliceProcessMode = "fill-missing", + ) -> bool: + if self.evaluations_task_runner is None: + log.warning( + "[EVAL] Taskiq client missing; cannot dispatch tensor slice", + run_id=run_id, + ) + return False + + run = await self.evaluations_service.fetch_run( + project_id=project_id, + run_id=run_id, + ) + if not run: + log.warning( + "[EVAL] tensor slice dispatch requires an existing run", + run_id=run_id, + ) + return False + + await self.evaluations_task_runner.process_rerun( + project_id=project_id, + user_id=user_id, + run_id=run_id, + scenario_ids=scenario_ids, + step_keys=step_keys, + repeat_idxs=repeat_idxs, + process_mode=process_mode, + ) + return True + + async def probe_slice( + self, + *, + project_id: UUID, + # + run_id: UUID, + scenario_ids: Optional[List[UUID]] = None, + step_keys: Optional[List[str]] = None, + repeat_idxs: Optional[List[int]] = None, + ) -> List[EvaluationResult]: + tensor_ops = self.evaluations_service.tensor_slice_operations + if tensor_ops is None: + log.warning("[EVAL] tensor ops not wired; cannot probe slice") + return [] + + return await tensor_ops.probe( + project_id=project_id, + tensor_slice=TensorSlice( + run_id=run_id, + scenario_ids=scenario_ids, + step_keys=step_keys, + repeat_idxs=repeat_idxs, + ), + ) + + async def populate_slice( + self, + *, + project_id: UUID, + user_id: UUID, + # + results: List[EvaluationResultCreate], + ) -> List[EvaluationResult]: + tensor_ops = self.evaluations_service.tensor_slice_operations + if tensor_ops is None: + log.warning("[EVAL] tensor ops not wired; cannot populate slice") + return [] + + return await tensor_ops.populate( + project_id=project_id, + user_id=user_id, + results=results, + ) + + async def prune_slice( + self, + *, + project_id: UUID, + user_id: UUID, + # + run_id: UUID, + scenario_ids: Optional[List[UUID]] = None, + step_keys: Optional[List[str]] = None, + repeat_idxs: Optional[List[int]] = None, + ) -> List[UUID]: + tensor_ops = self.evaluations_service.tensor_slice_operations + if tensor_ops is None: + log.warning("[EVAL] tensor ops not wired; cannot prune slice") + return [] + + return await tensor_ops.prune( + project_id=project_id, + user_id=user_id, + tensor_slice=TensorSlice( + run_id=run_id, + scenario_ids=scenario_ids, + step_keys=step_keys, + repeat_idxs=repeat_idxs, + ), + ) + + async def refresh_slice( + self, + *, + project_id: UUID, + user_id: UUID, + # + run_id: UUID, + scenario_ids: Optional[List[UUID]] = None, + step_keys: Optional[List[str]] = None, + repeat_idxs: Optional[List[int]] = None, + ) -> None: + """Recompute metrics over the slice scope (variational + aggregate). + + The metrics counterpart of populate/process: callers that wrote cells + without executing (e.g. the SDK, which runs workflows locally and + populates the finished cells) invoke this to roll up the per-scenario, + temporal, and global metric rows without re-running anything. + """ + tensor_ops = self.evaluations_service.tensor_slice_operations + if tensor_ops is None: + log.warning("[EVAL] tensor ops not wired; cannot refresh slice") + return + + await tensor_ops.refresh( + project_id=project_id, + user_id=user_id, + tensor_slice=TensorSlice( + run_id=run_id, + scenario_ids=scenario_ids, + step_keys=step_keys, + repeat_idxs=repeat_idxs, + ), + ) + + # --- GRAPH-DIMENSION OPS -------------------------------------------------- + # + # Modify the tensor's SHAPE (scenarios x steps x repeats) — distinct from the + # tensor ops (probe/populate/process/prune) that fill or clear cells WITHIN a + # fixed shape. Three axes, each with a paired add/remove (or set): + # + # height — `add_scenarios` / `remove_scenarios` (scenario rows) + # width — `add_steps` / `remove_steps` (graph step columns) + # depth — `set_repeats` (repeat count) + # + # `process` operates only on EXISTING coordinates — it never mints scenarios + # or steps, so callers grow the shape with these ops first when needed. + + async def _edit_run_data( + self, + *, + project_id: UUID, + user_id: UUID, + # + run: EvaluationRun, + new_data: EvaluationRunData, + ) -> Optional[EvaluationRun]: + """Persist a new `data` payload for `run`, preserving every other field.""" + return await self.evaluations_service.edit_run( + project_id=project_id, + user_id=user_id, + run=EvaluationRunEdit( + id=run.id, + name=run.name, + description=run.description, + tags=run.tags, + meta=run.meta, + status=run.status, + flags=run.flags, + data=new_data, + ), + ) + + async def add_scenarios( + self, + *, + project_id: UUID, + user_id: UUID, + # + run_id: UUID, + count: int, + timestamp: Optional[datetime] = None, + ) -> List[EvaluationScenario]: + """Create `count` scenario skeleton rows for the run (height dimension). + + Skeleton only: rows with no input cells and no results. `populate` writes + the input cells (the trace_id/testcase_id binding); `process` plans and + executes. Returns the created scenarios so the caller has their ids. + + `timestamp` buckets the new scenarios on the temporal (time) axis so they + participate in temporal metrics — mirroring the query path, which derives + a timestamp from its window. The bucket width (`interval`) is fixed at + `DEFAULT_REFRESH_INTERVAL` (1 minute); only the timestamp is caller-set, + and it is floored to the minute so it lands on the bucket boundary. + """ + if count <= 0: + return [] + + bucket = ( + timestamp.replace(second=0, microsecond=0) + if timestamp is not None + else None + ) + + return await self.evaluations_service.create_scenarios( + project_id=project_id, + user_id=user_id, + scenarios=[ + EvaluationScenarioCreate( + run_id=run_id, + status=EvaluationStatus.RUNNING, + timestamp=bucket, + interval=DEFAULT_REFRESH_INTERVAL if bucket else None, + ) + for _ in range(count) + ], + ) + + async def remove_scenarios( + self, + *, + project_id: UUID, + # + scenario_ids: List[UUID], + ) -> List[UUID]: + """Delete scenario rows (height dimension) — the inverse of `add_scenarios`. + + Removing a scenario drops its whole row (every step/repeat cell with it). + Returns the ids actually deleted. + + Deletion is scoped by `project_id` only — it does NOT verify the + scenarios belong to a particular run. The run-scoped HTTP endpoint + validates `scenario_ids` against its path `evaluation_id` before calling + here, so cross-run deletion cannot happen over the API. + """ + if not scenario_ids: + return [] + + return await self.evaluations_service.delete_scenarios( + project_id=project_id, + scenario_ids=scenario_ids, + ) + + async def add_steps( + self, + *, + project_id: UUID, + user_id: UUID, + # + run_id: UUID, + steps: List[EvaluationRunDataStep], + ) -> Optional[EvaluationRun]: + """Append graph steps to the run (width dimension). + + Adds new step columns to `run.data.steps`; the cells under them start + empty and a subsequent `process` fills them. Steps whose key already + exists are skipped (add is idempotent on key). Existing steps, scenarios, + and result cells are untouched. + """ + run = await self.evaluations_service.fetch_run( + project_id=project_id, + run_id=run_id, + ) + if not run or not run.data: + return None + if not steps: + return run + + existing = list(run.data.steps or []) + existing_keys = {step.key for step in existing} + fresh = [step for step in steps if step.key not in existing_keys] + if not fresh: + return run + + new_data = run.data.model_copy(update={"steps": existing + fresh}) + return await self._edit_run_data( + project_id=project_id, + user_id=user_id, + run=run, + new_data=new_data, + ) + + async def remove_steps( + self, + *, + project_id: UUID, + user_id: UUID, + # + run_id: UUID, + step_keys: List[str], + ) -> Optional[EvaluationRun]: + """Drop graph steps from the run by key (width dimension). + + The inverse of `add_steps`: removes the named step columns from + `run.data.steps`. The result cells under those steps are not deleted here + (that is `prune`); this op only narrows the graph's declared width. + """ + run = await self.evaluations_service.fetch_run( + project_id=project_id, + run_id=run_id, + ) + if not run or not run.data: + return None + if not step_keys: + return run + + drop = set(step_keys) + kept = [step for step in (run.data.steps or []) if step.key not in drop] + if len(kept) == len(run.data.steps or []): + return run + + new_data = run.data.model_copy(update={"steps": kept}) + return await self._edit_run_data( + project_id=project_id, + user_id=user_id, + run=run, + new_data=new_data, + ) + + async def set_repeats( + self, + *, + project_id: UUID, + user_id: UUID, + # + run_id: UUID, + repeats: int, + ) -> Optional[EvaluationRun]: + """Set the run's repeat (depth) dimension. + + `repeats` is fixed at run creation today; this is the first-class op to + change it. Growing it adds repeat_idx slots that subsequent `process` + runs plan and fill; the existing result cells are untouched. + """ + run = await self.evaluations_service.fetch_run( + project_id=project_id, + run_id=run_id, + ) + if not run or not run.data: + return None + + new_data = run.data.model_copy(update={"repeats": repeats}) + return await self._edit_run_data( + project_id=project_id, + user_id=user_id, + run=run, + new_data=new_data, + ) + async def close( self, *, @@ -2485,8 +3253,11 @@ async def _make_evaluation_run_data( evaluator_steps: Optional[Target] = None, # repeats: Optional[int] = None, + concurrency: Optional[EvaluationRunDataConcurrency] = None, # is_live: Optional[bool] = None, + # + default_evaluator_origin: Origin = DEFAULT_ORIGIN_EVALUATORS, ) -> Optional[EvaluationRunData]: # IMPLICIT FLAG: is_multivariate=False # IMPLICIT FLAG: all_inputs=True @@ -2791,7 +3562,7 @@ async def _make_evaluation_run_data( if isinstance(evaluator_steps, list): evaluator_steps = { - evaluator_revision_id: DEFAULT_ORIGIN_EVALUATORS + evaluator_revision_id: default_evaluator_origin for evaluator_revision_id in evaluator_steps } @@ -3044,6 +3815,7 @@ async def _make_evaluation_run_data( steps=steps, mappings=mappings, repeats=repeats or 1, + concurrency=concurrency, ) except Exception: # pylint: disable=broad-exception-caught @@ -3149,6 +3921,11 @@ async def _activate_evaluation_run( run.flags.is_active = True + # A (re)dispatched run is running until its slice finalizes. Reset to + # RUNNING on every activation — not just creation — so an extended + # finished run goes back to `running` while the new work executes and is + # re-finalized by the slice. The slice's terminal status then replaces + # this via the (corrected) severity floor in source_slice.py. run = await self.evaluations_service.edit_run( project_id=project_id, user_id=user_id, @@ -3163,7 +3940,7 @@ async def _activate_evaluation_run( tags=run.tags, meta=run.meta, # - status=EvaluationStatus.RUNNING if just_created else run.status, + status=EvaluationStatus.RUNNING, # data=run.data, ), @@ -3387,6 +4164,7 @@ async def create( evaluator_steps=queue.data.evaluators, repeats=repeats, is_live=False, + default_evaluator_origin="human", ) if not run_data or not run_data.steps: return None @@ -3409,7 +4187,7 @@ async def create( is_live=False, is_active=True, is_closed=False, - is_queue=True, + is_queue=False, ), tags=queue.tags, meta=queue.meta, @@ -3534,40 +4312,39 @@ async def query( run_ids_filter = list(dict.fromkeys(requested_run_ids)) + eligible_runs = await self.evaluations_service.query_runs( + project_id=project_id, + run=EvaluationRunQuery( + flags=EvaluationRunQueryFlags(is_queue=True), + ), + ) + eligible_run_ids = [run.id for run in eligible_runs if run and run.id] if query and query.kind is not None: - run_query = EvaluationRunQuery( - flags=EvaluationRunQueryFlags( - is_queue=True, - has_queries=query.kind == SimpleQueueKind.TRACES, - has_testsets=query.kind == SimpleQueueKind.TESTCASES, - ), - ) - runs = await self.evaluations_service.query_runs( - project_id=project_id, - run=run_query, - ) + eligible_run_ids = [ + run.id + for run in eligible_runs + if run and run.id and self._get_kind(run) == query.kind + ] + if not eligible_run_ids: + return [] - kind_run_ids = [run.id for run in runs if run and run.id] - if not kind_run_ids: + eligible_run_ids_set = set(eligible_run_ids) + if run_ids_filter is None: + run_ids_filter = eligible_run_ids + else: + run_ids_filter = [ + run_id for run_id in run_ids_filter if run_id in eligible_run_ids_set + ] + if not run_ids_filter: return [] - kind_run_ids_set = set(kind_run_ids) - if run_ids_filter is None: - run_ids_filter = kind_run_ids - else: - run_ids_filter = [ - run_id for run_id in run_ids_filter if run_id in kind_run_ids_set - ] - if not run_ids_filter: - return [] - queues = await self.evaluations_service.query_queues( project_id=project_id, queue=EvaluationQueueQuery( name=query.name if query else None, description=query.description if query else None, # - flags=EvaluationQueueQueryFlags(), + flags=EvaluationQueueQueryFlags(is_default=True), tags=query.tags if query else None, meta=query.meta if query else None, # @@ -3631,7 +4408,7 @@ async def add_traces( if self._get_kind(run) != SimpleQueueKind.TRACES: return None - ok = await self.simple_evaluations_service.evaluate_batch_traces( + ok = await self.simple_evaluations_service.dispatch_trace_slice( project_id=project_id, user_id=user_id, # @@ -3671,7 +4448,7 @@ async def add_testcases( if self._get_kind(run) != SimpleQueueKind.TESTCASES: return None - ok = await self.simple_evaluations_service.evaluate_batch_testcases( + ok = await self.simple_evaluations_service.dispatch_testcase_slice( project_id=project_id, user_id=user_id, # @@ -3799,63 +4576,33 @@ async def _dispatch_source_batches( if not run.id or not run.data or not run.data.steps: return False - dispatched = False - for step in run.data.steps: - if step.type != "input" or not step.key: - continue - - refs = step.references or {} - query_revision_ref = refs.get("query_revision") - testset_revision_ref = refs.get("testset_revision") - - if query_revision_ref and query_revision_ref.id: - query_revision = await self.simple_evaluations_service.queries_service.fetch_query_revision( - project_id=project_id, - query_revision_ref=query_revision_ref, - include_trace_ids=True, - ) - trace_ids = ( - query_revision.data.trace_ids - if query_revision - and query_revision.data - and query_revision.data.trace_ids - else [] - ) - if not trace_ids: - continue + batches = await resolve_queue_source_batches( + project_id=project_id, + run=run, + queries_service=self.simple_evaluations_service.queries_service, + testsets_service=self.simple_evaluations_service.testsets_service, + ) - ok = await self.simple_evaluations_service.evaluate_batch_traces( + dispatched = False + for batch in batches: + if batch.kind == "traces" and batch.trace_ids: + ok = await self.simple_evaluations_service.dispatch_trace_slice( project_id=project_id, user_id=user_id, run_id=run.id, - trace_ids=trace_ids, - input_step_key=step.key, + trace_ids=batch.trace_ids, + input_step_key=batch.step_key, ) dispatched = dispatched or ok continue - if testset_revision_ref and testset_revision_ref.id: - testset_revision = await self.simple_evaluations_service.testsets_service.fetch_testset_revision( - project_id=project_id, - testset_revision_ref=testset_revision_ref, - include_testcase_ids=True, - ) - testcase_ids = ( - testset_revision.data.testcase_ids - if testset_revision - and testset_revision.data - and testset_revision.data.testcase_ids - else [] - ) - if not testcase_ids: - continue - - ok = await self.simple_evaluations_service.evaluate_batch_testcases( + if batch.kind == "testcases" and batch.testcase_ids: + ok = await self.simple_evaluations_service.dispatch_testcase_slice( project_id=project_id, user_id=user_id, run_id=run.id, - testcase_ids=testcase_ids, - input_step_key=step.key, + testcase_ids=batch.testcase_ids, + input_step_key=batch.step_key, ) dispatched = dispatched or ok @@ -3882,9 +4629,7 @@ async def _make_run_data( annotation_mappings: List[EvaluationRunDataMapping] = [] annotation_step_keys: List[str] = [] - source_step_key = ( - "query-direct" if kind == SimpleQueueKind.TRACES else "testset-direct" - ) + source_step_key = "traces" if kind == SimpleQueueKind.TRACES else "testcases" source_step = EvaluationRunDataStep( key=source_step_key, type="input", @@ -4003,21 +4748,22 @@ def _get_kind(self, run: EvaluationRun) -> Optional[SimpleQueueKind]: if not run.flags or not run.flags.is_queue: return None - if run.flags.has_queries and not run.flags.has_testsets: - return SimpleQueueKind.TRACES - - if run.flags.has_testsets and not run.flags.has_queries: - return SimpleQueueKind.TESTCASES - - return None + families = [ + (run.flags.has_queries, SimpleQueueKind.QUERIES), + (run.flags.has_testsets, SimpleQueueKind.TESTSETS), + (run.flags.has_traces, SimpleQueueKind.TRACES), + (run.flags.has_testcases, SimpleQueueKind.TESTCASES), + ] + enabled = [kind for enabled, kind in families if enabled] + return enabled[0] if len(enabled) == 1 else None @staticmethod def _get_source_kind(*, queue_data: SimpleQueueData) -> Optional[SimpleQueueKind]: if queue_data.queries: - return SimpleQueueKind.TRACES + return SimpleQueueKind.QUERIES if queue_data.testsets: - return SimpleQueueKind.TESTCASES + return SimpleQueueKind.TESTSETS return None diff --git a/api/oss/src/core/evaluations/tasks/batch.py b/api/oss/src/core/evaluations/tasks/batch.py deleted file mode 100644 index 1416010344..0000000000 --- a/api/oss/src/core/evaluations/tasks/batch.py +++ /dev/null @@ -1,148 +0,0 @@ -from uuid import UUID - -from oss.src.utils.logging import get_module_logger -from oss.src.utils.common import is_ee - -if is_ee(): - pass - -from oss.src.dbs.postgres.queries.dbes import ( - QueryArtifactDBE, - QueryVariantDBE, - QueryRevisionDBE, -) -from oss.src.dbs.postgres.testcases.dbes import ( - TestcaseBlobDBE, -) -from oss.src.dbs.postgres.testsets.dbes import ( - TestsetArtifactDBE, - TestsetVariantDBE, - TestsetRevisionDBE, -) -from oss.src.dbs.postgres.workflows.dbes import ( - WorkflowArtifactDBE, - WorkflowVariantDBE, - WorkflowRevisionDBE, -) - -from oss.src.dbs.postgres.tracing.dao import TracingDAO -from oss.src.dbs.postgres.blobs.dao import BlobsDAO -from oss.src.dbs.postgres.git.dao import GitDAO -from oss.src.dbs.postgres.evaluations.dao import EvaluationsDAO - -from oss.src.core.tracing.service import TracingService -from oss.src.core.queries.service import QueriesService -from oss.src.core.testcases.service import TestcasesService -from oss.src.core.testsets.service import TestsetsService -from oss.src.core.testsets.service import SimpleTestsetsService -from oss.src.core.workflows.service import WorkflowsService -from oss.src.core.evaluators.service import EvaluatorsService -from oss.src.core.evaluators.service import SimpleEvaluatorsService -from oss.src.core.evaluations.service import EvaluationsService -from oss.src.core.annotations.service import AnnotationsService - - -log = get_module_logger(__name__) - - -# DBS -------------------------------------------------------------------------- - -tracing_dao = TracingDAO() - -testcases_dao = BlobsDAO( - BlobDBE=TestcaseBlobDBE, -) - -queries_dao = GitDAO( - ArtifactDBE=QueryArtifactDBE, - VariantDBE=QueryVariantDBE, - RevisionDBE=QueryRevisionDBE, -) - -testsets_dao = GitDAO( - ArtifactDBE=TestsetArtifactDBE, - VariantDBE=TestsetVariantDBE, - RevisionDBE=TestsetRevisionDBE, -) - -workflows_dao = GitDAO( - ArtifactDBE=WorkflowArtifactDBE, - VariantDBE=WorkflowVariantDBE, - RevisionDBE=WorkflowRevisionDBE, -) - -evaluations_dao = EvaluationsDAO() - -# CORE ------------------------------------------------------------------------- - -tracing_service = TracingService( - tracing_dao=tracing_dao, -) - -queries_service = QueriesService( - queries_dao=queries_dao, -) - -testcases_service = TestcasesService( - testcases_dao=testcases_dao, -) - -testsets_service = TestsetsService( - testsets_dao=testsets_dao, - testcases_service=testcases_service, -) - -simple_testsets_service = SimpleTestsetsService( - testsets_service=testsets_service, -) - -workflows_service = WorkflowsService( - workflows_dao=workflows_dao, -) - -evaluators_service = EvaluatorsService( - workflows_service=workflows_service, -) - -simple_evaluators_service = SimpleEvaluatorsService( - evaluators_service=evaluators_service, -) - -evaluations_service = EvaluationsService( - evaluations_dao=evaluations_dao, - tracing_service=tracing_service, - queries_service=queries_service, - testsets_service=testsets_service, - evaluators_service=evaluators_service, - # -) - -annotations_service = AnnotationsService( - tracing_service=tracing_service, - evaluators_service=evaluators_service, - simple_evaluators_service=simple_evaluators_service, -) - -# ------------------------------------------------------------------------------ - - -def evaluate_testsets( - self, - *, - project_id: UUID, - user_id: UUID, - # - run_id: UUID, -): - pass - - -def evaluate_queries( - self, - *, - project_id: UUID, - user_id: UUID, - # - run_id: UUID, -): - pass diff --git a/api/oss/src/core/evaluations/tasks/legacy.py b/api/oss/src/core/evaluations/tasks/legacy.py deleted file mode 100644 index 2ab518e421..0000000000 --- a/api/oss/src/core/evaluations/tasks/legacy.py +++ /dev/null @@ -1,2252 +0,0 @@ -from typing import Dict, List, Optional, Any - -from uuid import UUID -from json import dumps - -from fastapi import Request - -from oss.src.utils.logging import get_module_logger -from oss.src.services import llm_apps_service -from oss.src.models.shared_models import InvokationResult - -from oss.src.core.queries.service import QueriesService -from oss.src.core.testcases.service import TestcasesService -from oss.src.core.testsets.service import TestsetsService -from oss.src.core.applications.service import ApplicationsService -from oss.src.core.workflows.service import WorkflowsService -from oss.src.core.evaluators.service import SimpleEvaluatorsService -from oss.src.core.evaluations.service import EvaluationsService - -from oss.src.core.tracing.service import TracingService - - -from oss.src.core.evaluations.types import ( - EvaluationStatus, - EvaluationRun, - EvaluationRunEdit, - EvaluationScenarioCreate, - EvaluationScenarioEdit, - EvaluationResultCreate, - EvaluationMetricsRefresh, -) - -from oss.src.core.shared.dtos import Reference -from oss.src.core.workflows.dtos import ( - WorkflowServiceRequestData, - WorkflowServiceRequest, -) -from oss.src.core.git.utils import revision_references - - -from oss.src.core.evaluations.utils import ( - build_repeat_indices, - effective_is_split, - fetch_traces_by_hash, - fetch_trace, - make_hash, - plan_missing_traces, - required_traces_for_step, - select_traces_for_reuse, -) - - -def _serialize_references(references: Dict[str, Any]) -> Dict[str, Any]: - """Serialize Reference objects and UUIDs to JSON-compatible dicts.""" - from uuid import UUID - - def _deep_serialize(obj: Any) -> Any: - if isinstance(obj, UUID): - return str(obj) - elif isinstance(obj, Reference): - return obj.model_dump(exclude_none=True) - elif isinstance(obj, dict): - return {k: _deep_serialize(v) for k, v in obj.items()} - elif isinstance(obj, (list, tuple)): - return [_deep_serialize(item) for item in obj] - return obj - - return {k: _deep_serialize(v) for k, v in references.items()} - - -log = get_module_logger(__name__) - - -def _resolve_runtime_uri( - *, - revision_data: Optional[Any], -) -> Optional[str]: - if revision_data is None: - return None - - return WorkflowsService._get_service_url(revision_data=revision_data) - - -def _extract_root_span(trace: Optional[Any]) -> Optional[Any]: - if not trace: - # log.debug("[TRACE] [ROOT]", reason="missing-trace") - return None - - spans = getattr(trace, "spans", None) - - if not isinstance(spans, dict): - # log.debug( - # "[TRACE] [ROOT]", - # trace_id=str(getattr(trace, "trace_id", None)) - # if getattr(trace, "trace_id", None) - # else None, - # reason="spans-not-dict", - # spans_type=type(spans).__name__ if spans is not None else None, - # ) - return None - - if not spans: - # log.debug( - # "[TRACE] [ROOT]", - # trace_id=str(getattr(trace, "trace_id", None)) - # if getattr(trace, "trace_id", None) - # else None, - # reason="spans-empty", - # ) - return None - - root_span = list(spans.values())[0] - if isinstance(root_span, list): - # log.debug( - # "[TRACE] [ROOT]", - # trace_id=str(getattr(trace, "trace_id", None)) - # if getattr(trace, "trace_id", None) - # else None, - # reason="first-span-is-list", - # span_keys=list(spans.keys()), - # first_list_len=len(root_span), - # ) - return None - - # log.debug( - # "[TRACE] [ROOT]", - # trace_id=str(getattr(trace, "trace_id", None)) - # if getattr(trace, "trace_id", None) - # else None, - # reason="resolved", - # span_keys=list(spans.keys()), - # root_span_id=str(getattr(root_span, "span_id", None)) - # if getattr(root_span, "span_id", None) - # else None, - # ) - return root_span - - -def _build_trace_context( - *, - trace: Optional[Any], - error: Optional[Dict[str, Any]] = None, -) -> Optional[Dict[str, Any]]: - root_span = _extract_root_span(trace) - trace_id = getattr(trace, "trace_id", None) if trace else None - - if not root_span or not trace_id: - # log.debug( - # "[TRACE] [CONTEXT]", - # trace_id=str(trace_id) if trace_id else None, - # has_root_span=bool(root_span), - # has_error=bool(error), - # error=error, - # ) - return None - - # log.debug( - # "[TRACE] [CONTEXT]", - # trace_id=str(trace_id), - # span_id=str(getattr(root_span, "span_id", None)) - # if getattr(root_span, "span_id", None) - # else None, - # has_error=bool(error), - # ) - return { - "trace": trace, - "trace_id": str(trace_id), - "span_id": getattr(root_span, "span_id", None), - "root_span": root_span, - "error": error, - } - - -async def _resolve_testset_input_specs( - *, - project_id: UUID, - input_steps: List[Any], - testsets_service: TestsetsService, -) -> List[Dict[str, Any]]: - input_specs: List[Dict[str, Any]] = [] - - for input_step in input_steps: - input_refs = input_step.references or {} - testset_revision_ref = input_refs.get("testset_revision") - - if not testset_revision_ref or not isinstance(testset_revision_ref.id, UUID): - raise ValueError( - f"Evaluation input step {input_step.key} missing testset_revision reference." - ) - - testset_revision = await testsets_service.fetch_testset_revision( - project_id=project_id, - testset_revision_ref=testset_revision_ref, - ) - if not testset_revision: - raise ValueError( - f"Testset revision with id {testset_revision_ref.id} not found!" - ) - if not testset_revision.data or not testset_revision.data.testcases: - raise ValueError( - f"Testset revision with id {testset_revision_ref.id} has no testcases!" - ) - - testset_variant = await testsets_service.fetch_testset_variant( - project_id=project_id, - testset_variant_ref=Reference(id=testset_revision.variant_id), - ) - if not testset_variant: - raise ValueError( - f"Testset variant with id {testset_revision.variant_id} not found!" - ) - - testset = await testsets_service.fetch_testset( - project_id=project_id, - testset_ref=Reference(id=testset_variant.testset_id), - ) - if not testset: - raise ValueError(f"Testset with id {testset_variant.testset_id} not found!") - - testcases = testset_revision.data.testcases - input_specs.append( - { - "step_key": input_step.key, - "testset": testset, - "testset_variant": testset_variant, - "testset_revision": testset_revision, - "testcases": testcases, - "testcases_data": [ - {**testcase.data, "testcase_id": str(testcase.id)} - for testcase in testcases - ], - } - ) - - return input_specs - - -async def evaluate_batch_testset( - *, - project_id: UUID, - user_id: UUID, - # - run_id: UUID, - # - tracing_service: TracingService, - testsets_service: TestsetsService, - queries_service: QueriesService, - workflows_service: WorkflowsService, - applications_service: ApplicationsService, - evaluations_service: EvaluationsService, - # - simple_evaluators_service: SimpleEvaluatorsService, -): - """ - Annotates an application revision applied to a testset using auto evaluator(s). - - All testset, application, and evaluator information is extracted from the - evaluation run's data.steps references. - - Args: - project_id (UUID): The ID of the project. - user_id (UUID): The ID of the user. - run_id (UUID): The ID of the evaluation run. - - Returns: - None - """ - request = Request( - scope={ - "type": "http", - "http_version": "1.1", - "scheme": "http", - } - ) - request.state.project_id = str(project_id) - request.state.user_id = str(user_id) - - run = None - - try: - # ---------------------------------------------------------------------- - log.info( - "[SCOPE] ", run_id=run_id, project_id=project_id, user_id=user_id - ) - # ---------------------------------------------------------------------- - - # fetch run ------------------------------------------------------------ - run = await evaluations_service.fetch_run( - project_id=project_id, - # - run_id=run_id, - ) - - if not run: - raise ValueError(f"Evaluation run with id {run_id} not found!") - - if not run.data: - raise ValueError(f"Evaluation run with id {run_id} has no data!") - - if not run.data.steps: - raise ValueError(f"Evaluation run with id {run_id} has no steps!") - - steps = run.data.steps - repeats = run.data.repeats or 1 - repeat_indices = build_repeat_indices(repeats) - is_cached = bool(run.flags.is_cached) if run.flags else False - - input_steps = [step for step in steps if step.type == "input"] - invocation_steps = [step for step in steps if step.type == "invocation"] - annotation_steps = [step for step in steps if step.type == "annotation"] - - log.info( - "[STEPS] ", - run_id=run_id, - count=len(steps), - input_keys=[step.key for step in input_steps], - invocation_keys=[step.key for step in invocation_steps], - annotation_keys=[step.key for step in annotation_steps], - step_types=[getattr(step, "type", None) for step in steps], - ) - - if not input_steps or len(invocation_steps) != 1: - raise ValueError( - f"Evaluation run with id {run_id} must have at least one input and exactly one invocation step." - ) - - invocation_step = invocation_steps[0] - invocation_step_key = invocation_step.key - is_split = effective_is_split( - is_split=bool(run.flags.is_split) if run.flags else False, - has_application_steps=True, - has_evaluator_steps=bool(annotation_steps), - ) - application_required_count = required_traces_for_step( - repeats=repeats, - is_split=is_split, - step_kind="application", - has_evaluator_steps=bool(annotation_steps), - ) - evaluator_required_count = required_traces_for_step( - repeats=repeats, - is_split=is_split, - step_kind="evaluator", - has_evaluator_steps=bool(annotation_steps), - ) - - application_revision_ref = invocation_step.references.get( - "application_revision" - ) - if not application_revision_ref or not isinstance( - application_revision_ref.id, UUID - ): - raise ValueError( - f"Evaluation run with id {run_id} missing invocation.application_revision reference." - ) - - run_config = { - "batch_size": 10, - "max_retries": 3, - "retry_delay": 3, - "delay_between_batches": 5, - } - - input_specs = await _resolve_testset_input_specs( - project_id=project_id, - input_steps=input_steps, - testsets_service=testsets_service, - ) - testset_revision_ids = [ - str(input_spec["testset_revision"].id) for input_spec in input_specs - ] - - log.info("[TESTSET] ", run_id=run_id, ids=testset_revision_ids) - log.info( - "[APPLICATION] ", - run_id=run_id, - ids=[str(application_revision_ref.id)], - ) - # ---------------------------------------------------------------------- - - # flatten scenario sources --------------------------------------------- - scenario_specs = [ - { - "input_step_key": input_spec["step_key"], - "testset": input_spec["testset"], - "testset_revision": input_spec["testset_revision"], - "testcase": testcase, - "testcase_data": testcase_data, - } - for input_spec in input_specs - for testcase, testcase_data in zip( - input_spec["testcases"], - input_spec["testcases_data"], - ) - ] - nof_scenarios = len(scenario_specs) - # ---------------------------------------------------------------------- - - # fetch application ---------------------------------------------------- - application_revision = await applications_service.fetch_application_revision( - project_id=project_id, - application_revision_ref=application_revision_ref, - ) - - if application_revision is None: - raise ValueError( - f"App revision with id {application_revision_ref.id} not found!" - ) - - application_variant = await applications_service.fetch_application_variant( - project_id=project_id, - application_variant_ref=Reference( - id=application_revision.application_variant_id - ), - ) - - if application_variant is None: - raise ValueError( - f"Application variant with id {application_revision.application_variant_id} not found!" - ) - - application = await applications_service.fetch_application( - project_id=project_id, - application_ref=Reference(id=application_variant.application_id), - ) - - if application is None: - raise ValueError( - f"Application with id {application_variant.application_id} not found!" - ) - - uri = _resolve_runtime_uri(revision_data=application_revision.data) - - if not uri: - raise ValueError( - f"No deployment URI found for revision {application_revision_ref.id}!" - ) - - # fetch evaluators ----------------------------------------------------- - evaluator_references = {step.key: step.references for step in annotation_steps} - # log.debug( - # "[EVALUATORS] ", - # run_id=run_id, - # count=len(annotation_steps), - # refs={ - # step_key: ( - # { - # key: str(reference.id) - # if getattr(reference, "id", None) - # else None - # for key, reference in (references or {}).items() - # } - # ) - # for step_key, references in evaluator_references.items() - # }, - # ) - - evaluators = {} - for evaluator_key, evaluator_refs in evaluator_references.items(): - evaluators[evaluator_key] = await workflows_service.fetch_workflow_revision( - project_id=project_id, - # - workflow_revision_ref=evaluator_refs.get("evaluator_revision"), - ) - # log.debug( - # "[EVALUATORS] [FETCH]", - # run_id=run_id, - # resolved={ - # evaluator_key: ( - # str(evaluator_revision.id) - # if evaluator_revision and evaluator_revision.id - # else None - # ) - # for evaluator_key, evaluator_revision in evaluators.items() - # }, - # ) - # ---------------------------------------------------------------------- - - # create scenarios ----------------------------------------------------- - scenarios_create = [ - EvaluationScenarioCreate( - run_id=run_id, - # - status=EvaluationStatus.RUNNING, - ) - for _ in range(nof_scenarios) - ] - - scenarios = await evaluations_service.create_scenarios( - project_id=project_id, - user_id=user_id, - # - scenarios=scenarios_create, - ) - - if len(scenarios) != nof_scenarios: - raise ValueError(f"Failed to create evaluation scenarios for run {run_id}!") - # ---------------------------------------------------------------------- - - # create input steps --------------------------------------------------- - results_create = [ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=scenario_specs[idx]["input_step_key"], - repeat_idx=repeat_idx, - # - status=EvaluationStatus.SUCCESS, - # - testcase_id=scenario_specs[idx]["testcase"].id, - ) - for idx, scenario in enumerate(scenarios) - for repeat_idx in repeat_indices - ] - - steps = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - # - results=results_create, - ) - - if len(steps) != nof_scenarios * len(repeat_indices): - raise ValueError(f"Failed to create evaluation steps for run {run_id}!") - # ---------------------------------------------------------------------- - - # flatten testcases ---------------------------------------------------- - _testcases = [ - scenario_spec["testcase"].model_dump(mode="json") - for scenario_spec in scenario_specs - ] - - log.info( - "[BATCH] ", - run_id=run_id, - ids=testset_revision_ids, - count=len(_testcases), - size=len(dumps(_testcases).encode("utf-8")), - ) - # ---------------------------------------------------------------------- - - run_has_errors = 0 - run_has_pending = False - run_status = EvaluationStatus.SUCCESS - - # run invocations / evaluators ----------------------------------------- - for idx in range(nof_scenarios): - scenario = scenarios[idx] - scenario_spec = scenario_specs[idx] - testcase = scenario_spec["testcase"] - testcase_data = scenario_spec["testcase_data"] - testset_revision = scenario_spec["testset_revision"] - - scenario_has_errors = 0 - scenario_has_pending = False - scenario_status = EvaluationStatus.SUCCESS - application_references = { - "testcase": {"id": str(testcase.id)}, - **revision_references( - revision=testset_revision, - entity_type="testset", - ), - **revision_references( - revision=application_revision, - entity_type="application", - ), - } - - application_hash_id = make_hash( - references=application_references, - links=None, - ) - cached_application_traces = [] - if is_cached and application_hash_id: - cached_application_traces = await fetch_traces_by_hash( - tracing_service, - project_id, - hash_id=application_hash_id, - limit=application_required_count, - ) - - cached_application_contexts = [] - for reusable_trace in select_traces_for_reuse( - traces=cached_application_traces, - required_count=application_required_count, - ): - reusable_context = _build_trace_context(trace=reusable_trace) - if reusable_context: - cached_application_contexts.append(reusable_context) - - missing_application_count = plan_missing_traces( - required_count=application_required_count, - reusable_count=len(cached_application_contexts), - ) - - invoked_application_contexts = [] - if missing_application_count > 0: - invocations: List[ - InvokationResult - ] = await llm_apps_service.batch_invoke( - project_id=str(project_id), - user_id=str(user_id), - testset_data=[ - testcase_data for _ in range(missing_application_count) - ], # type: ignore[arg-type] - revision=application_revision, - uri=uri, - rate_limit_config=run_config, - application_id=str(application.id), - references=_serialize_references(application_references), - scenarios=[ - scenario.model_dump( - mode="json", - exclude_none=True, - ) - for _ in range(missing_application_count) - ], - ) - - if len(invocations) != missing_application_count: - raise ValueError( - f"Unexpected batch invocation count for scenario {scenario.id}!" - ) - - for invocation in invocations: - invocation_error = ( - invocation.result.error.model_dump(mode="json") - if invocation.result and invocation.result.error - else None - ) - invoked_trace = None - if not invocation_error and invocation.trace_id: - invoked_trace = await fetch_trace( - tracing_service=tracing_service, - project_id=project_id, - trace_id=invocation.trace_id, - ) - - invocation_context = ( - _build_trace_context( - trace=invoked_trace, - error=invocation_error, - ) - if invoked_trace - else None - ) - if invocation_context: - invoked_application_contexts.append(invocation_context) - else: - invoked_application_contexts.append( - { - "trace": invoked_trace, - "trace_id": invocation.trace_id, - "span_id": invocation.span_id, - "root_span": None, - "error": invocation_error - or { - "message": "Invocation trace missing or malformed." - }, - } - ) - - application_contexts = ( - cached_application_contexts + invoked_application_contexts - ) - application_context_by_repeat: Dict[int, Dict[str, Any]] = {} - if is_split: - for repeat_idx, context in zip(repeat_indices, application_contexts): - application_context_by_repeat[repeat_idx] = context - else: - shared_context = ( - application_contexts[0] if application_contexts else None - ) - if shared_context: - for repeat_idx in repeat_indices: - application_context_by_repeat[repeat_idx] = shared_context - - invocation_results_create = [] - scenario_invocation_failed = False - for repeat_idx in repeat_indices: - application_context = application_context_by_repeat.get(repeat_idx) - application_error = ( - application_context.get("error") - if application_context - else {"message": "Invocation trace missing."} - ) - has_invocation_error = not ( - application_context - and application_context.get("trace_id") - and application_context.get("root_span") - and not application_error - ) - if has_invocation_error: - scenario_invocation_failed = True - - invocation_results_create.append( - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=invocation_step_key, - repeat_idx=repeat_idx, - status=( - EvaluationStatus.FAILURE - if has_invocation_error - else EvaluationStatus.SUCCESS - ), - trace_id=( - application_context.get("trace_id") - if application_context - else None - ), - error=application_error if has_invocation_error else None, - ) - ) - - created_invocation_results = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - results=invocation_results_create, - ) - if len(created_invocation_results) != len(repeat_indices): - raise ValueError( - f"Failed to create invocation results for scenario {scenario.id}!" - ) - - if scenario_invocation_failed: - scenario_has_errors += 1 - - for annotation_step in annotation_steps: - annotation_step_key = annotation_step.key - - if annotation_step.origin in {"human", "custom"}: - # log.debug( - # "[EVALUATOR] [SKIP]", - # run_id=run_id, - # scenario_id=scenario.id, - # step_key=annotation_step_key, - # origin=annotation_step.origin, - # reason="non-auto-origin", - # ) - scenario_has_pending = True - run_has_pending = True - continue - - evaluator_revision = evaluators.get(annotation_step_key) - if not evaluator_revision: - # log.warning( - # "[EVALUATOR] [MISSING]", - # run_id=run_id, - # scenario_id=scenario.id, - # step_key=annotation_step_key, - # references={ - # key: str(reference.id) - # if getattr(reference, "id", None) - # else None - # for key, reference in ( - # evaluator_references.get(annotation_step_key, {}) or {} - # ).items() - # }, - # ) - log.error( - f"Evaluator revision for {annotation_step_key} not found!" - ) - scenario_has_errors += 1 - scenario_status = EvaluationStatus.ERRORS - continue - - _revision = evaluator_revision.model_dump( - mode="json", - exclude_none=True, - ) - interface = ( - dict( - uri=evaluator_revision.data.uri, - url=evaluator_revision.data.url, - headers=evaluator_revision.data.headers, - schemas=evaluator_revision.data.schemas, - ) - if evaluator_revision.data - else dict() - ) - configuration = ( - dict( - script=evaluator_revision.data.script, - parameters=evaluator_revision.data.parameters, - ) - if evaluator_revision.data - else dict() - ) - parameters = configuration.get("parameters") - flags = ( - evaluator_revision.flags.model_dump( - mode="json", - exclude_none=True, - exclude_unset=True, - ) - if evaluator_revision.flags - else None - ) - - base_references: Dict[str, Any] = { - "testcase": {"id": str(testcase.id)}, - **revision_references( - revision=testset_revision, - entity_type="testset", - ), - **revision_references( - revision=evaluator_revision, - entity_type="evaluator", - ), - } - - evaluator_results_create = [] - if not is_split: - shared_application_context = application_context_by_repeat.get( - repeat_indices[0] - ) - # log.debug( - # "[EVALUATOR] [PLAN]", - # run_id=run_id, - # scenario_id=scenario.id, - # step_key=annotation_step_key, - # repeats=repeat_indices, - # is_split=is_split, - # has_shared_application_context=bool(shared_application_context), - # has_shared_root_span=bool( - # shared_application_context - # and shared_application_context.get("root_span") - # ), - # ) - if ( - not shared_application_context - or not shared_application_context.get("root_span") - ): - scenario_has_errors += 1 - scenario_status = EvaluationStatus.ERRORS - evaluator_results_create = [ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=annotation_step_key, - repeat_idx=repeat_idx, - status=EvaluationStatus.FAILURE, - error={ - "message": "Evaluator skipped because invocation trace is missing." - }, - ) - for repeat_idx in repeat_indices - ] - else: - shared_trace = shared_application_context["trace"] - shared_root_span = shared_application_context["root_span"] - shared_links = { - invocation_step_key: { - "trace_id": shared_application_context["trace_id"], - "span_id": shared_application_context["span_id"], - } - } - workflow_service_request_data = WorkflowServiceRequestData( - revision=_revision, - parameters=parameters, - testcase=testcase.model_dump(mode="json"), - inputs=testcase.data, - trace=shared_trace.model_dump( - mode="json", - exclude_none=True, - ) - if shared_trace - else None, - outputs=( - ( - shared_root_span.model_dump( - mode="json", - exclude_none=True, - ) - .get("attributes", {}) - .get("ag", {}) - .get("data", {}) - ).get("outputs") - if shared_root_span - else None - ), - ) - workflow_service_request = WorkflowServiceRequest( - version="2025.07.14", - flags=flags, - interface=interface, - configuration=configuration, - data=workflow_service_request_data, - references=base_references, - links=shared_links, - ) - evaluator_hash_id = make_hash( - references=base_references, - links=shared_links, - ) - cached_evaluator_traces = [] - if is_cached and evaluator_hash_id: - cached_evaluator_traces = await fetch_traces_by_hash( - tracing_service, - project_id, - hash_id=evaluator_hash_id, - limit=evaluator_required_count, - ) - - reusable_evaluator_traces = select_traces_for_reuse( - traces=cached_evaluator_traces, - required_count=evaluator_required_count, - ) - evaluator_results_create.extend( - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=annotation_step_key, - repeat_idx=repeat_idx, - status=EvaluationStatus.SUCCESS, - trace_id=str(reusable_trace.trace_id), - ) - for repeat_idx, reusable_trace in zip( - repeat_indices, - reusable_evaluator_traces, - ) - if reusable_trace and reusable_trace.trace_id - ) - - for repeat_idx in repeat_indices[ - len(reusable_evaluator_traces) : - ]: - # log.debug( - # "[EVALUATOR] [INVOKE]", - # run_id=run_id, - # scenario_id=scenario.id, - # step_key=annotation_step_key, - # repeat_idx=repeat_idx, - # cached_reuse_count=len(reusable_evaluator_traces), - # trace_links=shared_links, - # ) - workflows_service_response = ( - await workflows_service.invoke_workflow( - project_id=project_id, - user_id=user_id, - request=workflow_service_request, - annotate=True, - ) - ) - has_error = workflows_service_response.status.code != 200 - result_trace_id = workflows_service_response.trace_id - result_error = None - result_status = EvaluationStatus.SUCCESS - - if has_error: - result_status = EvaluationStatus.FAILURE - result_error = ( - workflows_service_response.status.model_dump( - mode="json", - exclude_none=True, - ) - ) - scenario_has_errors += 1 - scenario_status = EvaluationStatus.ERRORS - elif result_trace_id: - fetched_evaluator_trace = await fetch_trace( - tracing_service=tracing_service, - project_id=project_id, - trace_id=result_trace_id, - ) - if not fetched_evaluator_trace: - result_status = EvaluationStatus.FAILURE - result_error = { - "message": "Evaluator trace missing after invocation." - } - scenario_has_errors += 1 - scenario_status = EvaluationStatus.ERRORS - else: - result_status = EvaluationStatus.FAILURE - result_error = { - "message": "Evaluator trace_id is missing." - } - scenario_has_errors += 1 - scenario_status = EvaluationStatus.ERRORS - - evaluator_results_create.append( - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=annotation_step_key, - repeat_idx=repeat_idx, - status=result_status, - trace_id=result_trace_id, - error=result_error, - ) - ) - else: - for repeat_idx in repeat_indices: - application_context = application_context_by_repeat.get( - repeat_idx - ) - # log.debug( - # "[EVALUATOR] [PLAN]", - # run_id=run_id, - # scenario_id=scenario.id, - # step_key=annotation_step_key, - # repeat_idx=repeat_idx, - # is_split=is_split, - # has_application_context=bool(application_context), - # has_root_span=bool( - # application_context - # and application_context.get("root_span") - # ), - # ) - if not application_context or not application_context.get( - "root_span" - ): - scenario_has_errors += 1 - scenario_status = EvaluationStatus.ERRORS - evaluator_results_create.append( - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=annotation_step_key, - repeat_idx=repeat_idx, - status=EvaluationStatus.FAILURE, - error={ - "message": "Evaluator skipped because invocation trace is missing." - }, - ) - ) - continue - - application_trace = application_context["trace"] - application_root_span = application_context["root_span"] - application_root_span_data = ( - application_root_span.model_dump( - mode="json", - exclude_none=True, - ) - .get("attributes", {}) - .get("ag", {}) - .get("data", {}) - ) - links = { - invocation_step_key: { - "trace_id": application_context["trace_id"], - "span_id": application_context["span_id"], - } - } - workflow_service_request = WorkflowServiceRequest( - version="2025.07.14", - flags=flags, - interface=interface, - configuration=configuration, - data=WorkflowServiceRequestData( - revision=_revision, - parameters=parameters, - testcase=testcase.model_dump(mode="json"), - inputs=testcase.data, - trace=application_trace.model_dump( - mode="json", - exclude_none=True, - ) - if application_trace - else None, - outputs=application_root_span_data.get("outputs"), - ), - references=base_references, - links=links, - ) - evaluator_hash_id = make_hash( - references=base_references, - links=links, - ) - cached_evaluator_trace = None - if is_cached and evaluator_hash_id: - cached_matches = await fetch_traces_by_hash( - tracing_service, - project_id, - hash_id=evaluator_hash_id, - limit=1, - ) - reusable_match = select_traces_for_reuse( - traces=cached_matches, - required_count=1, - ) - if reusable_match: - cached_evaluator_trace = reusable_match[0] - - if cached_evaluator_trace and cached_evaluator_trace.trace_id: - evaluator_results_create.append( - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=annotation_step_key, - repeat_idx=repeat_idx, - status=EvaluationStatus.SUCCESS, - trace_id=str(cached_evaluator_trace.trace_id), - ) - ) - continue - - # log.debug( - # "[EVALUATOR] [INVOKE]", - # run_id=run_id, - # scenario_id=scenario.id, - # step_key=annotation_step_key, - # repeat_idx=repeat_idx, - # trace_links=links, - # ) - workflows_service_response = ( - await workflows_service.invoke_workflow( - project_id=project_id, - user_id=user_id, - request=workflow_service_request, - annotate=True, - ) - ) - - result_trace_id = workflows_service_response.trace_id - result_error = None - result_status = EvaluationStatus.SUCCESS - has_error = workflows_service_response.status.code != 200 - if has_error: - result_status = EvaluationStatus.FAILURE - result_error = workflows_service_response.status.model_dump( - mode="json", - exclude_none=True, - ) - scenario_has_errors += 1 - scenario_status = EvaluationStatus.ERRORS - elif result_trace_id: - fetched_evaluator_trace = await fetch_trace( - tracing_service=tracing_service, - project_id=project_id, - trace_id=result_trace_id, - ) - if not fetched_evaluator_trace: - result_status = EvaluationStatus.FAILURE - result_error = { - "message": "Evaluator trace missing after invocation." - } - scenario_has_errors += 1 - scenario_status = EvaluationStatus.ERRORS - else: - result_status = EvaluationStatus.FAILURE - result_error = {"message": "Evaluator trace_id is missing."} - scenario_has_errors += 1 - scenario_status = EvaluationStatus.ERRORS - - evaluator_results_create.append( - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=annotation_step_key, - repeat_idx=repeat_idx, - status=result_status, - trace_id=result_trace_id, - error=result_error, - ) - ) - - created_annotation_results = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - results=evaluator_results_create, - ) - # log.debug( - # "[EVALUATOR] [RESULTS]", - # run_id=run_id, - # scenario_id=scenario.id, - # step_key=annotation_step_key, - # created=len(created_annotation_results), - # expected=len(repeat_indices), - # ) - - if len(created_annotation_results) != len(repeat_indices): - raise ValueError( - f"Failed to create evaluation results for scenario with id {scenario.id}!" - ) - - final_scenario_status = ( - EvaluationStatus.PENDING - if scenario_status == EvaluationStatus.SUCCESS and scenario_has_pending - else scenario_status - ) - - if final_scenario_status == EvaluationStatus.ERRORS: - run_has_errors += 1 - - scenario_edit = EvaluationScenarioEdit( - id=scenario.id, - tags=scenario.tags, - meta=scenario.meta, - status=final_scenario_status, - ) - - scenario = await evaluations_service.edit_scenario( - project_id=project_id, - user_id=user_id, - # - scenario=scenario_edit, - ) - - if not scenario: - raise ValueError( - f"Failed to edit evaluation scenario with id {scenario.id}!" - ) - - if scenario_status != EvaluationStatus.FAILURE: - try: - metrics = await evaluations_service.refresh_metrics( - project_id=project_id, - user_id=user_id, - # - metrics=EvaluationMetricsRefresh( - run_id=run_id, - scenario_id=scenario.id, - ), - ) - - if not metrics: - log.warning( - f"Refreshing metrics failed for {run_id} | {scenario.id}" - ) - - except Exception: - log.warning( - f"Refreshing metrics failed for {run_id} | {scenario.id}", - exc_info=True, - ) - # ---------------------------------------------------------------------- - - if run_status != EvaluationStatus.FAILURE: - if run_has_errors: - run_status = EvaluationStatus.ERRORS - elif run_has_pending: - run_status = EvaluationStatus.RUNNING - else: - run_status = EvaluationStatus.SUCCESS - - except Exception as e: # pylint: disable=broad-exception-caught - log.error( - f"An error occurred during evaluation: {e}", - exc_info=True, - ) - - run_status = EvaluationStatus.FAILURE - - if not run: - log.info("[FAIL] ", run_id=run_id, project_id=project_id, user_id=user_id) - return - - if run_status != EvaluationStatus.FAILURE: - try: - metrics = await evaluations_service.refresh_metrics( - project_id=project_id, - user_id=user_id, - # - metrics=EvaluationMetricsRefresh( - run_id=run_id, - ), - ) - - if not metrics: - log.warning(f"Refreshing metrics failed for {run_id}") - - run_status = EvaluationStatus.FAILURE - - except Exception: # pylint: disable=broad-exception-caught - log.warning(f"Refreshing metrics failed for {run_id}", exc_info=True) - - run_status = EvaluationStatus.FAILURE - - # edit evaluation run status ----------------------------------------------- - run_edit = EvaluationRunEdit( - id=run_id, - # - name=run.name, - description=run.description, - # - tags=run.tags, - meta=run.meta, - # - status=run_status, - flags=run.flags, - # - data=run.data, - ) - - await evaluations_service.edit_run( - project_id=project_id, - user_id=user_id, - # - run=run_edit, - ) - - log.info("[DONE] ", run_id=run_id, project_id=project_id, user_id=user_id) - - return - - -async def evaluate_batch_invocation( - *, - project_id: UUID, - user_id: UUID, - # - run_id: UUID, - # - tracing_service: TracingService, - testsets_service: TestsetsService, - applications_service: ApplicationsService, - evaluations_service: EvaluationsService, -): - """ - Run batch invocation over a testset without evaluator steps. - - This loop creates scenarios and input/invocation results, but does not - invoke evaluator workflows and does not refresh evaluation metrics. - """ - run = None - run_status = EvaluationStatus.SUCCESS - - try: - # ---------------------------------------------------------------------- - log.info( - "[SCOPE] ", run_id=run_id, project_id=project_id, user_id=user_id - ) - # ---------------------------------------------------------------------- - - # fetch run ------------------------------------------------------------ - run = await evaluations_service.fetch_run( - project_id=project_id, - run_id=run_id, - ) - - if not run: - raise ValueError(f"Evaluation run with id {run_id} not found!") - if not run.data or not run.data.steps: - raise ValueError(f"Evaluation run with id {run_id} has no steps!") - repeats = run.data.repeats or 1 - repeat_indices = build_repeat_indices(repeats) - is_cached = bool(run.flags.is_cached) if run.flags else False - application_required_count = required_traces_for_step( - repeats=repeats, - is_split=False, - step_kind="application", - has_evaluator_steps=False, - ) - - steps = run.data.steps - input_steps = [step for step in steps if step.type == "input"] - invocation_steps = [step for step in steps if step.type == "invocation"] - annotation_steps = [step for step in steps if step.type == "annotation"] - - if annotation_steps: - raise ValueError( - f"Evaluation run with id {run_id} contains annotation steps; " - "use evaluate_batch_testset instead." - ) - if not input_steps or len(invocation_steps) != 1: - raise ValueError( - f"Evaluation run with id {run_id} must have at least one input and exactly one invocation step." - ) - - invocation_step_key = invocation_steps[0].key - invocation_refs = invocation_steps[0].references or {} - - application_revision_ref = invocation_refs.get("application_revision") - if not application_revision_ref or not isinstance( - application_revision_ref.id, UUID - ): - raise ValueError( - f"Evaluation run with id {run_id} missing invocation.application_revision reference." - ) - # ---------------------------------------------------------------------- - - input_specs = await _resolve_testset_input_specs( - project_id=project_id, - input_steps=input_steps, - testsets_service=testsets_service, - ) - scenario_specs = [ - { - "input_step_key": input_spec["step_key"], - "testset": input_spec["testset"], - "testset_revision": input_spec["testset_revision"], - "testcase": testcase, - "testcase_data": testcase_data, - } - for input_spec in input_specs - for testcase, testcase_data in zip( - input_spec["testcases"], - input_spec["testcases_data"], - ) - ] - nof_scenarios = len(scenario_specs) - # ---------------------------------------------------------------------- - - # fetch application ---------------------------------------------------- - application_revision = await applications_service.fetch_application_revision( - project_id=project_id, - application_revision_ref=application_revision_ref, - ) - if not application_revision: - raise ValueError( - f"Application revision with id {application_revision_ref.id} not found!" - ) - - application_variant = await applications_service.fetch_application_variant( - project_id=project_id, - application_variant_ref=Reference( - id=application_revision.application_variant_id - ), - ) - if not application_variant: - raise ValueError( - f"Application variant with id {application_revision.application_variant_id} not found!" - ) - - application = await applications_service.fetch_application( - project_id=project_id, - application_ref=Reference(id=application_variant.application_id), - ) - if not application: - raise ValueError( - f"Application with id {application_variant.application_id} not found!" - ) - - uri = _resolve_runtime_uri(revision_data=application_revision.data) - if not uri: - raise ValueError( - f"No deployment URI found for revision {application_revision_ref.id}!" - ) - - # create scenarios ----------------------------------------------------- - scenarios = await evaluations_service.create_scenarios( - project_id=project_id, - user_id=user_id, - scenarios=[ - EvaluationScenarioCreate( - run_id=run_id, - status=EvaluationStatus.RUNNING, - ) - for _ in range(nof_scenarios) - ], - ) - if len(scenarios) != nof_scenarios: - raise ValueError(f"Failed to create evaluation scenarios for run {run_id}!") - # ---------------------------------------------------------------------- - - # create input results ------------------------------------------------- - input_results = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - results=[ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=scenario_specs[idx]["input_step_key"], - repeat_idx=repeat_idx, - status=EvaluationStatus.SUCCESS, - testcase_id=scenario_specs[idx]["testcase"].id, - ) - for idx, scenario in enumerate(scenarios) - for repeat_idx in repeat_indices - ], - ) - if len(input_results) != nof_scenarios * len(repeat_indices): - raise ValueError(f"Failed to create input results for run {run_id}!") - # ---------------------------------------------------------------------- - - # resolve cache / invoke application ----------------------------------- - run_config = { - "batch_size": 10, - "max_retries": 3, - "retry_delay": 3, - "delay_between_batches": 5, - } - scenario_invocations: Dict[tuple[int, int], Dict[str, Any]] = {} - for idx, scenario in enumerate(scenarios): - scenario_spec = scenario_specs[idx] - testcase = scenario_spec["testcase"] - testcase_data = scenario_spec["testcase_data"] - testset_revision = scenario_spec["testset_revision"] - references = { - "testcase": {"id": str(testcase.id)}, - **revision_references( - revision=testset_revision, - entity_type="testset", - ), - **revision_references( - revision=application_revision, - entity_type="application", - ), - } - hash_id = make_hash(references=references, links=None) - cached_traces = [] - if is_cached and hash_id: - cached_traces = await fetch_traces_by_hash( - tracing_service, - project_id, - hash_id=hash_id, - limit=application_required_count, - ) - reusable_traces = select_traces_for_reuse( - traces=cached_traces, - required_count=application_required_count, - ) - for repeat_idx, reusable_trace in zip(repeat_indices, reusable_traces): - scenario_invocations[(idx, repeat_idx)] = { - "status": EvaluationStatus.SUCCESS, - "trace_id": ( - str(reusable_trace.trace_id) - if reusable_trace and reusable_trace.trace_id - else None - ), - "error": None, - } - - missing_repeat_indices = repeat_indices[len(reusable_traces) :] - if missing_repeat_indices: - invocations = await llm_apps_service.batch_invoke( - project_id=str(project_id), - user_id=str(user_id), - testset_data=[ - testcase_data for _ in range(len(missing_repeat_indices)) - ], # type: ignore[arg-type] - revision=application_revision, - uri=uri, - rate_limit_config=run_config, - application_id=str(application.id), - references=_serialize_references(references), - scenarios=[ - scenario.model_dump( - mode="json", - exclude_none=True, - ) - for _ in range(len(missing_repeat_indices)) - ], - ) - if len(invocations) != len(missing_repeat_indices): - raise ValueError( - f"Unexpected batch invocation count for scenario {scenario.id}!" - ) - for repeat_idx, invocation in zip(missing_repeat_indices, invocations): - invocation_error = ( - invocation.result.error.model_dump(mode="json") - if invocation.result and invocation.result.error - else None - ) - scenario_invocations[(idx, repeat_idx)] = { - "status": ( - EvaluationStatus.FAILURE - if invocation_error - else EvaluationStatus.SUCCESS - ), - "trace_id": invocation.trace_id, - "error": invocation_error, - } - # ---------------------------------------------------------------------- - - # create invocation results + finalize scenarios ------------------------ - run_has_errors = 0 - invocation_results = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - results=[ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=invocation_step_key, - repeat_idx=repeat_idx, - status=( - scenario_invocations.get((idx, repeat_idx), {}).get("status") - or EvaluationStatus.FAILURE - ), - trace_id=scenario_invocations.get((idx, repeat_idx), {}).get( - "trace_id" - ), - error=scenario_invocations.get((idx, repeat_idx), {}).get("error"), - ) - for idx, scenario in enumerate(scenarios) - for repeat_idx in repeat_indices - ], - ) - if len(invocation_results) != nof_scenarios * len(repeat_indices): - raise ValueError(f"Failed to create invocation results for run {run_id}!") - - for idx, scenario in enumerate(scenarios): - scenario_status = ( - EvaluationStatus.SUCCESS - if all( - scenario_invocations.get((idx, repeat_idx), {}).get("status") - == EvaluationStatus.SUCCESS - for repeat_idx in repeat_indices - ) - else EvaluationStatus.ERRORS - ) - if not all( - scenario_invocations.get((idx, repeat_idx), {}).get("status") - == EvaluationStatus.SUCCESS - for repeat_idx in repeat_indices - ): - run_has_errors += 1 - - edited_scenario = await evaluations_service.edit_scenario( - project_id=project_id, - user_id=user_id, - scenario=EvaluationScenarioEdit( - id=scenario.id, - tags=scenario.tags, - meta=scenario.meta, - status=scenario_status, - ), - ) - if not edited_scenario: - raise ValueError( - f"Failed to edit evaluation scenario with id {scenario.id}!" - ) - - if run_has_errors: - run_status = EvaluationStatus.ERRORS - # ---------------------------------------------------------------------- - - except Exception as e: # pylint: disable=broad-exception-caught - log.error( - f"An error occurred during batch invocation: {e}", - exc_info=True, - ) - run_status = EvaluationStatus.FAILURE - - if not run: - log.info("[FAIL] ", run_id=run_id, project_id=project_id, user_id=user_id) - return - - await evaluations_service.edit_run( - project_id=project_id, - user_id=user_id, - run=EvaluationRunEdit( - id=run_id, - name=run.name, - description=run.description, - tags=run.tags, - meta=run.meta, - status=run_status, - flags=run.flags, - data=run.data, - ), - ) - - log.info("[DONE] ", run_id=run_id, project_id=project_id, user_id=user_id) - return - - -async def _evaluate_batch_items( - *, - project_id: UUID, - user_id: UUID, - # - run_id: UUID, - # - testcase_ids: Optional[List[UUID]] = None, - trace_ids: Optional[List[str]] = None, - input_step_key: Optional[str] = None, - # - tracing_service: Optional[TracingService] = None, - testcases_service: Optional[TestcasesService] = None, - workflows_service: WorkflowsService, - evaluations_service: EvaluationsService, -): - request = Request( - scope={ - "type": "http", - "http_version": "1.1", - "scheme": "http", - } - ) - request.state.project_id = str(project_id) - request.state.user_id = str(user_id) - - run: Optional[EvaluationRun] = None - scenarios = [] - run_status = EvaluationStatus.SUCCESS - - try: - run = await evaluations_service.fetch_run( - project_id=project_id, - run_id=run_id, - ) - if not run: - raise ValueError(f"Evaluation run with id {run_id} not found!") - if not run.flags or not run.flags.is_queue: - raise ValueError( - f"Evaluation run with id {run_id} is not configured for ad-hoc batching!" - ) - if not run.data or not run.data.steps: - raise ValueError(f"Evaluation run with id {run_id} has no data steps!") - repeats = run.data.repeats or 1 - repeat_indices = build_repeat_indices(repeats) - is_cached = bool(run.flags.is_cached) - - testcase_ids = testcase_ids or [] - trace_ids = trace_ids or [] - if not testcase_ids and not trace_ids: - raise ValueError( - f"Evaluation run with id {run_id} has no testcase_ids or trace_ids!" - ) - if trace_ids and tracing_service is None: - raise ValueError("tracing_service is required for trace batches") - if testcase_ids and testcases_service is None: - raise ValueError("testcases_service is required for testcase batches") - - steps = run.data.steps - input_steps = [step for step in steps if step.type == "input"] - invocation_steps = [step for step in steps if step.type == "invocation"] - annotation_steps = [step for step in steps if step.type == "annotation"] - - if input_step_key is not None: - matching_input_step = next( - (step for step in input_steps if step.key == input_step_key), - None, - ) - if matching_input_step is None: - raise ValueError( - f"Evaluation run with id {run_id} has no input step '{input_step_key}'!" - ) - else: - input_step_key = input_steps[0].key if input_steps else None - invocation_step_key = invocation_steps[0].key if invocation_steps else None - evaluator_references = { - step.key: step.references or {} for step in annotation_steps - } - evaluator_revisions: Dict[str, Any] = {} - for annotation_step_key, annotation_refs in evaluator_references.items(): - evaluator_revision_ref = annotation_refs.get("evaluator_revision") - evaluator_revisions[annotation_step_key] = ( - await workflows_service.fetch_workflow_revision( - project_id=project_id, - workflow_revision_ref=evaluator_revision_ref, - ) - if evaluator_revision_ref - else None - ) - - testcases = ( - await testcases_service.fetch_testcases( - project_id=project_id, - testcase_ids=testcase_ids, - ) - if testcase_ids - else [] - ) - testcases_by_id = { - testcase.id: testcase for testcase in testcases if testcase.id - } - - scenario_items = [] - for testcase_id in testcase_ids: - testcase = testcases_by_id.get(testcase_id) - scenario_items.append( - dict( - kind="testcase", - testcase=testcase, - testcase_id=testcase_id, - trace_id=None, - ) - ) - for trace_id in trace_ids: - scenario_items.append( - dict( - kind="trace", - testcase=None, - testcase_id=None, - trace_id=trace_id, - ) - ) - - scenarios = await evaluations_service.create_scenarios( - project_id=project_id, - user_id=user_id, - scenarios=[ - EvaluationScenarioCreate( - run_id=run_id, - status=EvaluationStatus.RUNNING, - ) - for _ in scenario_items - ], - ) - if len(scenarios) != len(scenario_items): - raise ValueError(f"Failed to create scenarios for run {run_id}") - - run_has_errors = False - run_has_pending = False - - for idx, scenario in enumerate(scenarios): - scenario_status = EvaluationStatus.SUCCESS - scenario_has_pending = False - scenario_item = scenario_items[idx] - - source_testcase = scenario_item["testcase"] - source_testcase_id = scenario_item["testcase_id"] - source_trace_id = scenario_item["trace_id"] - - _trace = None - inputs = None - outputs = None - query_span_id = None - - if source_testcase_id and source_testcase is None: - run_has_errors = True - scenario_status = EvaluationStatus.ERRORS - await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - results=[ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=step.key, - repeat_idx=repeat_idx, - status=EvaluationStatus.ERRORS, - testcase_id=source_testcase_id, - error={ - "message": f"Testcase {source_testcase_id} not found." - }, - ) - for step in annotation_steps - for repeat_idx in repeat_indices - ], - ) - - if source_testcase_id and source_testcase and input_step_key: - input_results = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - results=[ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=input_step_key, - repeat_idx=repeat_idx, - status=EvaluationStatus.SUCCESS, - testcase_id=source_testcase_id, - ) - for repeat_idx in repeat_indices - ], - ) - if len(input_results) != len(repeat_indices): - raise ValueError( - f"Failed to create input result for scenario {scenario.id}" - ) - - if source_testcase and source_testcase.data: - inputs = source_testcase.data - - if source_trace_id: - trace = await fetch_trace( - project_id=project_id, - trace_id=source_trace_id, - tracing_service=tracing_service, - ) - if not trace or not isinstance(trace.spans, dict): - scenario_status = EvaluationStatus.ERRORS - run_has_errors = True - else: - root_span = list(trace.spans.values())[0] - if isinstance(root_span, list): - scenario_status = EvaluationStatus.ERRORS - run_has_errors = True - else: - query_span_id = root_span.span_id - _trace = trace.model_dump(mode="json", exclude_none=True) - _root_span = root_span.model_dump( - mode="json", exclude_none=True - ) - - root_span_attributes: dict = _root_span.get("attributes") or {} - root_span_ag: dict = root_span_attributes.get("ag") or {} - root_span_ag_data: dict = root_span_ag.get("data") or {} - outputs = root_span_ag_data.get("outputs") - if not inputs: - inputs = root_span_ag_data.get("inputs") - - if ( - source_trace_id - and input_step_key - and scenario_status == EvaluationStatus.SUCCESS - ): - input_results = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - results=[ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=input_step_key, - repeat_idx=repeat_idx, - status=EvaluationStatus.SUCCESS, - trace_id=source_trace_id, - ) - for repeat_idx in repeat_indices - ], - ) - if len(input_results) != len(repeat_indices): - raise ValueError( - f"Failed to create trace input result for scenario {scenario.id}" - ) - - if ( - source_trace_id - and invocation_step_key - and scenario_status == EvaluationStatus.SUCCESS - ): - invocation_results = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - results=[ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=invocation_step_key, - repeat_idx=repeat_idx, - status=EvaluationStatus.SUCCESS, - trace_id=source_trace_id, - ) - for repeat_idx in repeat_indices - ], - ) - if len(invocation_results) != len(repeat_indices): - raise ValueError( - f"Failed to create invocation result for scenario {scenario.id}" - ) - - if scenario_status == EvaluationStatus.SUCCESS: - for annotation_step in annotation_steps: - annotation_step_key = annotation_step.key - if annotation_step.origin in {"human", "custom"}: - scenario_has_pending = True - run_has_pending = True - # Human/custom steps are not auto-invoked here. - # Results are created later by the annotator via the annotation submission flow. - continue - - evaluator_revision = evaluator_revisions.get(annotation_step_key) - if not evaluator_revision: - run_has_errors = True - scenario_status = EvaluationStatus.ERRORS - continue - - _revision = evaluator_revision.model_dump( - mode="json", - exclude_none=True, - ) - interface = ( - dict( - uri=evaluator_revision.data.uri, - url=evaluator_revision.data.url, - headers=evaluator_revision.data.headers, - schemas=evaluator_revision.data.schemas, - ) - if evaluator_revision.data - else dict() - ) - configuration = ( - dict( - script=evaluator_revision.data.script, - parameters=evaluator_revision.data.parameters, - ) - if evaluator_revision.data - else dict() - ) - parameters = configuration.get("parameters") - flags = ( - evaluator_revision.flags.model_dump( - mode="json", - exclude_none=True, - exclude_unset=True, - ) - if evaluator_revision.flags - else None - ) - - links: Dict[str, Any] = {} - source_step_key = invocation_step_key or input_step_key - if source_step_key and source_trace_id and query_span_id: - links[source_step_key] = dict( - trace_id=source_trace_id, - span_id=query_span_id, - ) - - workflow_service_request = WorkflowServiceRequest( - version="2025.07.14", - flags=flags, - interface=interface, - configuration=configuration, - data=WorkflowServiceRequestData( - revision=_revision, - parameters=parameters, - testcase=( - source_testcase.model_dump( - mode="json", exclude_none=True - ) - if source_testcase - else None - ), - inputs=inputs, - trace=_trace, - outputs=outputs, - ), - references=evaluator_references.get(annotation_step_key, {}), - links=links, - ) - hash_references: Dict[str, Any] = { - **(evaluator_references.get(annotation_step_key, {}) or {}) - } - if source_testcase_id: - hash_references["testcase"] = {"id": str(source_testcase_id)} - - hash_id = make_hash( - references=hash_references, - links=links, - ) - cached_traces = [] - if is_cached and hash_id and tracing_service is not None: - cached_traces = await fetch_traces_by_hash( - tracing_service, - project_id, - hash_id=hash_id, - limit=len(repeat_indices), - ) - - reusable_traces = select_traces_for_reuse( - traces=cached_traces, - required_count=len(repeat_indices), - ) - _ = plan_missing_traces( - required_count=len(repeat_indices), - reusable_count=len(reusable_traces), - ) - - results_payload = [ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=annotation_step_key, - repeat_idx=repeat_idx, - status=EvaluationStatus.SUCCESS, - testcase_id=source_testcase_id, - trace_id=str(reusable_trace.trace_id), - ) - for repeat_idx, reusable_trace in zip( - repeat_indices, - reusable_traces, - ) - if reusable_trace and reusable_trace.trace_id - ] - - for repeat_idx in repeat_indices[len(reusable_traces) :]: - workflows_service_response = ( - await workflows_service.invoke_workflow( - project_id=project_id, - user_id=user_id, - request=workflow_service_request, - annotate=True, - ) - ) - - has_error = workflows_service_response.status.code != 200 - result_trace_id = workflows_service_response.trace_id - result_error = None - result_status = EvaluationStatus.SUCCESS - if has_error: - result_status = EvaluationStatus.FAILURE - result_error = workflows_service_response.status.model_dump( - mode="json", - exclude_none=True, - ) - scenario_status = EvaluationStatus.ERRORS - run_has_errors = True - - results_payload.append( - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario.id, - step_key=annotation_step_key, - repeat_idx=repeat_idx, - status=result_status, - testcase_id=source_testcase_id, - trace_id=result_trace_id, - error=result_error, - ) - ) - - step_results = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - results=results_payload, - ) - if len(step_results) != len(repeat_indices): - raise ValueError( - f"Failed to create annotation results for scenario {scenario.id}" - ) - - final_scenario_status = ( - EvaluationStatus.PENDING - if scenario_status == EvaluationStatus.SUCCESS and scenario_has_pending - else scenario_status - ) - await evaluations_service.edit_scenario( - project_id=project_id, - user_id=user_id, - scenario=EvaluationScenarioEdit( - id=scenario.id, - tags=scenario.tags, - meta=scenario.meta, - status=final_scenario_status, - ), - ) - - try: - await evaluations_service.refresh_metrics( - project_id=project_id, - user_id=user_id, - metrics=EvaluationMetricsRefresh( - run_id=run_id, - scenario_id=scenario.id, - ), - ) - except Exception: # pylint: disable=broad-exception-caught - log.warning( - f"Refreshing metrics failed for {run_id} | {scenario.id}", - exc_info=True, - ) - - if run_has_errors: - run_status = EvaluationStatus.ERRORS - elif run_has_pending: - run_status = EvaluationStatus.RUNNING - else: - run_status = EvaluationStatus.SUCCESS - - except Exception as e: # pylint: disable=broad-exception-caught - log.error( - f"An error occurred during batch items evaluation: {e}", - exc_info=True, - ) - run_status = EvaluationStatus.FAILURE - - if not run: - return - - # For ad-hoc/queue runs (multiple independent batches writing to the same run), - # re-fetch the current stored status and never downgrade it to a less severe state. - # This prevents a later successful batch from overwriting ERRORS from an earlier one. - if run.flags and run.flags.is_queue and run_status != EvaluationStatus.FAILURE: - _severity = { - EvaluationStatus.FAILURE: 4, - EvaluationStatus.ERRORS: 3, - EvaluationStatus.RUNNING: 2, - EvaluationStatus.SUCCESS: 1, - EvaluationStatus.PENDING: 0, - } - current_run = await evaluations_service.fetch_run( - project_id=project_id, - run_id=run_id, - ) - if current_run and current_run.status: - stored_severity = _severity.get(current_run.status, 0) - if stored_severity > _severity.get(run_status, 0): - run_status = current_run.status - - try: - if run_status != EvaluationStatus.FAILURE: - await evaluations_service.refresh_metrics( - project_id=project_id, - user_id=user_id, - metrics=EvaluationMetricsRefresh(run_id=run_id), - ) - except Exception: # pylint: disable=broad-exception-caught - log.warning(f"Refreshing metrics failed for {run_id}", exc_info=True) - run_status = EvaluationStatus.FAILURE - - await evaluations_service.edit_run( - project_id=project_id, - user_id=user_id, - run=EvaluationRunEdit( - id=run_id, - name=run.name, - description=run.description, - tags=run.tags, - meta=run.meta, - status=run_status, - flags=run.flags, - data=run.data, - ), - ) - - log.info("[DONE] ", run_id=run_id, project_id=project_id, user_id=user_id) - - return - - -async def evaluate_batch_traces( - *, - project_id: UUID, - user_id: UUID, - # - run_id: UUID, - trace_ids: List[str], - input_step_key: Optional[str] = None, - # - tracing_service: TracingService, - workflows_service: WorkflowsService, - evaluations_service: EvaluationsService, -): - return await _evaluate_batch_items( - project_id=project_id, - user_id=user_id, - run_id=run_id, - # - trace_ids=trace_ids, - input_step_key=input_step_key, - tracing_service=tracing_service, - workflows_service=workflows_service, - evaluations_service=evaluations_service, - ) - - -async def evaluate_batch_testcases( - *, - project_id: UUID, - user_id: UUID, - # - run_id: UUID, - testcase_ids: List[UUID], - input_step_key: Optional[str] = None, - # - tracing_service: TracingService, - testcases_service: TestcasesService, - workflows_service: WorkflowsService, - evaluations_service: EvaluationsService, -): - return await _evaluate_batch_items( - project_id=project_id, - user_id=user_id, - run_id=run_id, - # - testcase_ids=testcase_ids, - input_step_key=input_step_key, - tracing_service=tracing_service, - testcases_service=testcases_service, - workflows_service=workflows_service, - evaluations_service=evaluations_service, - ) diff --git a/api/oss/src/core/evaluations/tasks/live.py b/api/oss/src/core/evaluations/tasks/live.py deleted file mode 100644 index 7a535b64a3..0000000000 --- a/api/oss/src/core/evaluations/tasks/live.py +++ /dev/null @@ -1,921 +0,0 @@ -from typing import Dict, Any, Optional -from uuid import UUID -from datetime import datetime, timezone - -from oss.src.utils.logging import get_module_logger - -from oss.src.dbs.postgres.queries.dbes import ( - QueryArtifactDBE, - QueryVariantDBE, - QueryRevisionDBE, -) -from oss.src.dbs.postgres.testcases.dbes import ( - TestcaseBlobDBE, -) -from oss.src.dbs.postgres.testsets.dbes import ( - TestsetArtifactDBE, - TestsetVariantDBE, - TestsetRevisionDBE, -) -from oss.src.dbs.postgres.workflows.dbes import ( - WorkflowArtifactDBE, - WorkflowVariantDBE, - WorkflowRevisionDBE, -) - -from oss.src.dbs.postgres.tracing.dao import TracingDAO -from oss.src.dbs.postgres.blobs.dao import BlobsDAO -from oss.src.dbs.postgres.git.dao import GitDAO -from oss.src.dbs.postgres.evaluations.dao import EvaluationsDAO - -from oss.src.core.tracing.service import TracingService -from oss.src.core.queries.service import QueriesService -from oss.src.core.testcases.service import TestcasesService -from oss.src.core.testsets.service import TestsetsService -from oss.src.core.testsets.service import SimpleTestsetsService -from oss.src.core.workflows.service import WorkflowsService -from oss.src.core.evaluators.service import EvaluatorsService -from oss.src.core.evaluators.service import SimpleEvaluatorsService -from oss.src.core.evaluations.service import EvaluationsService -from oss.src.core.annotations.service import AnnotationsService - - -from oss.src.core.evaluations.types import ( - EvaluationMetricsRefresh, - EvaluationStatus, - EvaluationScenarioCreate, - EvaluationScenarioEdit, - EvaluationResultCreate, -) -from oss.src.core.shared.dtos import ( - Reference, - Traces, -) -from oss.src.core.tracing.dtos import ( - Filtering, - Windowing, - Formatting, - Format, - Focus, - TracingQuery, - LogicalOperator, -) -from oss.src.core.workflows.dtos import ( - WorkflowServiceRequestData, - WorkflowServiceRequest, -) -from oss.src.core.queries.dtos import ( - QueryRevision, -) -from oss.src.core.evaluators.dtos import ( - EvaluatorRevision, -) - -from oss.src.core.evaluations.utils import ( - build_repeat_indices, - fetch_trace, - fetch_traces_by_hash, - make_hash, - select_traces_for_reuse, -) -from oss.src.core.git.utils import revision_references - - -log = get_module_logger(__name__) - - -# DBS -------------------------------------------------------------------------- - -tracing_dao = TracingDAO() - -testcases_dao = BlobsDAO( - BlobDBE=TestcaseBlobDBE, -) - -queries_dao = GitDAO( - ArtifactDBE=QueryArtifactDBE, - VariantDBE=QueryVariantDBE, - RevisionDBE=QueryRevisionDBE, -) - -testsets_dao = GitDAO( - ArtifactDBE=TestsetArtifactDBE, - VariantDBE=TestsetVariantDBE, - RevisionDBE=TestsetRevisionDBE, -) - -workflows_dao = GitDAO( - ArtifactDBE=WorkflowArtifactDBE, - VariantDBE=WorkflowVariantDBE, - RevisionDBE=WorkflowRevisionDBE, -) - -evaluations_dao = EvaluationsDAO() - -# CORE ------------------------------------------------------------------------- - -tracing_service = TracingService( - tracing_dao=tracing_dao, -) - -queries_service = QueriesService( - queries_dao=queries_dao, -) - -testcases_service = TestcasesService( - testcases_dao=testcases_dao, -) - -testsets_service = TestsetsService( - testsets_dao=testsets_dao, - testcases_service=testcases_service, -) - -simple_testsets_service = SimpleTestsetsService( - testsets_service=testsets_service, -) - -workflows_service = WorkflowsService( - workflows_dao=workflows_dao, -) - -evaluators_service = EvaluatorsService( - workflows_service=workflows_service, -) - -simple_evaluators_service = SimpleEvaluatorsService( - evaluators_service=evaluators_service, -) - -evaluations_service = EvaluationsService( - evaluations_dao=evaluations_dao, - tracing_service=tracing_service, - queries_service=queries_service, - testsets_service=testsets_service, - evaluators_service=evaluators_service, - # -) - -# APIS ------------------------------------------------------------------------- - -annotations_service = AnnotationsService( - tracing_service=tracing_service, - evaluators_service=evaluators_service, - simple_evaluators_service=simple_evaluators_service, -) - -# ------------------------------------------------------------------------------ - - -async def _resolve_query_revisions( - *, - queries_service: QueriesService, - project_id: UUID, - query_revision_refs: Dict[str, Reference], -) -> Dict[str, QueryRevision]: - """Fetch each referenced query revision and skip ones missing identity or data.""" - resolved: Dict[str, QueryRevision] = dict() - - for query_step_key, query_revision_ref in query_revision_refs.items(): - query_revision = await queries_service.fetch_query_revision( - project_id=project_id, - # - query_revision_ref=query_revision_ref, - ) - - if not query_revision: - log.warn( - "Skipping query revision: not found.", - query_step_key=query_step_key, - query_revision_ref=query_revision_ref.model_dump(mode="json"), - ) - continue - - if not query_revision.id or not query_revision.slug: - log.warn( - "Skipping query revision: found but missing identity.", - query_step_key=query_step_key, - query_revision_ref=query_revision_ref.model_dump(mode="json"), - has_id=bool(query_revision.id), - has_slug=bool(query_revision.slug), - ) - continue - - if not query_revision.data: - log.warn( - "Skipping query revision: found but missing data.", - query_step_key=query_step_key, - query_revision_ref=query_revision_ref.model_dump(mode="json"), - ) - continue - - resolved[query_step_key] = query_revision - - return resolved - - -async def _resolve_evaluator_revisions( - *, - evaluators_service: EvaluatorsService, - project_id: UUID, - evaluator_revision_refs: Dict[str, Reference], -) -> Dict[str, EvaluatorRevision]: - """Fetch each referenced evaluator revision and skip ones missing identity or data.""" - resolved: Dict[str, EvaluatorRevision] = dict() - - for evaluator_step_key, evaluator_revision_ref in evaluator_revision_refs.items(): - evaluator_revision = await evaluators_service.fetch_evaluator_revision( - project_id=project_id, - # - evaluator_revision_ref=evaluator_revision_ref, - ) - - if not evaluator_revision: - log.warn( - "Skipping evaluator revision: not found.", - evaluator_step_key=evaluator_step_key, - evaluator_revision_ref=evaluator_revision_ref.model_dump(mode="json"), - ) - continue - - if not evaluator_revision.id or not evaluator_revision.slug: - log.warn( - "Skipping evaluator revision: found but missing identity.", - evaluator_step_key=evaluator_step_key, - evaluator_revision_ref=evaluator_revision_ref.model_dump(mode="json"), - has_id=bool(evaluator_revision.id), - has_slug=bool(evaluator_revision.slug), - ) - continue - - if not evaluator_revision.data: - log.warn( - "Skipping evaluator revision: found but missing data.", - evaluator_step_key=evaluator_step_key, - evaluator_revision_ref=evaluator_revision_ref.model_dump(mode="json"), - ) - continue - - resolved[evaluator_step_key] = evaluator_revision - - return resolved - - -async def evaluate_live_query( - project_id: UUID, - user_id: UUID, - # - run_id: UUID, - # - newest: Optional[datetime] = None, - oldest: Optional[datetime] = None, - # - use_windowing: bool = False, -): - # count in minutes - timestamp = oldest or datetime.now(timezone.utc) - interval: Optional[int] = None - if newest and oldest: - interval = int((newest - oldest).total_seconds() / 60) - - try: - # ---------------------------------------------------------------------- - log.info( - "[SCOPE] ", - run_id=run_id, - project_id=project_id, - user_id=user_id, - ) - - log.info( - "[RANGE] ", - run_id=run_id, - timestamp=timestamp, - interval=interval, - newest=newest, - oldest=oldest, - use_windowing=use_windowing, - ) - # ---------------------------------------------------------------------- - - # fetch evaluation run ------------------------------------------------- - run = await evaluations_service.fetch_run( - project_id=project_id, - run_id=run_id, - ) - - if not run: - raise ValueError(f"Evaluation run with id {run_id} not found!") - - if not run.data: - raise ValueError(f"Evaluation run with id {run_id} has no data!") - - if not run.data.steps: - raise ValueError(f"Evaluation run with id {run_id} has no steps!") - - steps = run.data.steps - repeats = run.data.repeats or 1 - repeat_indices = build_repeat_indices(repeats) - is_cached = bool(getattr(run.flags, "is_cached", False)) - - input_steps = { - step.key: step - for step in steps - if step.type == "input" # -------- - } - invocation_steps = { - step.key: step for step in steps if step.type == "invocation" - } - annotation_steps = { - step.key: step for step in steps if step.type == "annotation" - } - - input_steps_keys = list(input_steps.keys()) - invocation_steps_keys = list(invocation_steps.keys()) # noqa: F841 - annotation_steps_keys = list(annotation_steps.keys()) - - nof_annotations = len(annotation_steps_keys) - # ---------------------------------------------------------------------- - - # initialize query variables ------------------------------------------- - query_revision_refs: Dict[str, Reference] = dict() - # - query_revisions: Dict[str, QueryRevision] = dict() - query_references: Dict[str, Dict[str, Reference]] = dict() - # - query_traces: Dict[str, Traces] = dict() - # ---------------------------------------------------------------------- - - # initialize evaluator variables --------------------------------------- - evaluator_revision_refs: Dict[str, Reference] = dict() - # - evaluator_revisions: Dict[str, EvaluatorRevision] = dict() - evaluator_references: Dict[str, Dict[str, Reference]] = dict() - # ---------------------------------------------------------------------- - - # get query steps references ------------------------------------------- - for input_step_key in input_steps_keys: - query_refs = input_steps[input_step_key].references - query_revision_ref = query_refs.get("query_revision") - - if query_revision_ref: - query_revision_refs[input_step_key] = query_revision_ref - - # ---------------------------------------------------------------------- - - # get evaluator steps references --------------------------------------- - for annotation_step_key in annotation_steps_keys: - evaluator_refs = annotation_steps[annotation_step_key].references - evaluator_revision_ref = evaluator_refs.get("evaluator_revision") - - if evaluator_revision_ref: - evaluator_revision_refs[annotation_step_key] = evaluator_revision_ref - # ---------------------------------------------------------------------- - - # fetch query revisions ------------------------------------------------ - query_revisions = await _resolve_query_revisions( - queries_service=queries_service, - project_id=project_id, - query_revision_refs=query_revision_refs, - ) - for query_step_key in query_revisions: - query_references[query_step_key] = input_steps[query_step_key].references - # ---------------------------------------------------------------------- - - # fetch evaluator revisions -------------------------------------------- - evaluator_revisions = await _resolve_evaluator_revisions( - evaluators_service=evaluators_service, - project_id=project_id, - evaluator_revision_refs=evaluator_revision_refs, - ) - for evaluator_step_key in evaluator_revisions: - evaluator_references[evaluator_step_key] = annotation_steps[ - evaluator_step_key - ].references - # ---------------------------------------------------------------------- - - # run query revisions -------------------------------------------------- - for query_step_key, query_revision in query_revisions.items(): - formatting = Formatting( - focus=Focus.TRACE, - format=Format.AGENTA, - ) - filtering = Filtering( - operator=LogicalOperator.AND, - conditions=list(), - ) - windowing = Windowing( - oldest=oldest, - newest=newest, - next=None, - limit=None, - order="ascending", - interval=None, - rate=None, - ) - - if query_revision.data: - if query_revision.data.filtering: - filtering = query_revision.data.filtering - - if query_revision.data.windowing: - query_windowing = query_revision.data.windowing - - if use_windowing: - windowing = Windowing( - oldest=query_windowing.oldest, - newest=query_windowing.newest, - limit=query_windowing.limit, - order=query_windowing.order, - rate=query_windowing.rate, - # next= - # interval= - ) - else: - windowing.rate = query_windowing.rate - - query = TracingQuery( - formatting=formatting, - filtering=filtering, - windowing=windowing, - ) - - query_traces_result = await tracing_service.query_traces( - project_id=project_id, - query=TracingQuery( - formatting=query.formatting, - filtering=query.filtering, - windowing=query.windowing, - ), - ) - - nof_traces = len(query_traces_result) - - log.info( - "[TRACES] ", - run_id=run_id, - count=nof_traces, - ) - - query_traces[query_step_key] = query_traces_result or [] - # ---------------------------------------------------------------------- - - total_traces = sum(len(traces) for traces in query_traces.values()) - if total_traces == 0: - return - - # run online evaluation ------------------------------------------------ - any_results_created = False - for query_step_key in query_traces.keys(): - query_step_traces = [ - trace - for trace in query_traces[query_step_key] - if trace and trace.trace_id - ] - if not query_step_traces: - continue - - # create scenarios ------------------------------------------------- - - nof_traces = len(query_step_traces) - - scenarios_create = [ - EvaluationScenarioCreate( - run_id=run_id, - timestamp=timestamp, - interval=interval, - # - status=EvaluationStatus.RUNNING, - ) - for _ in range(nof_traces) - ] - - scenarios = await evaluations_service.create_scenarios( - project_id=project_id, - user_id=user_id, - # - scenarios=scenarios_create, - ) - - if len(scenarios) != nof_traces: - log.error( - "[LIVE] Could not create evaluation scenarios", - run_id=run_id, - ) - continue - # ------------------------------------------------------------------ - - # create query steps ----------------------------------------------- - query_trace_ids = [ - trace.trace_id for trace in query_step_traces if trace.trace_id - ] - scenario_ids = [scenario.id for scenario in scenarios if scenario.id] - - results_create = [ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario_id, - step_key=query_step_key, - repeat_idx=repeat_idx, - timestamp=timestamp, - interval=interval, - # - status=EvaluationStatus.SUCCESS, - # - trace_id=query_trace_id, - ) - for scenario_id, query_trace_id in zip(scenario_ids, query_trace_ids) - for repeat_idx in repeat_indices - ] - - results = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - # - results=results_create, - ) - - if len(results) != nof_traces * len(repeat_indices): - raise ValueError( - f"Failed to create evaluation results for run {run_id}!" - ) - # ------------------------------------------------------------------ - - scenario_has_errors: Dict[int, int] = dict() - scenario_status: Dict[int, EvaluationStatus] = dict() - scenario_has_pending: Dict[int, bool] = dict() - - # iterate over query traces ---------------------------------------- - for idx, trace in enumerate(query_step_traces): - scenario_results_created = False - scenario_has_errors[idx] = 0 - scenario_status[idx] = EvaluationStatus.SUCCESS - scenario_has_pending[idx] = False - - scenario = scenarios[idx] - scenario_id = scenario_ids[idx] - query_trace_id = query_trace_ids[idx] - - if not isinstance(trace.spans, dict): - log.warn( - f"Trace with id {query_trace_id} has no root spans", - run_id=run_id, - ) - scenario_has_errors[idx] += 1 - scenario_status[idx] = EvaluationStatus.ERRORS - continue - - root_span = list(trace.spans.values())[0] - - if isinstance(root_span, list): - log.warn( - f"More than one root span for trace with id {query_trace_id}", - run_id=run_id, - ) - scenario_has_errors[idx] += 1 - scenario_status[idx] = EvaluationStatus.ERRORS - continue - - query_span_id = root_span.span_id - - log.info( - "[TRACE] ", - run_id=run_id, - trace_id=query_trace_id, - ) - - # run evaluator revisions -------------------------------------- - for jdx in range(nof_annotations): - annotation_step_key = annotation_steps_keys[jdx] - annotation_step = annotation_steps[annotation_step_key] - - if annotation_step.origin in {"human", "custom"}: - scenario_has_pending[idx] = True - continue - - # skip annotation steps whose evaluator revision was not - # resolved (missing identity or data) — both maps are only - # populated for successfully resolved revisions. - if annotation_step_key not in evaluator_revisions: - log.error( - "Evaluator revision not resolved; skipping annotation step.", - run_id=run_id, - annotation_step_key=annotation_step_key, - ) - scenario_has_errors[idx] += 1 - scenario_status[idx] = EvaluationStatus.ERRORS - continue - - step_status = EvaluationStatus.SUCCESS - - evaluator_revision = evaluator_revisions[annotation_step_key] - query_revision = query_revisions[query_step_key] - references: Dict[str, Any] = { - **revision_references( - revision=query_revision, - entity_type="query", - ), - **revision_references( - revision=evaluator_revision, - entity_type="evaluator", - ), - } - links: Dict[str, Any] = { - query_step_key: dict( - trace_id=query_trace_id, - span_id=query_span_id, - ) - } - - # invoke annotation workflow ------------------------------- - _revision = evaluator_revision.model_dump( - mode="json", - exclude_none=True, - ) - interface = ( - dict( - uri=evaluator_revision.data.uri, - url=evaluator_revision.data.url, - headers=evaluator_revision.data.headers, - schemas=evaluator_revision.data.schemas, - ) - if evaluator_revision.data - else dict() - ) - configuration = ( - dict( - script=evaluator_revision.data.script, - parameters=evaluator_revision.data.parameters, - ) - if evaluator_revision.data - else dict() - ) - parameters = configuration.get("parameters") - - _testcase = None - inputs = None - - _trace: Optional[dict] = ( - trace.model_dump( - mode="json", - exclude_none=True, - ) - if trace - else None - ) - - _root_span = root_span.model_dump(mode="json", exclude_none=True) - testcase_data = None - - root_span_attributes: dict = _root_span.get("attributes") or {} - root_span_attributes_ag: dict = root_span_attributes.get("ag") or {} - root_span_attributes_ag_data: dict = ( - root_span_attributes_ag.get("data") or {} - ) - root_span_attributes_ag_data_outputs = ( - root_span_attributes_ag_data.get("outputs") - ) - root_span_attributes_ag_data_inputs = ( - root_span_attributes_ag_data.get("inputs") - ) - - outputs = root_span_attributes_ag_data_outputs - inputs = testcase_data or root_span_attributes_ag_data_inputs - - workflow_service_request_data = WorkflowServiceRequestData( - revision=_revision, - parameters=parameters, - # - testcase=_testcase, - inputs=inputs, - # - trace=_trace, - outputs=outputs, - ) - - flags = ( - evaluator_revision.flags.model_dump( - mode="json", - exclude_none=True, - exclude_unset=True, - ) - if evaluator_revision.flags - else None - ) - - workflow_service_request = WorkflowServiceRequest( - version="2025.07.14", - # - flags=flags, - # - interface=interface, - configuration=configuration, - # - data=workflow_service_request_data, - # - references=references, - links=links, - ) - hash_id = make_hash(references=references, links=links) - cached_traces = [] - if is_cached and hash_id: - cached_traces = await fetch_traces_by_hash( - tracing_service, - project_id, - hash_id=hash_id, - limit=len(repeat_indices), - ) - - reusable_traces = select_traces_for_reuse( - traces=cached_traces, - required_count=len(repeat_indices), - ) - results_create = [ - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario_id, - step_key=annotation_step_key, - repeat_idx=repeat_idx, - # - timestamp=timestamp, - interval=interval, - # - status=EvaluationStatus.SUCCESS, - # - trace_id=str(reusable_trace.trace_id), - ) - for repeat_idx, reusable_trace in zip( - repeat_indices, - reusable_traces, - ) - if reusable_trace and reusable_trace.trace_id - ] - - for repeat_idx in repeat_indices[len(reusable_traces) :]: - log.info( - "Invoking evaluator... ", - scenario_id=scenario.id, - repeat_idx=repeat_idx, - trace_id=query_trace_id, - uri=interface.get("uri"), - ) - workflows_service_response = ( - await workflows_service.invoke_workflow( - project_id=project_id, - user_id=user_id, - # - request=workflow_service_request, - # - annotate=True, - ) - ) - log.info( - "Invoked evaluator ", - scenario_id=scenario.id, - repeat_idx=repeat_idx, - trace_id=workflows_service_response.trace_id, - ) - - trace_id = workflows_service_response.trace_id - error = None - step_status = EvaluationStatus.SUCCESS - has_error = workflows_service_response.status.code != 200 - - if has_error: - log.warn( - "There is an error in evaluator %s for query %s.", - annotation_step_key, - query_trace_id, - ) - step_status = EvaluationStatus.FAILURE - scenario_has_errors[idx] += 1 - scenario_status[idx] = EvaluationStatus.ERRORS - error = workflows_service_response.status.model_dump( - mode="json", - exclude_none=True, - ) - else: - annotation = workflows_service_response - trace_id = annotation.trace_id - - if not annotation.trace_id: - log.warn("annotation trace_id is missing.") - scenario_has_errors[idx] += 1 - scenario_status[idx] = EvaluationStatus.ERRORS - continue - - fetched_trace = await fetch_trace( - tracing_service=tracing_service, - project_id=project_id, - trace_id=annotation.trace_id, - ) - - if fetched_trace: - log.info( - "Trace found ", - scenario_id=scenario.id, - step_key=annotation_step_key, - trace_id=annotation.trace_id, - ) - else: - log.warn( - "Trace missing", - scenario_id=scenario.id, - step_key=annotation_step_key, - trace_id=annotation.trace_id, - ) - scenario_has_errors[idx] += 1 - scenario_status[idx] = EvaluationStatus.ERRORS - continue - - results_create.append( - EvaluationResultCreate( - run_id=run_id, - scenario_id=scenario_id, - step_key=annotation_step_key, - repeat_idx=repeat_idx, - # - timestamp=timestamp, - interval=interval, - # - status=step_status, - # - trace_id=trace_id, - error=error, - ) - ) - - results = await evaluations_service.create_results( - project_id=project_id, - user_id=user_id, - # - results=results_create, - ) - - if len(results) != len(repeat_indices): - raise ValueError( - f"Failed to create evaluation results for scenario with id {scenario.id}!" - ) - scenario_results_created = True - any_results_created = True - # -------------------------------------------------------------- - - scenario_edit = EvaluationScenarioEdit( - id=scenario.id, - tags=scenario.tags, - meta=scenario.meta, - status=( - EvaluationStatus.PENDING - if ( - scenario_status[idx] == EvaluationStatus.SUCCESS - and scenario_has_pending[idx] - ) - else scenario_status[idx] - ), - ) - - scenario = await evaluations_service.edit_scenario( - project_id=project_id, - user_id=user_id, - # - scenario=scenario_edit, - ) - - if not scenario or not scenario.id: - log.error( - f"Failed to update evaluation scenario with id {scenario_id}!", - run_id=run_id, - ) - - if scenario_results_created: - await evaluations_service.refresh_metrics( - project_id=project_id, - user_id=user_id, - # - metrics=EvaluationMetricsRefresh( - run_id=run_id, - scenario_id=scenario_id, - ), - ) - # ------------------------------------------------------------------ - - if any_results_created: - await evaluations_service.refresh_metrics( - project_id=project_id, - user_id=user_id, - # - metrics=EvaluationMetricsRefresh( - run_id=run_id, - timestamp=timestamp, - interval=interval, - ), - ) - except Exception as e: # pylint: disable=broad-exception-caught - log.error(e, exc_info=True) - - log.info( - "[DONE] ", - run_id=run_id, - ) - - return diff --git a/api/oss/src/core/evaluations/tasks/processor.py b/api/oss/src/core/evaluations/tasks/processor.py new file mode 100644 index 0000000000..a2449006e3 --- /dev/null +++ b/api/oss/src/core/evaluations/tasks/processor.py @@ -0,0 +1,845 @@ +import asyncio +from typing import Dict, List, Optional, Any, Tuple +from types import SimpleNamespace + +from uuid import UUID + +from agenta.sdk.evaluations.runtime.models import ( + EvaluationStep as SdkEvaluationStep, + ResolvedSourceItem as SdkResolvedSourceItem, +) +from agenta.sdk.evaluations.runtime.planner import EvaluationPlanner +from agenta.sdk.evaluations.runtime.processor import ( + process_sources as sdk_process_evaluation_source_slice, + run_status as compute_run_status, +) + +from oss.src.utils.logging import get_module_logger + +from oss.src.core.testcases.service import TestcasesService +from oss.src.core.applications.service import ApplicationsService +from oss.src.core.workflows.service import WorkflowsService +from oss.src.core.evaluations.service import EvaluationsService + +from oss.src.core.tracing.service import TracingService + + +from oss.src.core.evaluations.types import ( + EvaluationStatus, + EvaluationRun, + EvaluationRunEdit, + EvaluationResult, + EvaluationResultQuery, + EvaluationClosedConflict, +) + +from oss.src.core.evaluations.utils import ( + effective_is_split, +) +from oss.src.core.evaluations.runtime.adapters import ( + APICachedRunner, + APIMetricsRefresher, + APIResultSetter, + APIScenarioEditor, + APITraceFetcher, + APIWorkflowRunner, +) +from oss.src.core.evaluations.runtime.models import ( + ProcessSummary, + PlannedCell, + ResolvedSourceItem, + ScenarioBinding, +) +from oss.src.core.evaluations.runtime.sources import ( + resolve_direct_source_items, +) + + +log = get_module_logger(__name__) + + +def _cell_key(cell: Any) -> Tuple[str, int]: + return cell.step_key, int(cell.repeat_idx or 0) + + +def _to_sdk_source_item( + source_item: ResolvedSourceItem, + *, + step_key_fallback: Optional[str] = None, +) -> SdkResolvedSourceItem: + """Shape an api-side ResolvedSourceItem into the SDK engine's input type. + + The ingest and re-execute paths build this identically; the only variation + is the step key, which re-execute carries on the item and ingest derives + from the run's input step. Keeping one helper removes that duplication. + """ + return SdkResolvedSourceItem( + kind=source_item.kind, + step_key=source_item.step_key or step_key_fallback or "", + references=source_item.references or {}, + trace_id=source_item.trace_id, + span_id=source_item.span_id, + testcase_id=source_item.testcase_id, + testcase=source_item.testcase, + trace=source_item.trace, + inputs=source_item.inputs or getattr(source_item.testcase, "data", None), + outputs=source_item.outputs, + ) + + +def _cell_is_addressed( + *, + cell: PlannedCell, + requested_steps: set[str], + requested_repeats: set[int], +) -> bool: + if requested_steps and cell.step_key not in requested_steps: + return False + if requested_repeats and cell.repeat_idx not in requested_repeats: + return False + return True + + +def _seed_context_from_source( + *, + source_item: ResolvedSourceItem, + repeats: Optional[int], +) -> Dict[int, Dict[str, Any]]: + """Build per-repeat upstream context from an ALREADY-HYDRATED source item. + + The API-side counterpart of the SDK's `_initial_context_by_repeat`, for the + seeded ingest path: the trace was fetched once by the resolver and lives on + the `ResolvedSourceItem`, so the runner's upstream context is read straight + from it instead of re-fetching by id. Testcase sources carry no trace, so + there is no upstream context to seed (returns {}). + """ + trace = source_item.trace + trace_id = source_item.trace_id + if not trace and not trace_id: + return {} + if not trace_id: + return {} + + context = { + "trace": trace, + "trace_id": str(trace_id), + "span_id": source_item.span_id, + "outputs": source_item.outputs, + } + count = max(repeats or 1, 1) + return {repeat_idx: context for repeat_idx in range(count)} + + +async def _seed_context_by_repeat( + *, + project_id: UUID, + scenario_cells: List[EvaluationResult], + invocation_step_keys: set[str], + tracing_service: Optional[TracingService], +) -> Dict[int, Dict[str, Any]]: + trace_ids_by_repeat = { + int(cell.repeat_idx or 0): cell.trace_id + for cell in scenario_cells + if cell.step_key in invocation_step_keys and cell.trace_id + } + if not trace_ids_by_repeat: + return {} + + hydrated = await resolve_direct_source_items( + project_id=project_id, + trace_ids=list(dict.fromkeys(trace_ids_by_repeat.values())), + tracing_service=tracing_service, + ) + trace_items = {item.trace_id: item for item in hydrated if item.trace_id} + context_by_repeat: Dict[int, Dict[str, Any]] = {} + + for repeat_idx, trace_id in trace_ids_by_repeat.items(): + item = trace_items.get(trace_id) + context_by_repeat[repeat_idx] = { + "trace": item.trace if item is not None else None, + "trace_id": trace_id, + "span_id": item.span_id if item is not None else None, + "outputs": item.outputs if item is not None else None, + } + + return context_by_repeat + + +async def _resolve_runners_and_revisions( + *, + project_id: UUID, + user_id: UUID, + run: EvaluationRun, + invocation_steps: List[Any], + annotation_steps: List[Any], + tracing_service: Optional[TracingService], + workflows_service: Optional[WorkflowsService], + applications_service: Optional[ApplicationsService], +) -> tuple[Dict[str, Any], Dict[str, Any]]: + """Wire one cache-aware runner + its revision per executable step. + + The returned `runners` are `APICachedRunner`s, so hashed-trace reuse + (cache lookup by step references/links before invoking) is handled here for + both ingest and slice re-execution. `auto` annotations are wired; `human` + and `custom` annotations are not (the backend never executes them). + """ + run_id = run.id + runners: Dict[str, Any] = {} + revisions: Dict[str, Any] = {} + + if invocation_steps: + if applications_service is None: + raise ValueError("applications_service is required for invocation steps") + if workflows_service is None: + raise ValueError("workflows_service is required for invocation steps") + invocation_step = invocation_steps[0] + application_revision_ref = invocation_step.references.get( + "application_revision" + ) + if not application_revision_ref or not isinstance( + application_revision_ref.id, UUID + ): + raise ValueError( + f"Evaluation run with id {run_id} missing invocation.application_revision reference." + ) + application_revision = await applications_service.fetch_application_revision( + project_id=project_id, + application_revision_ref=application_revision_ref, + ) + if application_revision is None: + raise ValueError( + f"App revision with id {application_revision_ref.id} not found!" + ) + runners[invocation_step.key] = APICachedRunner( + runner=APIWorkflowRunner( + project_id=project_id, + user_id=user_id, + workflows_service=workflows_service, + ), + tracing_service=tracing_service, + project_id=project_id, + enabled=bool(run.flags and run.flags.is_cached), + ) + revisions[invocation_step.key] = application_revision + + auto_annotation_steps = [ + step for step in annotation_steps if step.origin not in {"human", "custom"} + ] + if auto_annotation_steps and workflows_service is None: + raise ValueError("workflows_service is required for auto annotation steps") + for annotation_step in auto_annotation_steps: + evaluator_revision_ref = (annotation_step.references or {}).get( + "evaluator_revision" + ) + evaluator_revision = ( + await workflows_service.fetch_workflow_revision( # type: ignore[union-attr] + project_id=project_id, + workflow_revision_ref=evaluator_revision_ref, + ) + if evaluator_revision_ref + else None + ) + if evaluator_revision is None: + continue + runners[annotation_step.key] = APICachedRunner( + runner=APIWorkflowRunner( + project_id=project_id, + user_id=user_id, + workflows_service=workflows_service, + ), + tracing_service=tracing_service, + project_id=project_id, + enabled=bool(run.flags and run.flags.is_cached), + ) + revisions[annotation_step.key] = evaluator_revision + + return runners, revisions + + +def _source_item_from_input_cells( + *, + input_steps: List[Any], + cells_by_step: Dict[str, List[EvaluationResult]], +) -> Optional[ResolvedSourceItem]: + """Rebuild the scenario's source binding from its stored input result cells. + + A slice addresses EXISTING scenarios, so the source is not re-resolved from + a query/testset — it is recovered from the input step's already-written + cell, which carries the bound `trace_id` (trace/query source) or + `testcase_id` (testcase/testset source). Trace/testcase context is + re-hydrated by the slice processor's trace loader / testcase fetch so cache + reuse and evaluator inputs match the original run. + """ + for step in input_steps: + cells = cells_by_step.get(step.key) or [] + if not cells: + continue + cell = cells[0] + if cell.trace_id: + return ResolvedSourceItem( + kind="trace", + step_key=step.key, + trace_id=cell.trace_id, + ) + if cell.testcase_id: + return ResolvedSourceItem( + kind="testcase", + step_key=step.key, + testcase_id=cell.testcase_id, + ) + return None + + +async def _resolve_source_from_input_cells( + *, + project_id: UUID, + input_steps: List[Any], + cells_by_step: Dict[str, List[EvaluationResult]], + tracing_service: Optional[Any], + testcases_service: Optional[Any], +) -> Optional[SdkResolvedSourceItem]: + """The input-cache ladder: recover a scenario's source from its stored cell. + + A scenario that already exists carries its source identity in its input + result cell (a `trace_id` or `testcase_id`). This reads that id back and + re-hydrates the trace/testcase payload — the same data the original run + resolved — so cache reuse and evaluator inputs match. Returns None when the + scenario has no usable input cell (nothing to reconstruct from); the caller + treats that as "must be populated first". + """ + source_item = _source_item_from_input_cells( + input_steps=input_steps, + cells_by_step=cells_by_step, + ) + if source_item is None: + return None + + hydrated = await resolve_direct_source_items( + project_id=project_id, + testcase_ids=[source_item.testcase_id] if source_item.testcase_id else None, + trace_ids=[source_item.trace_id] if source_item.trace_id else None, + testcases_service=testcases_service, + tracing_service=tracing_service, + ) + resolved = hydrated[0] if hydrated else source_item + resolved.step_key = source_item.step_key + return _to_sdk_source_item(resolved) + + +async def _run_sdk_source_slice( + *, + project_id: UUID, + user_id: UUID, + run: Any, + evaluations_service: Any, + sdk_source_items: List[SdkResolvedSourceItem], + sdk_steps: List[SdkEvaluationStep], + invocation_steps: List[Any], + annotation_steps: List[Any], + runners: Any, + revisions: Any, + # + # --- the seams (the only things that differ between callers) --- + create_scenario: Any, + refresh_metrics: Any, + log_pending: bool, + timestamp: Optional[Any] = None, + interval: Optional[int] = None, + refresh_metrics_without_auto_results: bool = True, + plan_cell_filter: Optional[Any] = None, + initial_context_by_repeat: Optional[Any] = None, + tracing_service: Optional[Any] = None, +) -> List[Any]: + """The single execution call: drive the SDK engine over a source slice. + + Both the ingest path (NEW scenarios from source ids) and the re-execute path + (EXISTING scenarios by coordinate) funnel through here. They differ only in + the injected seams: the scenario factory (create vs reuse), the metrics + refresher (inline vs deferred), `log_pending`, the cell filter, and the + per-repeat context seed. Everything else is intrinsic to `run` + services + and identical for both, so it lives here once. + """ + return await sdk_process_evaluation_source_slice( + run_id=run.id, + source_items=sdk_source_items, + steps=sdk_steps, + repeats=run.data.repeats, + create_scenario=create_scenario, + set_results=APIResultSetter( + project_id=project_id, + user_id=user_id, + timestamp=timestamp, + interval=interval, + evaluations_service=evaluations_service, + ), + edit_scenario=APIScenarioEditor( + project_id=project_id, + user_id=user_id, + evaluations_service=evaluations_service, + ), + refresh_metrics=refresh_metrics, + runners=runners, + revisions=revisions, + fetch_trace=( + APITraceFetcher( + project_id=project_id, + tracing_service=tracing_service, + ) + if tracing_service is not None + else None + ), + is_split=effective_is_split( + is_split=bool(run.flags and run.flags.is_split), + has_application_steps=bool(invocation_steps), + has_evaluator_steps=bool(annotation_steps), + ), + log_pending=log_pending, + refresh_metrics_without_auto_results=refresh_metrics_without_auto_results, + batch_size=run.data.concurrency.batch_size if run.data.concurrency else None, + max_retries=run.data.concurrency.max_retries if run.data.concurrency else None, + retry_delay=run.data.concurrency.retry_delay if run.data.concurrency else None, + initial_context_by_repeat=initial_context_by_repeat, + plan_cell_filter=plan_cell_filter, + ) + + +async def _finalize_run_after_slice( + *, + project_id: UUID, + user_id: UUID, + run: Any, + processed: List[Any], + run_status_override: Optional[Any] = None, + evaluations_service: Any, + finalize_run_status: bool = True, +) -> None: + """Shared "done" for `process`: finalize the RUN from the touched set. + + Per-scenario status is no longer written here — the engine + (`process_sources`) owns that via the injected `edit_scenario` adapter, the + same path for ingest, re-execute, and the SDK. This function is the + remaining RUN-level post-process every slice owns: + + 1. computes the run status from the touched set (or uses an override, e.g. + FAILURE when the slice itself raised); + 2. if `finalize_run_status`, floors the run status across concurrent slices + off the CURRENT run (not the caller's stale snapshot) and flips + `is_active` off on a terminal status. + + `finalize_run_status=False` is the live-query loop, which never finalizes — + it keeps ticking and leaves run status alone. + """ + # 1. run status from the touched set (or the override). The rollup itself is + # the shared engine verdict (same as the SDK); only the override + the + # cross-slice floor below are API-specific. + if run_status_override is not None: + run_status = run_status_override + else: + run_status = compute_run_status(processed) + + if not finalize_run_status: + return + + # 2. floor + is_active, off the CURRENT run (concurrent-slice safe). + # Concurrent slices each hold a stale snapshot of `run`; finalize both floors + # the status and flips `is_active`, so it must read the CURRENT run or a + # late-finishing slice clobbers another's status/flags (e.g. resurrecting + # is_active=True). + current_run = await evaluations_service.fetch_run( + project_id=project_id, + run_id=run.id, + ) + + if ( + run.flags + and ( + run.flags.has_traces + or run.flags.has_testcases + or run.flags.has_queries + or run.flags.has_testsets + ) + and run_status != EvaluationStatus.FAILURE + ): + # Only terminal "bad" statuses floor across slices: if an earlier slice + # already marked the run FAILURE/ERRORS, a later SUCCESS-only slice must + # not downgrade it. RUNNING/PENDING rank BELOW SUCCESS so a freshly + # computed terminal status (incl. SUCCESS) always replaces a stale + # RUNNING — otherwise a run pins at RUNNING forever. + severity = { + EvaluationStatus.FAILURE: 4, + EvaluationStatus.ERRORS: 3, + EvaluationStatus.SUCCESS: 2, + EvaluationStatus.RUNNING: 1, + EvaluationStatus.PENDING: 0, + } + if current_run and current_run.status: + stored_severity = severity.get(current_run.status, 0) + if stored_severity > severity.get(run_status, 0): + run_status = current_run.status + + # When the run reaches a terminal status, it is no longer active. A + # non-terminal status (RUNNING/PENDING) leaves it active so further slices + # can continue. Base the flags on the freshly-fetched run so a concurrent + # slice's flag updates are not lost, and only flip the one field this owns. + final_flags = (current_run.flags if current_run else None) or run.flags + if final_flags is not None and run_status in ( + EvaluationStatus.SUCCESS, + EvaluationStatus.ERRORS, + EvaluationStatus.FAILURE, + ): + final_flags = final_flags.model_copy(update={"is_active": False}) + + try: + await evaluations_service.edit_run( + project_id=project_id, + user_id=user_id, + run=EvaluationRunEdit( + id=run.id, + name=run.name, + description=run.description, + tags=run.tags, + meta=run.meta, + status=run_status, + flags=final_flags, + data=run.data, + ), + ) + except EvaluationClosedConflict: + # The run was closed (locked) mid-flight. Closing is a lock, not a + # failure — leave its status as-is rather than erroring finalization. + log.info( + "[WORKER] finalize skipped: run closed mid-flight", + run_id=str(run.id), + run_status=str(run_status), + ) + + +class APISliceProcessor: + """API `SliceProcessor`: re-execute the runnable cells in a slice. + + Unlike `process_evaluation_source_slice` (an INGEST loop that creates a new + scenario per source item), this re-executes EXISTING scenarios addressed by + a `TensorSlice` — the retry / fill-missing / re-run-one-evaluator axis. For + each scenario in the slice it: rebuilds the source binding from the stored + input cells, re-hydrates trace/testcase context, plans only the requested + `step_keys`/`repeat_idxs`, runs the cache-aware runners (so hashed traces are + reused), populates the result cells, and refreshes metrics. Modified steps + re-run because the plan is rebuilt from the run's CURRENT graph and the + fresh evaluator revisions resolved here. + """ + + def __init__( + self, + *, + evaluations_service: EvaluationsService, + tracing_service: Optional[TracingService] = None, + testcases_service: Optional[TestcasesService] = None, + workflows_service: Optional[WorkflowsService] = None, + applications_service: Optional[ApplicationsService] = None, + ): + self.evaluations_service = evaluations_service + self.tracing_service = tracing_service + self.testcases_service = testcases_service + self.workflows_service = workflows_service + self.applications_service = applications_service + + async def process( + self, + *, + project_id: UUID, + user_id: UUID, + tensor_slice, + seed_bindings: Optional[Dict[UUID, ScenarioBinding]] = None, + refresh_metrics_without_auto_results: bool = True, + finalize_run_status: bool = True, + ) -> ProcessSummary: + """Re-execute scenarios in the slice. + + When `seed_bindings` is supplied (the mint→populate→re-execute ingest + flows: run/slice), each addressed scenario's source is taken from its + binding's ALREADY-HYDRATED `ResolvedSourceItem` — no re-read of input + cells, no per-id trace/testcase re-fetch. Bindings also carry the + per-scenario `timestamp`/`interval` (live-query temporal coordinates). + + When `seed_bindings` is None (the tensor-retry flow), the source is + recovered from the scenario's stored input cells as before. + """ + seed_bindings = seed_bindings or {} + run_id = tensor_slice.run_id + run = await self.evaluations_service.fetch_run( + project_id=project_id, + run_id=run_id, + ) + if not run or not run.data or not run.data.steps: + return ProcessSummary() + + steps = run.data.steps + input_steps = [step for step in steps if step.type == "input"] + invocation_steps = [step for step in steps if step.type == "invocation"] + annotation_steps = [step for step in steps if step.type == "annotation"] + + # Probe the existing cells in the slice scope, grouped by scenario. + existing = await self.evaluations_service.query_results( + project_id=project_id, + result=EvaluationResultQuery( + run_id=run_id, + scenario_ids=tensor_slice.scenario_ids, + step_keys=tensor_slice.step_keys, + repeat_idxs=tensor_slice.repeat_idxs, + ), + ) + # Inputs are always needed to rebuild the source binding even when the + # slice's step_keys exclude them, so probe inputs separately per scenario. + scenario_ids = ( + sorted(set(tensor_slice.scenario_ids or []), key=str) + if tensor_slice.scenario_ids is not None + else sorted( + {cell.scenario_id for cell in existing if cell.scenario_id}, + key=str, + ) + ) + if not scenario_ids: + return ProcessSummary() + + runners, revisions = await _resolve_runners_and_revisions( + project_id=project_id, + user_id=user_id, + run=run, + invocation_steps=invocation_steps, + annotation_steps=annotation_steps, + tracing_service=self.tracing_service, + workflows_service=self.workflows_service, + applications_service=self.applications_service, + ) + + requested_steps = set(tensor_slice.step_keys or []) + requested_repeats = set(tensor_slice.repeat_idxs or []) + force_rerun = tensor_slice.process_mode == "force" + + sdk_steps_all = [ + SdkEvaluationStep( + key=step.key, + type=step.type, + origin=step.origin, + references=step.references or {}, + inputs=[step_input.key for step_input in (step.inputs or [])], + ) + for step in steps + ] + + summary = ProcessSummary() + + effective_is_split_value = effective_is_split( + is_split=bool(run.flags and run.flags.is_split), + has_application_steps=bool(invocation_steps), + has_evaluator_steps=bool(annotation_steps), + ) + + # --- Recovery pass (per scenario): resolve each scenario's source, + # compute its addressed/target cells and reuse, and recover its upstream + # context. NO execution here — we collect, then run ONE batched slice so + # the engine's gather+semaphore give cross-scenario concurrency (matching + # the SDK and the design's process_slice(all scenarios)). + scenarios_in_order: List[Any] = [] + batch_source_items: List[SdkResolvedSourceItem] = [] + target_keys_by_scenario: Dict[UUID, set] = {} + context_by_scenario: Dict[UUID, Dict[int, Any]] = {} + # timestamp/interval are constant across a single process call (seeded: + # one value from _mint_and_bind; recovered: None), so capture once. + slice_timestamp: Optional[Any] = None + slice_interval: Optional[int] = None + + for scenario_id in scenario_ids: + binding = seed_bindings.get(scenario_id) + + if binding is not None: + # Seeded (ingest) path: the source is already hydrated in memory, + # so skip the input-cell read and per-id re-fetch. Fresh scenarios + # have no prior cells, hence no reuse to account for. + existing_cell_keys: set = set() + sdk_source_item = _to_sdk_source_item(binding.source) + scenario_context = _seed_context_from_source( + source_item=binding.source, + repeats=run.data.repeats, + ) + slice_timestamp = binding.timestamp + slice_interval = binding.interval + else: + input_cells = await self.evaluations_service.query_results( + project_id=project_id, + result=EvaluationResultQuery( + run_id=run_id, + scenario_id=scenario_id, + ), + ) + cells_by_step: Dict[str, List[EvaluationResult]] = {} + for cell in input_cells: + cells_by_step.setdefault(cell.step_key, []).append(cell) + existing_cell_keys = {_cell_key(cell) for cell in input_cells} + + sdk_source_item = await _resolve_source_from_input_cells( + project_id=project_id, + input_steps=input_steps, + cells_by_step=cells_by_step, + tracing_service=self.tracing_service, + testcases_service=self.testcases_service, + ) + if sdk_source_item is None: + # No populated input for this scenario (no trace_id/testcase_id + # to reconstruct from): there is nothing to run, so the line is + # skipped — not a failure (which means execution errored). + summary.skipped += 1 + continue + scenario_context = await _seed_context_by_repeat( + project_id=project_id, + scenario_cells=input_cells, + invocation_step_keys={step.key for step in invocation_steps}, + tracing_service=self.tracing_service, + ) + preview_plan = EvaluationPlanner().plan( + run_id=run_id, + scenario_id=scenario_id, + source=sdk_source_item, + steps=sdk_steps_all, + repeats=run.data.repeats, + is_split=effective_is_split_value, + ) + addressed_cells = [ + cell + for cell in preview_plan.cells + if _cell_is_addressed( + cell=cell, + requested_steps=requested_steps, + requested_repeats=requested_repeats, + ) + ] + if not force_rerun: + summary.reused += sum( + 1 + for cell in addressed_cells + if _cell_key(cell) in existing_cell_keys + ) + target_keys = { + _cell_key(cell) + for cell in addressed_cells + if force_rerun or _cell_key(cell) not in existing_cell_keys + } + if not target_keys: + continue + + scenarios_in_order.append(SimpleNamespace(id=scenario_id)) + batch_source_items.append(sdk_source_item) + target_keys_by_scenario[scenario_id] = target_keys + context_by_scenario[scenario_id] = scenario_context + summary.created += len(target_keys) + + # --- Single batched execution over all recovered scenarios. The engine + # creates scenarios via the ordered cursor, filters cells per-scenario, + # and resolves each scenario's recovered context lazily via the callable. + all_processed: List[Any] = [] + if batch_source_items: + + async def _scenario_context( + scenario_id: UUID, + _ctx: Dict[UUID, Dict[int, Any]] = context_by_scenario, + ) -> Dict[int, Any]: + return _ctx.get(scenario_id, {}) + + def _plan_cell_filter( + cell: Any, + _keys: Dict[UUID, set] = target_keys_by_scenario, + ) -> bool: + return _cell_key(cell) in _keys.get(cell.scenario_id, set()) + + all_processed = await _run_sdk_source_slice( + project_id=project_id, + user_id=user_id, + run=run, + evaluations_service=self.evaluations_service, + sdk_source_items=batch_source_items, + sdk_steps=sdk_steps_all, + invocation_steps=invocation_steps, + annotation_steps=annotation_steps, + runners=runners, + revisions=revisions, + # reuse existing scenarios in order; do NOT mint new ones. + create_scenario=_OrderedScenarios(scenarios_in_order), + # process is a tensor-write op: it refreshes the touched scope's + # metrics incrementally per-scenario (and rolls up), the same as + # ingest — re-execute no longer opts out with a no-op. + refresh_metrics=APIMetricsRefresher( + project_id=project_id, + user_id=user_id, + evaluations_service=self.evaluations_service, + ), + log_pending=True, + refresh_metrics_without_auto_results=refresh_metrics_without_auto_results, + # Seeded scenarios carry the run's temporal coordinates (live + # query); recovered scenarios already have them on their cells. + # Constant across the slice, so passed once. + timestamp=slice_timestamp, + interval=slice_interval, + tracing_service=self.tracing_service, + initial_context_by_repeat=_scenario_context, + plan_cell_filter=_plan_cell_filter, + ) + + for item in all_processed: + if item.has_pending: + summary.pending += 1 + if item.has_errors: + summary.failed += 1 + + # Shared "done": write per-scenario status + finalize the run from the + # touched set — identical to ingest (a re-run IS a run-state change). + if all_processed: + await _finalize_run_after_slice( + project_id=project_id, + user_id=user_id, + run=run, + processed=all_processed, + evaluations_service=self.evaluations_service, + finalize_run_status=finalize_run_status, + ) + + log.info( + "[SLICE] re-execute complete", + run_id=str(run_id), + scenarios=len(scenario_ids), + created=summary.created, + pending=summary.pending, + failed=summary.failed, + skipped=summary.skipped, + requested_steps=sorted(requested_steps) or None, + requested_repeats=sorted(requested_repeats) or None, + ) + return summary + + +class _OrderedScenarios: + """`create_scenario` adapter handing back EXISTING scenarios in order. + + The engine calls `create_scenario(run_id)` once per source item; for a + batched re-execute slice we hand back the recovered scenarios in the order + their sources were collected, instead of minting new ones — the API analogue + of the SDK's `_PreMintedScenarios`. + + Order/concurrency: the engine runs scenarios concurrently (gather + + semaphore), so multiple coroutines call this. `create_scenario` is the FIRST + statement of the engine's `_process_one` and this body has no `await`, so + each task runs through the pop synchronously before any real suspension — + i.e. the pops happen in source-item order, pairing scenario i with source i. + The lock makes the index increment atomic so the ordering can never degrade + into a double-hand-out if scheduling shifts. + """ + + def __init__(self, scenarios: List[Any]): + self._scenarios = list(scenarios) + self._idx = 0 + self._lock = asyncio.Lock() + + async def __call__(self, run_id: UUID): + async with self._lock: + scenario = self._scenarios[self._idx] + self._idx += 1 + return scenario diff --git a/api/oss/src/core/evaluations/tasks/run.py b/api/oss/src/core/evaluations/tasks/run.py new file mode 100644 index 0000000000..c858754a3c --- /dev/null +++ b/api/oss/src/core/evaluations/tasks/run.py @@ -0,0 +1,683 @@ +from datetime import datetime, timezone +from typing import Any, List, Literal, Optional +from uuid import UUID + +from oss.src.core.applications.service import ApplicationsService +from oss.src.core.evaluations.runtime.models import ( + ResolvedSourceItem, + ScenarioBinding, + SliceProcessMode, + TensorSlice, +) +from oss.src.core.evaluations.runtime.tensor import TensorSliceOperations +from oss.src.core.evaluations.runtime.topology import classify_run_topology +from oss.src.core.evaluations.runtime.sources import ( + resolve_direct_source_items, + resolve_query_source_items, + resolve_testset_input_specs, +) +from oss.src.core.evaluations.runtime.adapters import ( + APIScenarioFactory, +) +from oss.src.core.evaluations.service import EvaluationsService +from oss.src.core.evaluations.types import ( + EvaluationRun, + EvaluationRunEdit, + EvaluationRunFlags, + EvaluationStatus, +) +from oss.src.core.evaluations.tasks.processor import APISliceProcessor +from oss.src.core.evaluators.service import SimpleEvaluatorsService +from oss.src.core.queries.service import QueriesService +from oss.src.core.testcases.service import TestcasesService +from oss.src.core.testsets.service import TestsetsService +from oss.src.core.tracing.service import TracingService +from oss.src.core.workflows.service import WorkflowsService +from oss.src.utils.logging import get_module_logger + +log = get_module_logger(__name__) + +EvaluationSliceSource = Literal["traces", "testcases"] + + +# ============================================================================= +# Shared skeleton — every flow is: validate run -> resolve+mint+bind (the +# per-flow seam) -> re-execute via APISliceProcessor with the hydrated source +# seeded in -> finalize. The only thing that differs per flow is HOW source +# items are produced; the mint/bind/execute/finalize machinery is shared. +# The input cell is written once, by the SDK on execute — not pre-written. +# ============================================================================= + + +async def _fetch_validated_run( + *, + project_id: UUID, + run_id: UUID, + evaluations_service: EvaluationsService, + require: bool = False, +) -> Optional[EvaluationRun]: + """Fetch a run and assert it has executable steps. + + `require=False` (default) returns None on a missing/empty run so the caller + can warn-and-skip; `require=True` raises so a pre-execution error finalizes + the run to FAILURE through the caller's handler. + """ + run = await evaluations_service.fetch_run( + project_id=project_id, + run_id=run_id, + ) + if not run or not run.data or not run.data.steps: + if require: + raise ValueError(f"Evaluation run with id {run_id} not found or empty!") + return None + return run + + +def _input_step_keys(run: EvaluationRun) -> List[str]: + return [step.key for step in run.data.steps if step.type == "input"] + + +async def _mint_and_bind( + *, + project_id: UUID, + user_id: UUID, + run: EvaluationRun, + source_items: List[ResolvedSourceItem], + default_step_key: str, + timestamp: Optional[datetime], + interval: Optional[int], + evaluations_service: EvaluationsService, +) -> List[ScenarioBinding]: + """Bulk-mint one scenario per source item and bind its hydrated source. + + Shared by every ingest flow. The source item is ALREADY hydrated (the + resolver fetched the trace/testcase once), so the returned bindings carry it + straight into the executor — no re-read, no per-id re-fetch downstream. + + The input cell is NOT written here. The SDK slice loop logs the input step + first (before any runnable cell), writing the same `trace_id`/`testcase_id` + and temporal coordinates via `APIResultSetter`. Pre-writing it here would + just be overwritten by that log — a redundant DB round-trip per scenario. + The durable input cell a later tensor retry recovers from is the SDK's. + """ + if not source_items: + return [] + + factory = APIScenarioFactory( + project_id=project_id, + user_id=user_id, + evaluations_service=evaluations_service, + ) + scenarios = await factory.bulk_create( + run.id, + count=len(source_items), + timestamp=timestamp, + interval=interval, + ) + + bindings: List[ScenarioBinding] = [] + for scenario, source_item in zip(scenarios, source_items): + step_key = source_item.step_key or default_step_key + bound_source = source_item.model_copy(update={"step_key": step_key}) + bindings.append( + ScenarioBinding( + scenario_id=scenario.id, + source=bound_source, + timestamp=timestamp, + interval=interval, + ) + ) + return bindings + + +async def _execute_bindings( + *, + project_id: UUID, + user_id: UUID, + run_id: UUID, + bindings: List[ScenarioBinding], + tracing_service: Optional[TracingService], + testcases_service: Optional[TestcasesService], + workflows_service: Optional[WorkflowsService], + applications_service: Optional[ApplicationsService], + evaluations_service: EvaluationsService, + refresh_metrics_without_auto_results: bool = True, + finalize_run_status: bool = True, +) -> None: + """Re-execute freshly-minted scenarios with their hydrated source seeded in. + + `process_mode="force"` because the scenarios are new: their runnable cells + must be filled even though no result exists yet. `APISliceProcessor` owns + per-scenario status writes and run finalization for terminal status. + + `refresh_metrics_without_auto_results=False` (the query flow) skips the + metric refresh for scenarios that produced no auto results — e.g. a live + run whose only annotation step is human — matching the legacy query loop. + + `finalize_run_status=False` (the LIVE query flow) leaves the run RUNNING and + active so the scheduler keeps polling; batch/testset runs finalize. + """ + slice_processor = APISliceProcessor( + evaluations_service=evaluations_service, + tracing_service=tracing_service, + testcases_service=testcases_service, + workflows_service=workflows_service, + applications_service=applications_service, + ) + await slice_processor.process( + project_id=project_id, + user_id=user_id, + tensor_slice=TensorSlice( + run_id=run_id, + scenario_ids=[binding.scenario_id for binding in bindings], + process_mode="force", + ), + seed_bindings={binding.scenario_id: binding for binding in bindings}, + refresh_metrics_without_auto_results=refresh_metrics_without_auto_results, + finalize_run_status=finalize_run_status, + ) + + +async def _finalize_run_terminal( + *, + project_id: UUID, + user_id: UUID, + run_id: UUID, + status: EvaluationStatus, + evaluations_service: EvaluationsService, +) -> None: + """Flip a run to a terminal status + inactive, tolerating a vanished run. + + Used for the two cases the slice processor's own finalize never sees: an + empty result set (nothing to execute) and a pre-execution error. Re-fetches + the run so it does not clobber concurrent flag updates. + """ + try: + current = await evaluations_service.fetch_run( + project_id=project_id, + run_id=run_id, + ) + if not current: + return + flags = current.flags.model_copy() if current.flags else EvaluationRunFlags() + flags.is_active = False + await evaluations_service.edit_run( + project_id=project_id, + user_id=user_id, + run=EvaluationRunEdit( + id=run_id, + name=current.name, + description=current.description, + tags=current.tags, + meta=current.meta, + status=status, + flags=flags, + data=current.data, + ), + ) + except Exception as finalize_error: # pylint: disable=broad-exception-caught + # Best-effort: a run closed mid-flight or vanished must not mask the + # original outcome. + log.error( + "[EVAL] failed to finalize run", + run_id=str(run_id), + status=str(status), + error=str(finalize_error), + ) + + +# ============================================================================= +# Entry point 1: run_from_source — orchestrate a run by topology. +# ============================================================================= + + +async def run_from_source( + *, + project_id: UUID, + user_id: UUID, + run_id: UUID, + newest: Optional[datetime] = None, + oldest: Optional[datetime] = None, + tracing_service: TracingService, + testsets_service: TestsetsService, + queries_service: QueriesService, + workflows_service: WorkflowsService, + applications_service: ApplicationsService, + evaluations_service: EvaluationsService, + simple_evaluators_service: SimpleEvaluatorsService, +) -> bool: + run = await _fetch_validated_run( + project_id=project_id, + run_id=run_id, + evaluations_service=evaluations_service, + ) + if not run: + log.warning("[EVAL] [process-run] run not found or empty", run_id=run_id) + return False + + topology = classify_run_topology(run) + + if topology.dispatch in ("live_query", "batch_query"): + use_windowing = topology.dispatch == "batch_query" + await _run_query_source( + project_id=project_id, + user_id=user_id, + run=run, + # Batch query runs window from the query revision's own bounds, not + # the scheduler tick — so the tick's newest/oldest are dropped. Live + # runs carry the tick's range for temporal bucketing. + newest=None if use_windowing else newest, + oldest=None if use_windowing else oldest, + use_windowing=use_windowing, + tracing_service=tracing_service, + queries_service=queries_service, + workflows_service=workflows_service, + applications_service=applications_service, + evaluations_service=evaluations_service, + ) + return True + + if topology.dispatch in ("batch_testset", "batch_invocation"): + await _run_testset_source( + project_id=project_id, + user_id=user_id, + run=run, + tracing_service=tracing_service, + testsets_service=testsets_service, + testcases_service=None, + workflows_service=workflows_service, + applications_service=applications_service, + evaluations_service=evaluations_service, + ) + return True + + if topology.dispatch in ("queue_traces", "queue_testcases"): + # An open queue: direct trace/testcase batches arrive later via + # run_from_batch and each finalizes its own scenarios. There + # is nothing to execute at run-start; leave the run RUNNING and active. + log.info( + "[EVAL] [process-run] queue run started; awaiting batches", + run_id=str(run.id), + ) + return True + + log.warning( + "[EVAL] [process-run] unsupported run topology", + run_id=run_id, + topology=topology.label, + topology_status=topology.status, + reason=topology.reason, + ) + return False + + +async def _run_testset_source( + *, + project_id: UUID, + user_id: UUID, + run: EvaluationRun, + tracing_service: TracingService, + testsets_service: TestsetsService, + testcases_service: Optional[TestcasesService], + workflows_service: WorkflowsService, + applications_service: ApplicationsService, + evaluations_service: EvaluationsService, +) -> None: + """Resolve testset rows -> mint -> populate -> re-execute. + + Batch (non-live) runs finalize: an empty testset goes straight to SUCCESS + so it does not hang RUNNING; a pre-execution error goes to FAILURE. + """ + try: + input_steps = [step for step in run.data.steps if step.type == "input"] + input_specs = await resolve_testset_input_specs( + project_id=project_id, + input_steps=input_steps, + testsets_service=testsets_service, + ) + source_items = [ + ResolvedSourceItem( + kind="testcase", + step_key=input_spec.step_key, + references={ + "testcase": {"id": str(testcase.id)}, + "testset": {"id": str(input_spec.testset.id)}, + "testset_variant": { + "id": str(input_spec.testset_revision.variant_id) + }, + "testset_revision": {"id": str(input_spec.testset_revision.id)}, + }, + testcase_id=testcase.id, + testcase=testcase, + inputs=testcase_data, + ) + for input_spec in input_specs + for testcase, testcase_data in zip( + input_spec.testcases, + input_spec.testcases_data, + ) + ] + except Exception as e: # pylint: disable=broad-exception-caught + log.error("[EVAL] [process-run] testset resolution failed", error=str(e)) + await _finalize_run_terminal( + project_id=project_id, + user_id=user_id, + run_id=run.id, + status=EvaluationStatus.FAILURE, + evaluations_service=evaluations_service, + ) + return + + if not source_items: + await _finalize_run_terminal( + project_id=project_id, + user_id=user_id, + run_id=run.id, + status=EvaluationStatus.SUCCESS, + evaluations_service=evaluations_service, + ) + return + + try: + bindings = await _mint_and_bind( + project_id=project_id, + user_id=user_id, + run=run, + source_items=source_items, + default_step_key=( + _input_step_keys(run)[0] if _input_step_keys(run) else "" + ), + timestamp=None, + interval=None, + evaluations_service=evaluations_service, + ) + await _execute_bindings( + project_id=project_id, + user_id=user_id, + run_id=run.id, + bindings=bindings, + tracing_service=tracing_service, + testcases_service=testcases_service, + workflows_service=workflows_service, + applications_service=applications_service, + evaluations_service=evaluations_service, + ) + except Exception as e: # pylint: disable=broad-exception-caught + log.error("[EVAL] [process-run] testset execution failed", error=str(e)) + await _finalize_run_terminal( + project_id=project_id, + user_id=user_id, + run_id=run.id, + status=EvaluationStatus.FAILURE, + evaluations_service=evaluations_service, + ) + + +async def _run_query_source( + *, + project_id: UUID, + user_id: UUID, + run: EvaluationRun, + newest: Optional[datetime], + oldest: Optional[datetime], + use_windowing: bool, + tracing_service: TracingService, + queries_service: QueriesService, + workflows_service: WorkflowsService, + applications_service: ApplicationsService, + evaluations_service: EvaluationsService, +) -> None: + """Resolve query traces -> mint -> populate -> re-execute, per query step. + + timestamp/interval are TEMPORAL coordinates and only meaningful for LIVE + runs (use_windowing=False), which bucket metrics over time. Batch query runs + (use_windowing=True) have no temporal axis, so they stay None. + + Batch query runs finalize (empty -> SUCCESS, error -> FAILURE). Live runs + intentionally never finalize — the scheduler keeps polling — so an empty + tick or an error leaves the run untouched. + """ + timestamp: Optional[datetime] = None + interval: Optional[int] = None + if not use_windowing: + timestamp = oldest or datetime.now(timezone.utc) + if newest and oldest: + interval = int((newest - oldest).total_seconds() / 60) + + finalize = use_windowing + + try: + source_items_by_step = await resolve_query_source_items( + project_id=project_id, + run=run, + queries_service=queries_service, + tracing_service=tracing_service, + newest=newest, + oldest=oldest, + use_windowing=use_windowing, + ) + total = sum(len(items) for items in source_items_by_step.values()) + + if total == 0: + # Live run: just wait for the next tick. Batch run: finalize SUCCESS + # so it does not hang RUNNING. + if finalize: + await _finalize_run_terminal( + project_id=project_id, + user_id=user_id, + run_id=run.id, + status=EvaluationStatus.SUCCESS, + evaluations_service=evaluations_service, + ) + return + + for step_key, source_items in source_items_by_step.items(): + if not source_items: + continue + bindings = await _mint_and_bind( + project_id=project_id, + user_id=user_id, + run=run, + source_items=source_items, + default_step_key=step_key, + timestamp=timestamp, + interval=interval, + evaluations_service=evaluations_service, + ) + await _execute_bindings( + project_id=project_id, + user_id=user_id, + run_id=run.id, + bindings=bindings, + tracing_service=tracing_service, + testcases_service=None, + workflows_service=workflows_service, + applications_service=applications_service, + evaluations_service=evaluations_service, + # Query runs only refresh metrics when auto results were created + # (a human-only live tick produces none) — matches the legacy + # query loop's refresh_metrics_without_auto_results=False. + refresh_metrics_without_auto_results=False, + # Live runs (finalize=False) keep ticking; batch runs finalize. + finalize_run_status=finalize, + ) + except Exception as e: # pylint: disable=broad-exception-caught + log.error("[EVAL] [process-run] query flow failed", error=str(e)) + # A pre-execution error in a batch run never reaches the slice's own + # finalize, so flip it to FAILURE here. Live runs keep ticking. + if finalize: + await _finalize_run_terminal( + project_id=project_id, + user_id=user_id, + run_id=run.id, + status=EvaluationStatus.FAILURE, + evaluations_service=evaluations_service, + ) + + +# ============================================================================= +# Entry point 2: run_from_batch — ingest a DIRECT id batch into an +# open queue run. Same mint -> populate -> re-execute as the run flows; the only +# difference is the source comes from explicit trace_ids / testcase_ids. +# ============================================================================= + + +async def run_from_batch( + *, + project_id: UUID, + user_id: UUID, + run_id: UUID, + source_kind: EvaluationSliceSource, + trace_ids: Optional[list[str]] = None, + testcase_ids: Optional[list[UUID]] = None, + input_step_key: Optional[str] = None, + tracing_service: TracingService, + testcases_service: TestcasesService, + workflows_service: WorkflowsService, + applications_service: ApplicationsService, + evaluations_service: EvaluationsService, +) -> bool: + """Ingest a batch of DIRECT source ids (trace_ids / testcase_ids). + + NOTE: not idempotent. There is no dedup on source id, so dispatching the + SAME batch twice mints a second set of scenarios for the same ids. The task + is allow_concurrency=True, so a re-dispatch with a fresh job id also bypasses + the singleton run lock. Acceptable today because batches are dispatched once + per ingest. + """ + if source_kind == "traces": + ids: List[Any] = list(trace_ids or []) + elif source_kind == "testcases": + ids = list(testcase_ids or []) + else: + log.warning( + "[EVAL] [process-slice] unsupported source kind", + run_id=run_id, + source_kind=source_kind, + ) + return False + + if not ids: + return True + + run = await _fetch_validated_run( + project_id=project_id, + run_id=run_id, + evaluations_service=evaluations_service, + ) + if not run: + log.warning( + "[EVAL] [process-slice] run not found or has no steps", run_id=run_id + ) + return False + + input_keys = _input_step_keys(run) + step_key = input_step_key or (input_keys[0] if input_keys else None) + if step_key is None: + log.warning("[EVAL] [process-slice] run has no input step", run_id=run_id) + return False + + # Resolve the direct ids into hydrated source items (one batched fetch for + # testcases; per-id for traces — same as before, but now only once). + source_items = await resolve_direct_source_items( + project_id=project_id, + trace_ids=list(trace_ids or []) if source_kind == "traces" else None, + testcase_ids=list(testcase_ids or []) if source_kind == "testcases" else None, + testcases_service=testcases_service, + tracing_service=tracing_service, + ) + for source_item in source_items: + source_item.step_key = step_key + + bindings = await _mint_and_bind( + project_id=project_id, + user_id=user_id, + run=run, + source_items=source_items, + default_step_key=step_key, + timestamp=None, + interval=None, + evaluations_service=evaluations_service, + ) + await _execute_bindings( + project_id=project_id, + user_id=user_id, + run_id=run_id, + bindings=bindings, + tracing_service=tracing_service, + testcases_service=testcases_service, + workflows_service=workflows_service, + applications_service=applications_service, + evaluations_service=evaluations_service, + ) + return True + + +# ============================================================================= +# Entry point 3: rerun — re-execute EXISTING scenarios +# by coordinate. No mint; the source is RECOVERED from stored input cells (no +# seed_bindings). Owns the aggregate-metrics boundary the ingest flows don't. +# ============================================================================= + + +async def rerun( + *, + project_id: UUID, + user_id: UUID, + run_id: UUID, + scenario_ids: Optional[List[UUID]] = None, + step_keys: Optional[List[str]] = None, + repeat_idxs: Optional[List[int]] = None, + process_mode: SliceProcessMode = "fill-missing", + tracing_service: TracingService, + testcases_service: TestcasesService, + workflows_service: WorkflowsService, + applications_service: ApplicationsService, + evaluations_service: EvaluationsService, +) -> bool: + """Re-execute EXISTING scenarios addressed by a tensor coordinate slice. + + The coordinate counterpart of the ingest flows: it re-runs the runnable + cells of scenarios that already exist (retry, fill-missing, or run a + newly-added step). It rebuilds each scenario's source from its stored input + cell rather than receiving a hydrated one — so NO seed_bindings. + + `process` is results-only by design; this entry point owns the metrics + `refresh` boundary, invoking it after execution over the same slice scope. + """ + tensor_slice = TensorSlice( + run_id=run_id, + scenario_ids=scenario_ids, + step_keys=step_keys, + repeat_idxs=repeat_idxs, + process_mode=process_mode, + ) + + slice_processor = APISliceProcessor( + evaluations_service=evaluations_service, + tracing_service=tracing_service, + testcases_service=testcases_service, + workflows_service=workflows_service, + applications_service=applications_service, + ) + tensor_ops = TensorSliceOperations( + evaluations_service=evaluations_service, + slice_processor=slice_processor, + ) + + await tensor_ops.process( + project_id=project_id, + user_id=user_id, + tensor_slice=tensor_slice, + ) + + # Metrics boundary for the slice. `refresh` recomputes both the per-scenario + # (variational) rows and the AGGREGATE the slice affected — temporal buckets + # for live runs, the global row for non-live. `process` already refreshed + # variational inline; recomputing it is idempotent. + await tensor_ops.refresh( + project_id=project_id, + user_id=user_id, + tensor_slice=tensor_slice, + ) + return True diff --git a/api/oss/src/core/evaluations/types.py b/api/oss/src/core/evaluations/types.py index b9167f13a3..c4d9ec2d2b 100644 --- a/api/oss/src/core/evaluations/types.py +++ b/api/oss/src/core/evaluations/types.py @@ -75,6 +75,106 @@ def __str__(self): return _message +class EvaluationMetricsInvalid(Exception): + """Raised when a metric does not match any of the three valid shapes: + global (scenario_id IS NULL, timestamp IS NULL), + variational (scenario_id set, timestamp IS NULL), + temporal (scenario_id IS NULL, timestamp set). + """ + + def __init__( + self, + message: str = "Metric does not match a valid global/variational/temporal shape.", + run_id: Optional[UUID] = None, + scenario_id: Optional[UUID] = None, + timestamp: Optional[datetime] = None, + ): + super().__init__(message) + + self.message = message + self.run_id = run_id + self.scenario_id = scenario_id + self.timestamp = timestamp + + def __str__(self): + _message = self.message + + if self.run_id: + _message += f" run_id={self.run_id}" + if self.scenario_id: + _message += f" scenario_id={self.scenario_id}" + if self.timestamp: + _message += f" timestamp={self.timestamp}" + + return _message + + +class DefaultQueueError(Exception): + """Base exception for default-queue policy violations.""" + + def __init__(self, message: str, queue_id: Optional[UUID] = None): + super().__init__(message) + self.message = message + self.queue_id = queue_id + + def __str__(self): + _message = self.message + if self.queue_id: + _message += f" queue_id={self.queue_id}" + return _message + + +class DefaultQueueDataInvalid(DefaultQueueError): + """Raised when a default queue carries scenario/step/assignment/batch filters.""" + + def __init__( + self, + message: str = ( + "default queues cannot filter scenarios, steps, assignments, or batches" + ), + queue_id: Optional[UUID] = None, + ): + super().__init__(message, queue_id=queue_id) + + +class DefaultQueueDemotionForbidden(DefaultQueueError): + """Raised when an edit attempts to demote a default queue.""" + + def __init__( + self, + message: str = "default queues cannot be demoted", + queue_id: Optional[UUID] = None, + ): + super().__init__(message, queue_id=queue_id) + + +class DefaultQueueDeletionForbidden(DefaultQueueError): + """Raised when a hard delete targets a default queue (must be archived instead).""" + + def __init__( + self, + message: str = "default queues must be archived, not hard deleted", + queue_id: Optional[UUID] = None, + ): + super().__init__(message, queue_id=queue_id) + + +class DefaultQueueArchiveForbidden(DefaultQueueError): + """Raised when a user-facing archive targets a default queue. + + Default queues are system-managed: their archive/unarchive lifecycle is + driven by run reconciliation, not by direct user action. (Internally, + reconcile archives via `force=True`.) + """ + + def __init__( + self, + message: str = "default queues are system-managed and cannot be archived directly", + queue_id: Optional[UUID] = None, + ): + super().__init__(message, queue_id=queue_id) + + # - EVALUATION RUN ------------------------------------------------------------- @@ -82,12 +182,14 @@ class EvaluationRunFlags(BaseModel): is_live: bool = False # Indicates if the run has live queries is_active: bool = False # Indicates if the run is currently active is_closed: bool = False # Indicates if the run is modifiable - is_queue: bool = False # Indicates this run belongs to a simple annotation queue + is_queue: bool = False # Indicates active default queue + active human work is_cached: bool = False # Indicates the run should reuse traces by hash is_split: bool = False # Indicates repeats fan out at the application step # - has_queries: bool = False # Indicates if the run has queries - has_testsets: bool = False # Indicates if the run has testsets + has_queries: bool = False # Indicates if the run has query-backed inputs + has_testsets: bool = False # Indicates if the run has testset-backed inputs + has_traces: bool = False # Indicates if the run has direct trace inputs + has_testcases: bool = False # Indicates if the run has direct testcase inputs has_evaluators: bool = False # Indicates if the run has evaluators # has_custom: bool = False # Indicates if the run has custom evaluators @@ -100,13 +202,19 @@ class EvaluationRunQueryFlags(BaseModel): is_active: Optional[bool] = None # Indicates if the run is currently active is_closed: Optional[bool] = None # Indicates if the run is modifiable is_queue: Optional[bool] = ( - None # Indicates this run belongs to a simple annotation queue + None # Indicates active default queue + active human work ) is_cached: Optional[bool] = None # Indicates the run should reuse traces by hash is_split: Optional[bool] = None # Indicates repeats fan out at the application step # - has_queries: Optional[bool] = None # Indicates if the run has queries - has_testsets: Optional[bool] = None # Indicates if the run has testsets + has_queries: Optional[bool] = None # Indicates if the run has query-backed inputs + has_testsets: Optional[bool] = ( + None # Indicates if the run has testset-backed inputs + ) + has_traces: Optional[bool] = None # Indicates if the run has direct trace inputs + has_testcases: Optional[bool] = ( + None # Indicates if the run has direct testcase inputs + ) has_evaluators: Optional[bool] = None # Indicates if the run has evaluators # has_custom: Optional[bool] = None # Indicates if the run has custom evaluators @@ -141,9 +249,16 @@ class EvaluationRunDataMapping(BaseModel): step: EvaluationRunDataMappingStep +class EvaluationRunDataConcurrency(BaseModel): + batch_size: Optional[int] = None + max_retries: Optional[int] = None + retry_delay: Optional[float] = None + + class EvaluationRunData(BaseModel): steps: Optional[List[EvaluationRunDataStep]] = None repeats: Optional[int] = 1 + concurrency: Optional[EvaluationRunDataConcurrency] = None mappings: Optional[List[EvaluationRunDataMapping]] = None @field_validator("repeats") @@ -387,10 +502,12 @@ class EvaluationMetricsSpecsRefresh(BaseModel): class EvaluationQueueFlags(BaseModel): is_sequential: bool = False + is_default: bool = False class EvaluationQueueQueryFlags(BaseModel): is_sequential: Optional[bool] = None + is_default: Optional[bool] = None class EvaluationQueueData(BaseModel): @@ -466,6 +583,7 @@ class EvaluationQueueEdit(Identifier, Header, Metadata): class EvaluationQueueQuery(Header, Metadata): flags: Optional[EvaluationQueueQueryFlags] = None # type: ignore + include_archived: Optional[bool] = None user_id: Optional[UUID] = None user_ids: Optional[List[UUID]] = None @@ -500,6 +618,7 @@ class SimpleEvaluationData(BaseModel): evaluator_steps: Optional[Target] = None repeats: Optional[int] = None + concurrency: Optional[EvaluationRunDataConcurrency] = None class SimpleEvaluation(Version, Identifier, Lifecycle, Header, Metadata): @@ -534,6 +653,8 @@ class SimpleEvaluationQuery(Header, Metadata): class SimpleQueueKind(str, Enum): + QUERIES = "queries" + TESTSETS = "testsets" TRACES = "traces" TESTCASES = "testcases" @@ -592,8 +713,12 @@ class SimpleQueueData(BaseModel): evaluators: Optional[Target] = None """ The evaluators to run on each scenario. - Either a list of evaluator revision UUIDs (all treated as 'human'), - or a dict mapping evaluator revision UUID -> origin ('human' | 'auto' | 'custom'). + Either a list of evaluator revision UUIDs (origin-less; for a simple queue + these default to 'human'), or a dict mapping evaluator revision UUID -> + origin ('human' | 'auto' | 'custom'). A simple queue must resolve to at + least one 'human' evaluator; an explicit dict of only non-human evaluators + is rejected at the simple-queue endpoint (the underlying run itself has no + such restriction). """ repeats: Optional[int] = None @@ -632,6 +757,28 @@ def validate_sources(self): if not has_kind and not has_queries and not has_testsets: raise ValueError("simple queue requires kind, queries, or testsets") + if has_queries and self.kind not in (None, SimpleQueueKind.QUERIES): + raise ValueError("query-backed queues must use kind='queries'") + if has_testsets and self.kind not in (None, SimpleQueueKind.TESTSETS): + raise ValueError("testset-backed queues must use kind='testsets'") + if self.kind == SimpleQueueKind.QUERIES and not has_queries: + raise ValueError("kind='queries' requires query sources") + if self.kind == SimpleQueueKind.TESTSETS and not has_testsets: + raise ValueError("kind='testsets' requires testset sources") + + # Simple-queue constraint (endpoint-only, not a run-layer rule): a queue + # is human work, so it must resolve to at least one human evaluator. + # An evaluator is human when it is origin-less (defaults to human at + # build time) or explicitly origin="human". So: + # - a bare list is all origin-less -> always valid. + # - an explicit dict is valid only if at least one value is "human"; + # a dict with only non-human origins (any of auto/custom, or a mix + # of them) has no human evaluator and is rejected. + # The underlying evaluation run has no such restriction. + if isinstance(self.evaluators, dict) and self.evaluators: + if not any(origin == "human" for origin in self.evaluators.values()): + raise ValueError("simple queues must have at least one human evaluator") + return self diff --git a/api/oss/src/core/evaluations/utils.py b/api/oss/src/core/evaluations/utils.py index 8cdc68905b..f27efe659b 100644 --- a/api/oss/src/core/evaluations/utils.py +++ b/api/oss/src/core/evaluations/utils.py @@ -370,11 +370,12 @@ def effective_is_split( *, is_split: bool, is_live: bool = False, - is_queue: bool = False, + has_traces: bool = False, + has_testcases: bool = False, has_application_steps: bool = False, has_evaluator_steps: bool = False, ) -> bool: - if is_live or is_queue: + if is_live or has_traces or has_testcases: return False if not has_application_steps or not has_evaluator_steps: return False @@ -412,16 +413,6 @@ async def fetch_trace( project_id=project_id, trace_id=trace_id, ) - # spans = getattr(trace, "spans", None) if trace else None - # log.debug( - # "[EVAL] [trace] fetch attempt", - # trace_id=trace_id, - # attempt=attempt + 1, - # found=bool(trace), - # spans_type=type(spans).__name__ if spans is not None else None, - # span_count=len(spans) if isinstance(spans, dict) else None, - # usable_root_span=_has_usable_root_span(trace) if trace else False, - # ) if trace and _has_usable_root_span(trace): if isinstance(trace, Trace): return trace diff --git a/api/oss/src/core/events/streaming.py b/api/oss/src/core/events/streaming.py index c8d6f2b0ba..90e19726b4 100644 --- a/api/oss/src/core/events/streaming.py +++ b/api/oss/src/core/events/streaming.py @@ -4,7 +4,6 @@ from orjson import dumps, loads from pydantic import BaseModel -from redis.asyncio import Redis try: from asyncpg.pgproto.pgproto import UUID as AsyncpgUUID @@ -12,7 +11,7 @@ AsyncpgUUID = None from oss.src.core.events.dtos import Event -from oss.src.utils.env import env +from oss.src.dbs.redis.shared.engine import get_streams_engine from oss.src.utils.logging import get_module_logger log = get_module_logger(__name__) @@ -24,16 +23,9 @@ def _orjson_default(obj): raise TypeError(f"Type is not JSON serializable: {type(obj)}") -_redis: Optional[Redis] = None - - -def _get_redis() -> Optional[Redis]: - global _redis - - if _redis is None and env.redis.uri_durable: - _redis = Redis.from_url(env.redis.uri_durable, decode_responses=False) - - return _redis +def _get_redis(): + engine = get_streams_engine() + return engine.get_redis() if engine else None class EventMessage(BaseModel): diff --git a/api/oss/src/core/events/utils.py b/api/oss/src/core/events/utils.py index 5e3c3682cb..8770de46cf 100644 --- a/api/oss/src/core/events/utils.py +++ b/api/oss/src/core/events/utils.py @@ -273,11 +273,11 @@ async def _check_l1_events_quota( try: # Deferred import: EE-only symbols stay out of the OSS import graph. - from ee.src.utils.entitlements import ( # noqa: PLC0415 + from ee.src.core.access.entitlements.service import ( # noqa: PLC0415 check_entitlements, scope_from, ) - from ee.src.core.entitlements.types import Counter # noqa: PLC0415 + from ee.src.core.access.entitlements.types import Counter # noqa: PLC0415 allowed, _, _ = await check_entitlements( key=Counter.EVENTS_INGESTED, diff --git a/api/oss/src/core/secrets/services.py b/api/oss/src/core/secrets/services.py index f986ed911a..ebd527ecb8 100644 --- a/api/oss/src/core/secrets/services.py +++ b/api/oss/src/core/secrets/services.py @@ -64,6 +64,7 @@ async def update_secret( update_secret_dto: UpdateSecretDTO, project_id: UUID | None = None, organization_id: UUID | None = None, + user_id: UUID | None = None, ): with set_data_encryption_key( data_encryption_key=self._data_encryption_key, @@ -73,6 +74,7 @@ async def update_secret( update_secret_dto=update_secret_dto, project_id=project_id, organization_id=organization_id, + user_id=user_id, ) return secret_dto diff --git a/api/oss/src/core/tracing/streaming.py b/api/oss/src/core/tracing/streaming.py index eb41d27aa7..81a145aa77 100644 --- a/api/oss/src/core/tracing/streaming.py +++ b/api/oss/src/core/tracing/streaming.py @@ -1,29 +1,20 @@ import zlib -from typing import List, Optional +from typing import List from uuid import UUID from orjson import dumps, loads from pydantic import BaseModel -from redis.asyncio import Redis -from oss.src.utils.env import env +from oss.src.dbs.redis.shared.engine import get_streams_engine from oss.src.utils.logging import get_module_logger from oss.src.core.tracing.dtos import OTelFlatSpan log = get_module_logger(__name__) -_redis: Optional[Redis] = None -def _get_redis() -> Redis: - global _redis - - if _redis is None: - if not env.redis.uri_durable: - raise RuntimeError("REDIS_URI_DURABLE is required for tracing streams.") - _redis = Redis.from_url(env.redis.uri_durable, decode_responses=False) - - return _redis +def _get_redis(): + return get_streams_engine().get_redis() class SpanMessage(BaseModel): diff --git a/api/oss/src/core/tracing/utils/hashing.py b/api/oss/src/core/tracing/utils/hashing.py index c54fd8fe4c..a7cde121ca 100644 --- a/api/oss/src/core/tracing/utils/hashing.py +++ b/api/oss/src/core/tracing/utils/hashing.py @@ -1,6 +1,6 @@ from hashlib import blake2b from json import dumps -from typing import Dict, Optional, Tuple, Union +from typing import Dict, Optional, Tuple from uuid import UUID from oss.src.core.tracing.dtos import OTelSpan @@ -12,15 +12,11 @@ } -def _trace_id_from_uuid(trace_id: Union[UUID, str]) -> str: - if isinstance(trace_id, UUID): - return trace_id.hex +def _trace_id_from_uuid(trace_id: str) -> str: return UUID(trace_id).hex -def _span_id_from_uuid(span_id: Union[UUID, str]) -> str: - if isinstance(span_id, UUID): - return span_id.hex[16:] +def _span_id_from_uuid(span_id: str) -> str: return UUID(span_id).hex[16:] diff --git a/api/oss/src/core/tracing/utils/parsing.py b/api/oss/src/core/tracing/utils/parsing.py index bc00dc7ea2..759599012b 100644 --- a/api/oss/src/core/tracing/utils/parsing.py +++ b/api/oss/src/core/tracing/utils/parsing.py @@ -132,8 +132,7 @@ def parse_trace_id_to_uuid( clean_trace_id = str(UUID(trace_id)) except Exception as e: log.error( - "trace_id must be a UUID, got %s [%s]", - type(trace_id), + "trace_id must be a UUID-style or hex string, got %r", trace_id, ) raise TypeError() from e @@ -160,8 +159,7 @@ def parse_span_id_to_uuid( clean_span_id = str(UUID(span_id)) except Exception as e: log.error( - "span_id must be a UUID, got %s [%s]", - type(span_id), + "span_id must be a UUID-style or hex string, got %r", span_id, ) raise TypeError() from e @@ -170,23 +168,15 @@ def parse_span_id_to_uuid( def parse_trace_id_from_uuid( - trace_id: Union[UUID, str], + trace_id: str, ): - if isinstance(trace_id, UUID): - return trace_id.hex - - if isinstance(trace_id, str): - return UUID(trace_id).hex + return UUID(trace_id).hex def parse_span_id_from_uuid( - span_id: Union[UUID, str], + span_id: str, ): - if isinstance(span_id, UUID): - return span_id.hex[16:] - - if isinstance(span_id, str): - return UUID(span_id).hex[16:] + return UUID(span_id).hex[16:] def parse_span_kind_to_enum( diff --git a/api/oss/src/core/workflows/service.py b/api/oss/src/core/workflows/service.py index 4ba9e7bd91..1a754ffef2 100644 --- a/api/oss/src/core/workflows/service.py +++ b/api/oss/src/core/workflows/service.py @@ -95,7 +95,7 @@ ResolutionInfo, ) -from oss.src.services.auth_service import sign_secret_token +from oss.src.middlewares.auth import sign_secret_token from oss.src.services.db_manager import get_project_by_id from agenta.sdk.decorators.running import ( diff --git a/api/oss/src/dbs/postgres/blobs/dao.py b/api/oss/src/dbs/postgres/blobs/dao.py index a14bc1bc5b..527e5e9e8a 100644 --- a/api/oss/src/dbs/postgres/blobs/dao.py +++ b/api/oss/src/dbs/postgres/blobs/dao.py @@ -15,7 +15,10 @@ from oss.src.dbs.postgres.shared.utils import apply_windowing from oss.src.dbs.postgres.shared.exceptions import check_entity_creation_conflict -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) from oss.src.dbs.postgres.blobs.mappings import map_dbe_to_dto, map_dto_to_dbe @@ -30,8 +33,12 @@ def __init__( self, *, BlobDBE: Type[T], + engine: TransactionsEngine = None, ): self.BlobDBE = BlobDBE # pylint: disable=invalid-name + if engine is None: + engine = get_transactions_engine() + self.engine = engine # ─ blobs ────────────────────────────────────────────────────────────────── @@ -63,7 +70,7 @@ async def add_blob( ) try: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.BlobDBE).filter( self.BlobDBE.project_id == project_id, # type: ignore ) @@ -106,7 +113,7 @@ async def fetch_blob( # blob_id: UUID, ) -> Optional[Blob]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.BlobDBE).filter( self.BlobDBE.project_id == project_id, # type: ignore ) @@ -138,7 +145,7 @@ async def edit_blob( # blob_edit: BlobEdit, ) -> Optional[Blob]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.BlobDBE).filter( self.BlobDBE.project_id == project_id, # type: ignore ) @@ -179,7 +186,7 @@ async def remove_blob( # blob_id: UUID, ) -> Optional[Blob]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.BlobDBE).filter( self.BlobDBE.project_id == project_id, # type: ignore ) @@ -239,7 +246,7 @@ async def add_blobs( blob_ids = [blob.id for blob in blobs] try: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.BlobDBE).filter( self.BlobDBE.project_id == project_id, # type: ignore ) @@ -295,7 +302,7 @@ async def fetch_blobs( # blob_ids: List[UUID], ) -> List[Blob]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.BlobDBE).filter( self.BlobDBE.project_id == project_id, # type: ignore ) @@ -331,7 +338,7 @@ async def edit_blobs( # blob_edits: List[BlobEdit], ) -> List[Blob]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.BlobDBE).filter( self.BlobDBE.project_id == project_id, # type: ignore ) @@ -381,7 +388,7 @@ async def remove_blobs( # blob_ids: List[UUID], ) -> List[Blob]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.BlobDBE).filter( self.BlobDBE.project_id == project_id, # type: ignore ) @@ -422,7 +429,7 @@ async def query_blobs( # windowing: Optional[Windowing] = None, ) -> List[Blob]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.BlobDBE).filter( self.BlobDBE.project_id == project_id, # type: ignore ) diff --git a/api/oss/src/dbs/postgres/evaluations/dao.py b/api/oss/src/dbs/postgres/evaluations/dao.py index 7fd522e215..1d4a88bfba 100644 --- a/api/oss/src/dbs/postgres/evaluations/dao.py +++ b/api/oss/src/dbs/postgres/evaluations/dao.py @@ -15,7 +15,10 @@ from oss.src.core.shared.exceptions import EntityCreationConflict from oss.src.core.shared.dtos import Windowing from oss.src.core.evaluations.interfaces import EvaluationsDAOInterface -from oss.src.core.evaluations.types import EvaluationClosedConflict +from oss.src.core.evaluations.types import ( + EvaluationClosedConflict, + EvaluationMetricsInvalid, +) from oss.src.core.evaluations.types import ( EvaluationStatus, EvaluationRunFlags, @@ -31,12 +34,10 @@ # EvaluationResult, EvaluationResultCreate, - EvaluationResultEdit, EvaluationResultQuery, # EvaluationMetrics, EvaluationMetricsCreate, - EvaluationMetricsEdit, EvaluationMetricsQuery, # EvaluationQueue, @@ -47,7 +48,10 @@ from oss.src.dbs.postgres.shared.utils import apply_windowing from oss.src.dbs.postgres.shared.exceptions import check_entity_creation_conflict -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) from oss.src.dbs.postgres.evaluations.utils import ( create_run_references, edit_run_references, @@ -74,8 +78,10 @@ class EvaluationsDAO(EvaluationsDAOInterface): - def __init__(self): - pass + def __init__(self, engine: TransactionsEngine = None): + if engine is None: + engine = get_transactions_engine() + self.engine = engine # - EVALUATION RUN --------------------------------------------------------- @@ -113,7 +119,7 @@ async def create_run( run_dbe.data = _run.data.model_dump(mode="json") # type: ignore try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add(run_dbe) await session.commit() @@ -172,7 +178,7 @@ async def create_runs( run_dbes.append(run_dbe) try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add_all(run_dbes) await session.commit() @@ -200,7 +206,7 @@ async def fetch_run( # run_id: UUID, ) -> Optional[EvaluationRun]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, ) @@ -233,7 +239,7 @@ async def fetch_runs( # run_ids: List[UUID], ) -> List[EvaluationRun]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, ) @@ -258,7 +264,7 @@ async def fetch_runs( return _runs - @suppress_exceptions() + @suppress_exceptions(exclude=[EvaluationClosedConflict]) async def edit_run( self, *, @@ -267,7 +273,7 @@ async def edit_run( # run: EvaluationRunEdit, ) -> Optional[EvaluationRun]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, ) @@ -309,6 +315,11 @@ async def edit_run( references=run_references, ) + # `status` is a VARCHAR but `edit_dbe_from_dto` dumps without `mode="json"`. + if run.status is not None: + run_dbe.status = run.status.value # type: ignore + flag_modified(run_dbe, "status") + if run.data: run_dbe.data = run.data.model_dump(mode="json") # type: ignore @@ -321,7 +332,7 @@ async def edit_run( return _run - @suppress_exceptions(default=[]) + @suppress_exceptions(default=[], exclude=[EvaluationClosedConflict]) async def edit_runs( self, *, @@ -332,7 +343,7 @@ async def edit_runs( ) -> List[EvaluationRun]: run_ids = [run.id for run in runs] - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, ) @@ -406,7 +417,7 @@ async def delete_run( # run_id: UUID, ) -> Optional[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, ) @@ -438,7 +449,7 @@ async def delete_runs( # run_ids: List[UUID], ) -> List[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, ) @@ -478,7 +489,7 @@ async def close_run( # status: Optional[EvaluationStatus] = None, ) -> Optional[EvaluationRun]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, EvaluationRunDBE.id == run_id, @@ -526,7 +537,7 @@ async def close_runs( # run_ids: List[UUID], ) -> List[EvaluationRun]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, EvaluationRunDBE.id.in_(run_ids), @@ -576,7 +587,7 @@ async def open_run( # run_id: UUID, ) -> Optional[EvaluationRun]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, EvaluationRunDBE.id == run_id, @@ -620,7 +631,7 @@ async def open_runs( # run_ids: List[UUID], ) -> List[EvaluationRun]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, EvaluationRunDBE.id.in_(run_ids), @@ -671,7 +682,7 @@ async def query_runs( # windowing: Optional[Windowing] = None, ) -> List[EvaluationRun]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE).filter( EvaluationRunDBE.project_id == project_id, ) @@ -704,11 +715,7 @@ async def query_runs( EvaluationRunDBE.tags.contains(run.tags), ) - # meta is JSON (not JSONB) — containment (@>) is not supported - # if run.meta is not None: - # stmt = stmt.filter( - # EvaluationRunDBE.meta.contains(run.meta), - # ) + # meta is JSON (not JSONB) — containment (@>) filtering unsupported if run.status is not None: stmt = stmt.filter( @@ -759,7 +766,7 @@ async def fetch_live_runs( *, windowing: Optional[Windowing] = None, ) -> List[Tuple[UUID, EvaluationRun]]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationRunDBE) stmt = stmt.filter( @@ -806,7 +813,7 @@ async def fetch_live_runs( # - EVALUATION SCENARIO ---------------------------------------------------- - @suppress_exceptions(exclude=[EntityCreationConflict]) + @suppress_exceptions(exclude=[EntityCreationConflict, EvaluationClosedConflict]) async def create_scenario( self, *, @@ -841,7 +848,7 @@ async def create_scenario( ) try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add(scenario_dbe) await session.commit() @@ -858,7 +865,9 @@ async def create_scenario( raise - @suppress_exceptions(default=[], exclude=[EntityCreationConflict]) + @suppress_exceptions( + default=[], exclude=[EntityCreationConflict, EvaluationClosedConflict] + ) async def create_scenarios( self, *, @@ -900,7 +909,7 @@ async def create_scenarios( ] try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add_all(scenario_dbes) await session.commit() @@ -928,7 +937,7 @@ async def fetch_scenario( # scenario_id: UUID, ) -> Optional[EvaluationScenario]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationScenarioDBE).filter( EvaluationScenarioDBE.project_id == project_id, ) @@ -961,7 +970,7 @@ async def fetch_scenarios( # scenario_ids: List[UUID], ) -> List[EvaluationScenario]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationScenarioDBE).filter( EvaluationScenarioDBE.project_id == project_id, ) @@ -986,7 +995,7 @@ async def fetch_scenarios( return _scenarios - @suppress_exceptions() + @suppress_exceptions(exclude=[EvaluationClosedConflict]) async def edit_scenario( self, *, @@ -995,7 +1004,7 @@ async def edit_scenario( # scenario: EvaluationScenarioEdit, ) -> Optional[EvaluationScenario]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationScenarioDBE).filter( EvaluationScenarioDBE.project_id == project_id, ) @@ -1041,7 +1050,7 @@ async def edit_scenario( return _scenario - @suppress_exceptions(default=[]) + @suppress_exceptions(default=[], exclude=[EvaluationClosedConflict]) async def edit_scenarios( self, *, @@ -1052,7 +1061,7 @@ async def edit_scenarios( ) -> List[EvaluationScenario]: scenario_ids = [scenario.id for scenario in scenarios] - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationScenarioDBE).filter( EvaluationScenarioDBE.project_id == project_id, ) @@ -1108,7 +1117,7 @@ async def edit_scenarios( return _scenarios - @suppress_exceptions() + @suppress_exceptions(exclude=[EvaluationClosedConflict]) async def delete_scenario( self, *, @@ -1116,7 +1125,7 @@ async def delete_scenario( # scenario_id: UUID, ) -> Optional[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationScenarioDBE).filter( EvaluationScenarioDBE.project_id == project_id, ) @@ -1152,7 +1161,7 @@ async def delete_scenario( return scenario_id - @suppress_exceptions(default=[]) + @suppress_exceptions(default=[], exclude=[EvaluationClosedConflict]) async def delete_scenarios( self, *, @@ -1160,7 +1169,7 @@ async def delete_scenarios( # scenario_ids: List[UUID], ) -> List[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationScenarioDBE).filter( EvaluationScenarioDBE.project_id == project_id, ) @@ -1207,7 +1216,7 @@ async def query_scenario_ids( # scenario: Optional[EvaluationScenarioQuery] = None, ) -> List[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationScenarioDBE.id).filter( EvaluationScenarioDBE.project_id == project_id, ) @@ -1254,7 +1263,7 @@ async def query_scenarios( # windowing: Optional[Windowing] = None, ) -> List[EvaluationScenario]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationScenarioDBE).filter( EvaluationScenarioDBE.project_id == project_id, ) @@ -1305,11 +1314,7 @@ async def query_scenarios( EvaluationScenarioDBE.tags.contains(scenario.tags), ) - # meta is JSON (not JSONB) — containment (@>) is not supported - # if scenario.meta is not None: - # stmt = stmt.filter( - # EvaluationScenarioDBE.meta.contains(scenario.meta), - # ) + # meta is JSON (not JSONB) — containment (@>) filtering unsupported if scenario.status is not None: stmt = stmt.filter( @@ -1346,7 +1351,7 @@ async def query_scenarios( # - EVALUATION RESULT ------------------------------------------------------ - @suppress_exceptions(exclude=[EntityCreationConflict]) + @suppress_exceptions(exclude=[EntityCreationConflict, EvaluationClosedConflict]) async def create_result( self, *, @@ -1381,7 +1386,7 @@ async def create_result( ) try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add(result_dbe) await session.commit() @@ -1398,8 +1403,8 @@ async def create_result( raise - @suppress_exceptions(default=[], exclude=[EntityCreationConflict]) - async def create_results( + @suppress_exceptions(default=[], exclude=[EvaluationClosedConflict]) + async def set_results( self, *, project_id: UUID, @@ -1407,6 +1412,21 @@ async def create_results( # results: List[EvaluationResultCreate], ) -> List[EvaluationResult]: + """Create or update results (upsert via the composite unique index). + + Conflict target: (project_id, run_id, scenario_id, step_key, repeat_idx). + + Fields updated on conflict: + - Lifecycle: updated_at, updated_by_id + - Data: hash_id, trace_id, testcase_id, error, status, interval, + timestamp, flags, tags, meta (user-defined) + - Management: version (from user data) + + Fields preserved: + - created_at, created_by_id (original) + - id, project_id, run_id, scenario_id, step_key, repeat_idx (identity) + """ + for result in results: run_flags = await _get_run_flags( project_id=project_id, @@ -1418,41 +1438,98 @@ async def create_results( run_id=result.run_id, ) - async with engine.core_session() as session: - _results = [ - EvaluationResult( - **result.model_dump( - mode="json", - exclude_none=True, - ), - created_at=datetime.now(timezone.utc), - created_by_id=user_id, - ) - for result in results - ] + _results = [ + EvaluationResult( + **result.model_dump( + mode="json", + exclude_none=True, + ), + created_at=datetime.now(timezone.utc), + created_by_id=user_id, + ) + for result in results + ] - result_dbes = [ - create_dbe_from_dto( - DBE=EvaluationResultDBE, - project_id=project_id, - dto=_result, - ) - for _result in _results - ] + result_dbes = [ + create_dbe_from_dto( + DBE=EvaluationResultDBE, + project_id=project_id, + dto=_result, + ) + for _result in _results + ] - session.add_all(result_dbes) + # Batch upsert over the single composite unique index, returning the + # upserted rows in the same statement (no per-row follow-up SELECT). + async with self.engine.session() as session: + returned_result_dbes = [] + # Convert DBE instances to dicts using SQLAlchemy's inspection + mapper = inspect(EvaluationResultDBE) + column_names = {col.name for col in mapper.columns} - await session.commit() + now = datetime.now(timezone.utc) + values_list = [] + for dbe in result_dbes: + values_dict = { + k: v + for k, v in dbe.__dict__.items() + if k in column_names and not (k == "id" and v is None) + } + # Add lifecycle values + values_dict["updated_at"] = now + values_dict["updated_by_id"] = user_id + values_list.append(values_dict) - _results = [ - create_dto_from_dbe( - DTO=EvaluationResult, - dbe=result_dbe, - ) - for result_dbe in result_dbes + # A multi-row pg_insert().values([...]) renders a single VALUES + # clause from a uniform column set, so every dict must share the + # same keys (DTOs are dumped with exclude_none, so optional columns + # vary per row). Backfill missing keys with None. + values_list = _uniform_values(values_list) + + # Fields refreshed on conflict (identity/created_* are preserved). + conflict_update_cols = [ + EvaluationResultDBE.updated_at, + EvaluationResultDBE.updated_by_id, + EvaluationResultDBE.hash_id, + EvaluationResultDBE.trace_id, + EvaluationResultDBE.testcase_id, + EvaluationResultDBE.error, + EvaluationResultDBE.status, + EvaluationResultDBE.interval, + EvaluationResultDBE.timestamp, + EvaluationResultDBE.flags, + EvaluationResultDBE.tags, + EvaluationResultDBE.meta, + EvaluationResultDBE.version, ] - return _results + if values_list: + stmt = pg_insert(EvaluationResultDBE).values(values_list) + stmt = stmt.on_conflict_do_update( + index_elements=[ + EvaluationResultDBE.project_id, + EvaluationResultDBE.run_id, + EvaluationResultDBE.scenario_id, + EvaluationResultDBE.step_key, + EvaluationResultDBE.repeat_idx, + ], + set_={ + col.name: stmt.excluded[col.name] + for col in conflict_update_cols + }, + ) + res = await session.execute(stmt.returning(EvaluationResultDBE)) + returned_result_dbes.extend(res.scalars().all()) + + await session.commit() + + return [ + create_dto_from_dbe( + DTO=EvaluationResult, + dbe=result_dbe, + ) + for result_dbe in returned_result_dbes + ] @suppress_exceptions() async def fetch_result( @@ -1462,7 +1539,7 @@ async def fetch_result( # result_id: UUID, ) -> Optional[EvaluationResult]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationResultDBE).filter( EvaluationResultDBE.project_id == project_id, ) @@ -1495,7 +1572,7 @@ async def fetch_results( # result_ids: List[UUID], ) -> List[EvaluationResult]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationResultDBE).filter( EvaluationResultDBE.project_id == project_id, ) @@ -1520,131 +1597,7 @@ async def fetch_results( return _results - @suppress_exceptions() - async def edit_result( - self, - *, - project_id: UUID, - user_id: UUID, - # - result: EvaluationResultEdit, - ) -> Optional[EvaluationResult]: - async with engine.core_session() as session: - stmt = select(EvaluationResultDBE).filter( - EvaluationResultDBE.project_id == project_id, - ) - - stmt = stmt.filter( - EvaluationResultDBE.id == result.id, - ) - - stmt = stmt.limit(1) - - res = await session.execute(stmt) - - result_dbe = res.scalars().first() - - if result_dbe is None: - return None - - run_flags = await _get_run_flags( - session=session, - project_id=project_id, - run_id=result_dbe.run_id, # type: ignore - ) - - if run_flags.get("is_closed", False): - raise EvaluationClosedConflict( - run_id=result_dbe.run_id, # type: ignore - scenario_id=result_dbe.scenario_id, # type: ignore - result_id=result_dbe.id, # type: ignore - ) - - result_dbe = edit_dbe_from_dto( - dbe=result_dbe, - dto=result, - updated_at=datetime.now(timezone.utc), - updated_by_id=user_id, - ) - - await session.commit() - - _result = create_dto_from_dbe( - DTO=EvaluationResult, - dbe=result_dbe, - ) - - return _result - - @suppress_exceptions(default=[]) - async def edit_results( - self, - *, - project_id: UUID, - user_id: UUID, - # - results: List[EvaluationResultEdit], - ) -> List[EvaluationResult]: - result_ids = [result.id for result in results] - - async with engine.core_session() as session: - stmt = select(EvaluationResultDBE).filter( - EvaluationResultDBE.project_id == project_id, - ) - - stmt = stmt.filter( - EvaluationResultDBE.id.in_(result_ids), - ) - - stmt = stmt.limit(len(result_ids)) - - res = await session.execute(stmt) - - result_dbes = res.scalars().all() - - if not result_dbes: - return [] - - for result_dbe in result_dbes: - run_flags = await _get_run_flags( - session=session, - project_id=project_id, - run_id=result_dbe.run_id, # type: ignore - ) - - if run_flags.get("is_closed", False): - raise EvaluationClosedConflict( - run_id=result_dbe.run_id, # type: ignore - scenario_id=result_dbe.scenario_id, # type: ignore - result_id=result_dbe.id, # type: ignore - ) - - result = next( - (s for s in results if s.id == result_dbe.id), - None, - ) - - if result is not None: - result_dbe = edit_dbe_from_dto( - dbe=result_dbe, - dto=result, - updated_at=datetime.now(timezone.utc), - updated_by_id=user_id, - ) - - await session.commit() - - _results = [ - create_dto_from_dbe( - DTO=EvaluationResult, - dbe=result_dbe, - ) - for result_dbe in result_dbes - ] - - return _results - - @suppress_exceptions() + @suppress_exceptions(exclude=[EvaluationClosedConflict]) async def delete_result( self, *, @@ -1652,7 +1605,7 @@ async def delete_result( # result_id: UUID, ) -> Optional[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationResultDBE).filter( EvaluationResultDBE.project_id == project_id, ) @@ -1689,7 +1642,7 @@ async def delete_result( return result_id - @suppress_exceptions(default=[]) + @suppress_exceptions(default=[], exclude=[EvaluationClosedConflict]) async def delete_results( self, *, @@ -1697,7 +1650,7 @@ async def delete_results( # result_ids: List[UUID], ) -> List[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationResultDBE).filter( EvaluationResultDBE.project_id == project_id, ) @@ -1745,7 +1698,7 @@ async def query_results( # windowing: Optional[Windowing] = None, ) -> List[EvaluationResult]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationResultDBE).filter( EvaluationResultDBE.project_id == project_id, ) @@ -1826,11 +1779,7 @@ async def query_results( EvaluationResultDBE.tags.contains(result.tags), ) - # meta is JSON (not JSONB) — containment (@>) is not supported - # if result.meta is not None: - # stmt = stmt.filter( - # EvaluationResultDBE.meta.contains(result.meta), - # ) + # meta is JSON (not JSONB) — containment (@>) filtering unsupported if result.status is not None: stmt = stmt.filter( @@ -1865,9 +1814,73 @@ async def query_results( return results + @suppress_exceptions(default=[]) + async def query_result_ids( + self, + *, + project_id: UUID, + # + result: Optional[EvaluationResultQuery] = None, + ) -> List[UUID]: + # ID-only counterpart of `query_results` (mirrors `query_scenario_ids`): + # selects just the id column so callers that delete/count by id do not + # hydrate full result DTOs. Supports the slice-coordinate filters that + # the tensor ops address by; full-row filters live on `query_results`. + async with self.engine.session() as session: + stmt = select(EvaluationResultDBE.id).filter( + EvaluationResultDBE.project_id == project_id, + ) + + if result is not None: + if result.ids is not None: + stmt = stmt.filter(EvaluationResultDBE.id.in_(result.ids)) + + if result.run_id is not None: + stmt = stmt.filter(EvaluationResultDBE.run_id == result.run_id) + + if result.run_ids is not None: + stmt = stmt.filter(EvaluationResultDBE.run_id.in_(result.run_ids)) + + if result.scenario_id is not None: + stmt = stmt.filter( + EvaluationResultDBE.scenario_id == result.scenario_id + ) + + if result.scenario_ids is not None: + stmt = stmt.filter( + EvaluationResultDBE.scenario_id.in_(result.scenario_ids) + ) + + if result.step_key is not None: + stmt = stmt.filter(EvaluationResultDBE.step_key == result.step_key) + + if result.step_keys is not None: + stmt = stmt.filter( + EvaluationResultDBE.step_key.in_(result.step_keys) + ) + + if result.repeat_idx is not None: + stmt = stmt.filter( + EvaluationResultDBE.repeat_idx == result.repeat_idx + ) + + if result.repeat_idxs is not None: + stmt = stmt.filter( + EvaluationResultDBE.repeat_idx.in_(result.repeat_idxs) + ) + + stmt = stmt.order_by(EvaluationResultDBE.id.asc()) + + res = await session.execute(stmt) + + return list(res.scalars().all()) + # - EVALUATION METRICS ----------------------------------------------------- - async def create_metrics( + @suppress_exceptions( + default=[], exclude=[EvaluationClosedConflict, EvaluationMetricsInvalid] + ) + async def set_metrics( self, *, project_id: UUID, @@ -1926,12 +1939,13 @@ async def create_metrics( ] # Classify metrics into 3 groups based on NULL pattern, then batch upsert - async with engine.core_session() as session: + async with self.engine.session() as session: returned_metric_dbes = [] # Convert DBE instances to dicts using SQLAlchemy's inspection mapper = inspect(EvaluationMetricsDBE) column_names = {col.name for col in mapper.columns} + now = datetime.now(timezone.utc) values_list = [] for dbe in metric_dbes: values_dict = { @@ -1939,10 +1953,12 @@ async def create_metrics( for k, v in dbe.__dict__.items() if k in column_names and not (k == "id" and v is None) } + # Add lifecycle values + values_dict["updated_at"] = now + values_dict["updated_by_id"] = user_id values_list.append(values_dict) # Precompute which metrics belong to each of the 3 index types - now = datetime.now(timezone.utc) global_metrics = [] # scenario_id IS NULL AND timestamp IS NULL variational_metrics = [] # scenario_id IS NOT NULL AND timestamp IS NULL temporal_metrics = [] # scenario_id IS NULL AND timestamp IS NOT NULL @@ -1951,10 +1967,6 @@ async def create_metrics( scenario_id = value_dict.get("scenario_id") timestamp = value_dict.get("timestamp") - # Add lifecycle values - value_dict["updated_at"] = now - value_dict["updated_by_id"] = user_id - if scenario_id is None and timestamp is None: global_metrics.append(value_dict) elif timestamp is None and scenario_id is not None: @@ -1962,27 +1974,31 @@ async def create_metrics( elif scenario_id is None and timestamp is not None: temporal_metrics.append(value_dict) else: - get_module_logger(__name__).warning( - f"Unexpected metric pattern: scenario_id={scenario_id}, " - f"timestamp={timestamp}. Skipping upsert." - ) - - # Upsert each metric type with its corresponding partial unique index - # Shared update set for all upserts - conflict_update_set = { - EvaluationMetricsDBE.updated_at: EvaluationMetricsDBE.updated_at, - EvaluationMetricsDBE.updated_by_id: EvaluationMetricsDBE.updated_by_id, - EvaluationMetricsDBE.data: EvaluationMetricsDBE.data, - EvaluationMetricsDBE.flags: EvaluationMetricsDBE.flags, - EvaluationMetricsDBE.tags: EvaluationMetricsDBE.tags, - EvaluationMetricsDBE.meta: EvaluationMetricsDBE.meta, - EvaluationMetricsDBE.status: EvaluationMetricsDBE.status, - EvaluationMetricsDBE.version: EvaluationMetricsDBE.version, - } + # scenario_id AND timestamp both set matches no unique index; + # surface it rather than silently dropping the metric. + raise EvaluationMetricsInvalid( + run_id=value_dict.get("run_id"), + scenario_id=scenario_id, + timestamp=timestamp, + ) + + # Fields refreshed on conflict (identity/created_* are preserved). + conflict_update_cols = [ + EvaluationMetricsDBE.updated_at, + EvaluationMetricsDBE.updated_by_id, + EvaluationMetricsDBE.data, + EvaluationMetricsDBE.flags, + EvaluationMetricsDBE.tags, + EvaluationMetricsDBE.meta, + EvaluationMetricsDBE.status, + EvaluationMetricsDBE.version, + ] # Global: (project_id, run_id) WHERE scenario_id IS NULL AND timestamp IS NULL if global_metrics: - stmt = pg_insert(EvaluationMetricsDBE).values(global_metrics) + stmt = pg_insert(EvaluationMetricsDBE).values( + _uniform_values(global_metrics) + ) stmt = stmt.on_conflict_do_update( index_elements=[ EvaluationMetricsDBE.project_id, @@ -1992,16 +2008,19 @@ async def create_metrics( EvaluationMetricsDBE.scenario_id.is_(None), EvaluationMetricsDBE.timestamp.is_(None), ), - set_=dict( - (k, stmt.excluded[k.name]) for k in conflict_update_set.keys() - ), + set_={ + col.name: stmt.excluded[col.name] + for col in conflict_update_cols + }, ) res = await session.execute(stmt.returning(EvaluationMetricsDBE)) returned_metric_dbes.extend(res.scalars().all()) # Variational: (project_id, run_id, scenario_id) WHERE timestamp IS NULL AND scenario_id IS NOT NULL if variational_metrics: - stmt = pg_insert(EvaluationMetricsDBE).values(variational_metrics) + stmt = pg_insert(EvaluationMetricsDBE).values( + _uniform_values(variational_metrics) + ) stmt = stmt.on_conflict_do_update( index_elements=[ EvaluationMetricsDBE.project_id, @@ -2012,16 +2031,19 @@ async def create_metrics( EvaluationMetricsDBE.scenario_id.isnot(None), EvaluationMetricsDBE.timestamp.is_(None), ), - set_=dict( - (k, stmt.excluded[k.name]) for k in conflict_update_set.keys() - ), + set_={ + col.name: stmt.excluded[col.name] + for col in conflict_update_cols + }, ) res = await session.execute(stmt.returning(EvaluationMetricsDBE)) returned_metric_dbes.extend(res.scalars().all()) # Temporal: (project_id, run_id, timestamp) WHERE scenario_id IS NULL AND timestamp IS NOT NULL if temporal_metrics: - stmt = pg_insert(EvaluationMetricsDBE).values(temporal_metrics) + stmt = pg_insert(EvaluationMetricsDBE).values( + _uniform_values(temporal_metrics) + ) stmt = stmt.on_conflict_do_update( index_elements=[ EvaluationMetricsDBE.project_id, @@ -2032,9 +2054,10 @@ async def create_metrics( EvaluationMetricsDBE.scenario_id.is_(None), EvaluationMetricsDBE.timestamp.isnot(None), ), - set_=dict( - (k, stmt.excluded[k.name]) for k in conflict_update_set.keys() - ), + set_={ + col.name: stmt.excluded[col.name] + for col in conflict_update_cols + }, ) res = await session.execute(stmt.returning(EvaluationMetricsDBE)) returned_metric_dbes.extend(res.scalars().all()) @@ -2059,7 +2082,7 @@ async def fetch_metrics( # metrics_ids: List[UUID], ) -> List[EvaluationMetrics]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationMetricsDBE).filter( EvaluationMetricsDBE.project_id == project_id, ) @@ -2084,75 +2107,7 @@ async def fetch_metrics( return _metrics - @suppress_exceptions(default=[]) - async def edit_metrics( - self, - *, - project_id: UUID, - user_id: UUID, - # - metrics: List[EvaluationMetricsEdit], - ) -> List[EvaluationMetrics]: - metrics_ids = [metric.id for metric in metrics] - - async with engine.core_session() as session: - stmt = select(EvaluationMetricsDBE).filter( - EvaluationMetricsDBE.project_id == project_id, - ) - - stmt = stmt.filter( - EvaluationMetricsDBE.id.in_(metrics_ids), - ) - - stmt = stmt.limit(len(metrics_ids)) - - res = await session.execute(stmt) - - metric_dbes = res.scalars().all() - - if not metric_dbes: - return [] - - for metric_dbe in metric_dbes: - run_flags = await _get_run_flags( - session=session, - project_id=project_id, - run_id=metric_dbe.run_id, # type: ignore - ) - - if run_flags.get("is_closed", False): - raise EvaluationClosedConflict( - run_id=metric_dbe.run_id, # type: ignore - scenario_id=metric_dbe.scenario_id, # type: ignore - metrics_id=metric_dbe.id, # type: ignore - ) - - metric = next( - (m for m in metrics if m.id == metric_dbe.id), - None, - ) - - if metric is not None: - metric_dbe = edit_dbe_from_dto( - dbe=metric_dbe, - dto=metric, - updated_at=datetime.now(timezone.utc), - updated_by_id=user_id, - ) - - await session.commit() - - _metrics = [ - create_dto_from_dbe( - DTO=EvaluationMetrics, - dbe=metric_dbe, - ) - for metric_dbe in metric_dbes - ] - - return _metrics - - @suppress_exceptions(default=[]) + @suppress_exceptions(default=[], exclude=[EvaluationClosedConflict]) async def delete_metrics( self, *, @@ -2160,7 +2115,7 @@ async def delete_metrics( # metrics_ids: Optional[List[UUID]] = None, ) -> List[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: if metrics_ids is None: return [] @@ -2211,7 +2166,7 @@ async def query_metrics( # windowing: Optional[Windowing] = None, ) -> List[EvaluationMetrics]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationMetricsDBE).filter( EvaluationMetricsDBE.project_id == project_id, ) @@ -2284,11 +2239,7 @@ async def query_metrics( EvaluationMetricsDBE.tags.contains(metric.tags), ) - # meta is JSON (not JSONB) — containment (@>) is not supported - # if metric.meta is not None: - # stmt = stmt.filter( - # EvaluationMetricsDBE.meta.contains(metric.meta), - # ) + # meta is JSON (not JSONB) — containment (@>) filtering unsupported if metric.status is not None: stmt = stmt.filter( @@ -2325,7 +2276,7 @@ async def query_metrics( # - EVALUATION QUEUE ------------------------------------------------------- - @suppress_exceptions(exclude=[EntityCreationConflict]) + @suppress_exceptions(exclude=[EntityCreationConflict, EvaluationClosedConflict]) async def create_queue( self, *, @@ -2366,7 +2317,7 @@ async def create_queue( queue_dbe.user_ids = _flatten_queue_user_ids(queue.data) try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add(queue_dbe) await session.commit() @@ -2379,11 +2330,13 @@ async def create_queue( return _queue except Exception as e: - check_entity_creation_conflict(e) + check_entity_creation_conflict(e, entity="Evaluation queue") raise - @suppress_exceptions(default=[], exclude=[EntityCreationConflict]) + @suppress_exceptions( + default=[], exclude=[EntityCreationConflict, EvaluationClosedConflict] + ) async def create_queues( self, *, @@ -2422,7 +2375,7 @@ async def create_queues( queue_dbe.user_ids = _flatten_queue_user_ids(queue.data) try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add_all(queue_dbes) await session.commit() @@ -2450,7 +2403,7 @@ async def fetch_queue( # queue_id: UUID, ) -> Optional[EvaluationQueue]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationQueueDBE).filter( EvaluationQueueDBE.project_id == project_id, ) @@ -2483,7 +2436,7 @@ async def fetch_queues( # queue_ids: List[UUID], ) -> List[EvaluationQueue]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationQueueDBE).filter( EvaluationQueueDBE.project_id == project_id, ) @@ -2508,7 +2461,7 @@ async def fetch_queues( return _queues - @suppress_exceptions() + @suppress_exceptions(exclude=[EvaluationClosedConflict]) async def edit_queue( self, *, @@ -2517,7 +2470,7 @@ async def edit_queue( # queue: EvaluationQueueEdit, ) -> Optional[EvaluationQueue]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationQueueDBE).filter( EvaluationQueueDBE.project_id == project_id, ) @@ -2569,7 +2522,7 @@ async def edit_queue( return _queue - @suppress_exceptions(default=[]) + @suppress_exceptions(default=[], exclude=[EvaluationClosedConflict]) async def edit_queues( self, *, @@ -2579,7 +2532,7 @@ async def edit_queues( ) -> List[EvaluationQueue]: queue_ids = [queue.id for queue in queues] - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationQueueDBE).filter( EvaluationQueueDBE.project_id == project_id, ) @@ -2641,6 +2594,63 @@ async def edit_queues( return _queues + @suppress_exceptions() + async def archive_queue( + self, + *, + project_id: UUID, + user_id: UUID, + queue_id: UUID, + ) -> Optional[EvaluationQueue]: + async with self.engine.session() as session: + stmt = ( + select(EvaluationQueueDBE) + .filter( + EvaluationQueueDBE.project_id == project_id, + EvaluationQueueDBE.id == queue_id, + ) + .limit(1) + ) + queue_dbe = (await session.execute(stmt)).scalars().first() + if queue_dbe is None: + return None + + now = datetime.now(timezone.utc) + queue_dbe.deleted_at = now + queue_dbe.deleted_by_id = user_id + queue_dbe.updated_at = now + queue_dbe.updated_by_id = user_id + await session.commit() + return create_dto_from_dbe(DTO=EvaluationQueue, dbe=queue_dbe) + + @suppress_exceptions() + async def unarchive_queue( + self, + *, + project_id: UUID, + user_id: UUID, + queue_id: UUID, + ) -> Optional[EvaluationQueue]: + async with self.engine.session() as session: + stmt = ( + select(EvaluationQueueDBE) + .filter( + EvaluationQueueDBE.project_id == project_id, + EvaluationQueueDBE.id == queue_id, + ) + .limit(1) + ) + queue_dbe = (await session.execute(stmt)).scalars().first() + if queue_dbe is None: + return None + + queue_dbe.deleted_at = None + queue_dbe.deleted_by_id = None + queue_dbe.updated_at = datetime.now(timezone.utc) + queue_dbe.updated_by_id = user_id + await session.commit() + return create_dto_from_dbe(DTO=EvaluationQueue, dbe=queue_dbe) + @suppress_exceptions() async def delete_queue( self, @@ -2649,7 +2659,7 @@ async def delete_queue( # queue_id: UUID, ) -> Optional[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationQueueDBE).filter( EvaluationQueueDBE.project_id == project_id, ) @@ -2681,7 +2691,7 @@ async def delete_queues( # queue_ids: List[UUID], ) -> List[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationQueueDBE).filter( EvaluationQueueDBE.project_id == project_id, ) @@ -2716,11 +2726,14 @@ async def query_queues( # windowing: Optional[Windowing] = None, ) -> List[EvaluationQueue]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(EvaluationQueueDBE).filter( EvaluationQueueDBE.project_id == project_id, ) + if not queue or not queue.include_archived: + stmt = stmt.filter(EvaluationQueueDBE.deleted_at.is_(None)) + if queue is not None: if queue.ids is not None: stmt = stmt.filter( @@ -2763,11 +2776,7 @@ async def query_queues( EvaluationQueueDBE.tags.contains(queue.tags), ) - # meta is JSON (not JSONB) — containment (@>) is not supported - # if queue.meta is not None: - # stmt = stmt.filter( - # EvaluationQueueDBE.meta.contains(queue.meta), - # ) + # meta is JSON (not JSONB) — containment (@>) filtering unsupported if queue.name is not None: stmt = stmt.filter( @@ -2805,6 +2814,33 @@ async def query_queues( # -------------------------------------------------------------------------- +def _uniform_values(values_list: list[dict]) -> list[dict]: + """Backfill a list of insert dicts to a uniform key set. + + A multi-row ``pg_insert().values([...])`` renders ONE VALUES clause whose + columns are taken from the union of keys; rows missing a key would shift + columns and silently corrupt (or empty) the insert. DTOs are dumped with + ``exclude_none``, so optional columns are absent on some rows. We backfill + every missing key with ``None`` so all rows align. + + ``id`` is special: it is the server-defaulted primary key, so forcing + ``id=None`` would override the default. It is therefore all-or-nothing — + kept only when EVERY row already carries it (explicit-id upsert), otherwise + dropped from all rows so the DB generates it. + """ + if not values_list: + return values_list + + all_keys = set() + for values in values_list: + all_keys.update(values.keys()) + + if not all(("id" in values) for values in values_list): + all_keys.discard("id") + + return [{key: values.get(key) for key in all_keys} for values in values_list] + + async def _get_run_flags( *, project_id: UUID, @@ -2812,7 +2848,8 @@ async def _get_run_flags( session: Optional[AsyncSession] = None, ) -> dict: if session is None: - async with engine.core_session() as session: + engine = get_transactions_engine() + async with engine.session() as session: return await _get_run_flags( project_id=project_id, run_id=run_id, diff --git a/api/oss/src/dbs/postgres/evaluations/dbes.py b/api/oss/src/dbs/postgres/evaluations/dbes.py index e3ad7e8490..bf971022cb 100644 --- a/api/oss/src/dbs/postgres/evaluations/dbes.py +++ b/api/oss/src/dbs/postgres/evaluations/dbes.py @@ -3,6 +3,7 @@ ForeignKeyConstraint, UniqueConstraint, Index, + text, ) from oss.src.dbs.postgres.shared.base import Base @@ -286,6 +287,13 @@ class EvaluationQueueDBE( "ix_evaluation_queues_run_id", "run_id", ), # for filtering + Index( + "ux_evaluation_queues_default_per_run", + "project_id", + "run_id", + unique=True, + postgresql_where=text("(flags ->> 'is_default')::boolean = true"), + ), Index( "ix_evaluation_queues_user_ids", "user_ids", diff --git a/api/oss/src/dbs/postgres/evaluations/utils.py b/api/oss/src/dbs/postgres/evaluations/utils.py index f1d60fcfa7..9bd596bb8c 100644 --- a/api/oss/src/dbs/postgres/evaluations/utils.py +++ b/api/oss/src/dbs/postgres/evaluations/utils.py @@ -8,6 +8,14 @@ EvaluationRunQuery, ) +# Source-family detection keys. An input step's family comes from the exact +# reference key the resolvers read (`query_revision` / `testset_revision`), or — +# for reference-less direct sources — from the exact step key. +QUERY_REFERENCE_KEY = "query_revision" +TESTSET_REFERENCE_KEY = "testset_revision" +DIRECT_TRACE_STEP_KEYS = {"traces", "query-direct"} +DIRECT_TESTCASE_STEP_KEYS = {"testcases", "testset-direct"} + def _make_run_references( run: Optional[Union[EvaluationRun, EvaluationRunEdit]] = None, @@ -91,8 +99,14 @@ def _make_run_flags( if not run.data or not run.data.steps: return flags + # `is_queue` is deliberately NOT recomputed here: it depends on the + # default-queue lifecycle (which the DAO does not load) and is owned by + # EvaluationsService._reconcile_default_queue. Writing it via the DAO without + # running reconcile leaves it stale. Only the `has_*` shape flags below. flags.has_queries = False flags.has_testsets = False + flags.has_traces = False + flags.has_testcases = False flags.has_evaluators = False # flags.has_custom = False @@ -103,21 +117,22 @@ def _make_run_flags( if _step.type == "input": _references = _step.references or dict() - if flags.is_queue and not _references: + if not _references: step_key = (_step.key or "").lower() - if "query" in step_key: - flags.has_queries = True - if "testset" in step_key: - flags.has_testsets = True - - for _key in _references.keys(): - step_key = str(_key).lower() - - if "query" in step_key: - flags.has_queries = True - if "testset" in step_key: - flags.has_testsets = True + # Direct source inputs are explicit source families. Legacy + # direct keys remain recognized for old rows. + if step_key in DIRECT_TRACE_STEP_KEYS: + flags.has_traces = True + if step_key in DIRECT_TESTCASE_STEP_KEYS: + flags.has_testcases = True + + # Match the exact reference key, not a substring: a substring rule + # misfires on incidental keys like `query_anchor` / `testset_metadata`. + if QUERY_REFERENCE_KEY in _references: + flags.has_queries = True + if TESTSET_REFERENCE_KEY in _references: + flags.has_testsets = True if _step.type == "annotation": flags.has_evaluators = True diff --git a/api/oss/src/dbs/postgres/events/dao.py b/api/oss/src/dbs/postgres/events/dao.py index fcedab16cc..8dd86d0b19 100644 --- a/api/oss/src/dbs/postgres/events/dao.py +++ b/api/oss/src/dbs/postgres/events/dao.py @@ -13,12 +13,14 @@ map_event_dto_to_dbe, map_event_dbe_to_dto, ) -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import AnalyticsEngine, get_analytics_engine class EventsDAO(EventsDAOInterface): - def __init__(self): - pass + def __init__(self, engine: AnalyticsEngine = None): + if engine is None: + engine = get_analytics_engine() + self.engine = engine ### EVENTS @@ -32,7 +34,7 @@ async def ingest( if not events: return 0 - async with engine.tracing_session() as session: + async with self.engine.session() as session: total_ingested = 0 for event in events: @@ -87,7 +89,7 @@ async def query( # windowing: Optional[Windowing] = None, ) -> List[Event]: - async with engine.tracing_session() as session: + async with self.engine.session() as session: # BASE stmt = select(EventDBE) diff --git a/api/oss/src/dbs/postgres/folders/dao.py b/api/oss/src/dbs/postgres/folders/dao.py index 2a03df3a93..c81b062f8d 100644 --- a/api/oss/src/dbs/postgres/folders/dao.py +++ b/api/oss/src/dbs/postgres/folders/dao.py @@ -20,7 +20,10 @@ FolderPathDepthExceeded, FolderPathLengthExceeded, ) -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) from oss.src.dbs.postgres.folders.dbes import FolderDBE from oss.src.dbs.postgres.folders.mappings import ( create_dbe_from_dto, @@ -111,8 +114,10 @@ async def _delete_folder_tree( class FoldersDAO(FoldersDAOInterface): - def __init__(self): - pass + def __init__(self, engine: TransactionsEngine = None): + if engine is None: + engine = get_transactions_engine() + self.engine = engine @suppress_exceptions( exclude=[ @@ -133,7 +138,7 @@ async def create( parent_path = None if folder_create.parent_id: - async with engine.core_session() as session: + async with self.engine.session() as session: parent = await _get_folder_row( session=session, folder_id=folder_create.parent_id, @@ -153,7 +158,7 @@ async def create( ) folder_dbe.created_by_id = user_id - async with engine.core_session() as session: + async with self.engine.session() as session: try: session.add(folder_dbe) await session.commit() @@ -175,7 +180,7 @@ async def fetch( project_id: UUID, folder_id: UUID, ) -> Optional[Folder]: - async with engine.core_session() as session: + async with self.engine.session() as session: folder = await _get_folder_row( session=session, folder_id=folder_id, @@ -207,7 +212,7 @@ async def edit( ) -> Optional[Folder]: kind = folder_edit.kind or FolderKind.APPLICATIONS - async with engine.core_session() as session: + async with self.engine.session() as session: folder = await _get_folder_row( session=session, folder_id=folder_edit.id, @@ -285,7 +290,7 @@ async def delete( user_id: UUID, folder_id: UUID, ) -> Optional[UUID]: - async with engine.core_session() as session: + async with self.engine.session() as session: folder = await _get_folder_row( session=session, folder_id=folder_id, @@ -311,7 +316,7 @@ async def query( project_id: UUID, folder_query: FolderQuery, ) -> List[Folder]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(FolderDBE).filter(FolderDBE.project_id == project_id) if folder_query.id is not None: diff --git a/api/oss/src/dbs/postgres/git/dao.py b/api/oss/src/dbs/postgres/git/dao.py index 154381cad6..c601db6ebd 100644 --- a/api/oss/src/dbs/postgres/git/dao.py +++ b/api/oss/src/dbs/postgres/git/dao.py @@ -35,7 +35,10 @@ from oss.src.dbs.postgres.shared.utils import apply_windowing from oss.src.dbs.postgres.shared.exceptions import check_entity_creation_conflict from oss.src.utils.exceptions import suppress_exceptions -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) from oss.src.dbs.postgres.git.mappings import ( map_dbe_to_dto, map_dto_to_dbe, @@ -55,10 +58,14 @@ def __init__( ArtifactDBE: Type[T], VariantDBE: Type[T], RevisionDBE: Type[T], + engine: TransactionsEngine = None, ): self.ArtifactDBE = ArtifactDBE # pylint: disable=invalid-name self.VariantDBE = VariantDBE # pylint: disable=invalid-name self.RevisionDBE = RevisionDBE # pylint: disable=invalid-name + if engine is None: + engine = get_transactions_engine() + self.engine = engine # ─ artifacts ────────────────────────────────────────────────────────────── @@ -97,7 +104,7 @@ async def create_artifact( ) try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add(artifact_dbe) await session.commit() @@ -130,7 +137,7 @@ async def fetch_artifact( if not artifact_ref: return None - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.ArtifactDBE).filter( self.ArtifactDBE.project_id == project_id, # type: ignore ) @@ -168,7 +175,7 @@ async def edit_artifact( # artifact_edit: ArtifactEdit, ) -> Optional[Artifact]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.ArtifactDBE).filter( self.ArtifactDBE.project_id == project_id, # type: ignore ) @@ -225,7 +232,7 @@ async def archive_artifact( # artifact_id: UUID, ) -> Optional[Artifact]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.ArtifactDBE).filter( self.ArtifactDBE.project_id == project_id, # type: ignore ) @@ -267,7 +274,7 @@ async def unarchive_artifact( # artifact_id: UUID, ) -> Optional[Artifact]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.ArtifactDBE).filter( self.ArtifactDBE.project_id == project_id, # type: ignore ) @@ -314,7 +321,7 @@ async def query_artifacts( # windowing: Optional[Windowing] = None, ) -> List[Artifact]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.ArtifactDBE).filter( self.ArtifactDBE.project_id == project_id, # type: ignore ) @@ -453,7 +460,7 @@ async def create_variant( ) try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add(variant_dbe) await session.commit() @@ -487,9 +494,7 @@ async def fetch_variant( if not artifact_ref and not variant_ref: return None - applied_identifying_filter = False - - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( select(self.VariantDBE) .options( @@ -502,6 +507,12 @@ async def fetch_variant( pick_default_variant = False + # Track whether any identifying filter was applied. If we reach the + # query without one, bail out instead of running + # `WHERE project_id = ... LIMIT 1`, which would return an arbitrary + # row. + applied_identifying_filter = False + if artifact_ref: if artifact_ref.id: stmt = stmt.filter(self.VariantDBE.artifact_id == artifact_ref.id) # type: ignore @@ -566,7 +577,7 @@ async def edit_variant( # variant_edit: VariantEdit, ) -> Optional[Variant]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.VariantDBE).filter( self.VariantDBE.project_id == project_id, # type: ignore ) @@ -613,7 +624,7 @@ async def archive_variant( # variant_id: UUID, ) -> Optional[Variant]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.VariantDBE).filter( self.VariantDBE.project_id == project_id, # type: ignore ) @@ -672,7 +683,7 @@ async def unarchive_variant( # variant_id: UUID, ) -> Optional[Variant]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.VariantDBE).filter( self.VariantDBE.project_id == project_id, # type: ignore ) @@ -736,7 +747,7 @@ async def query_variants( # windowing: Optional[Windowing] = None, ) -> List[Variant]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( select(self.VariantDBE) .options( @@ -1031,7 +1042,7 @@ async def create_revision( ) try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add(revision_dbe) await session.commit() @@ -1077,15 +1088,7 @@ async def fetch_revision( if not variant_ref and not revision_ref: return None - # Track whether any identifying filter was applied. If we reach the - # query without one (e.g. `revision_ref` carries only a `version` and - # no `variant_ref` is provided to scope it), bail out instead of - # running `WHERE project_id = ... LIMIT 1`, which would return an - # arbitrary row. A version is a per-variant sequence number and cannot - # identify a revision on its own. - applied_identifying_filter = False - - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( select(self.RevisionDBE) .options( @@ -1097,6 +1100,14 @@ async def fetch_revision( ) ) + # Track whether any identifying filter was applied. If we reach the + # query without one (e.g. `revision_ref` carries only a `version` and + # no `variant_ref` is provided to scope it), bail out instead of + # running `WHERE project_id = ... LIMIT 1`, which would return an + # arbitrary row. A version is a per-variant sequence number and cannot + # identify a revision on its own. + applied_identifying_filter = False + if revision_ref and (revision_ref.id or revision_ref.slug): if revision_ref.id: stmt = stmt.filter(self.RevisionDBE.id == revision_ref.id) # type: ignore @@ -1163,7 +1174,7 @@ async def edit_revision( # revision_edit: RevisionEdit, ) -> Optional[Revision]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.RevisionDBE).filter( self.RevisionDBE.project_id == project_id, # type: ignore ) @@ -1210,7 +1221,7 @@ async def archive_revision( # revision_id: UUID, ) -> Optional[Revision]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.RevisionDBE).filter( self.RevisionDBE.project_id == project_id, # type: ignore ) @@ -1252,7 +1263,7 @@ async def unarchive_revision( # revision_id: UUID, ) -> Optional[Revision]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.RevisionDBE).filter( self.RevisionDBE.project_id == project_id, # type: ignore ) @@ -1303,7 +1314,7 @@ async def query_revisions( # windowing: Optional[Windowing] = None, ) -> List[Revision]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( select(self.RevisionDBE) .options( @@ -1591,7 +1602,7 @@ async def commit_revision( ) try: - async with engine.core_session() as session: + async with self.engine.session() as session: if initial and revision_commit.variant_id: # Lock the variant row so concurrent initial commits queue up rather # than racing through the count check below. @@ -1727,7 +1738,7 @@ async def log_revisions( # `include_archived`. ROW_NUMBER() over that set gives us each row's # 1-indexed position; we then keep rows up to the target's position # and limit to `depth` from the tail. - async with engine.core_session() as session: + async with self.engine.session() as session: visibility_filter = ( (self.RevisionDBE.deleted_at.is_(None),) # type: ignore if not include_archived @@ -1793,7 +1804,7 @@ async def _get_version( variant_id: UUID, revision_id: UUID, ) -> str: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( select(func.count()) # pylint: disable=not-callable .select_from(self.RevisionDBE) # type: ignore @@ -1817,7 +1828,7 @@ async def _set_version( revision_id: UUID, version: str, ) -> None: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = update(self.RevisionDBE).filter( self.RevisionDBE.project_id == project_id, # type: ignore ) @@ -1836,7 +1847,7 @@ async def _null_revision_fields( project_id: UUID, revision_id: UUID, ) -> None: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( update(self.RevisionDBE) .filter( diff --git a/api/oss/src/dbs/postgres/secrets/dao.py b/api/oss/src/dbs/postgres/secrets/dao.py index e356e84059..7a755bb8fc 100644 --- a/api/oss/src/dbs/postgres/secrets/dao.py +++ b/api/oss/src/dbs/postgres/secrets/dao.py @@ -3,8 +3,10 @@ from oss.src.dbs.postgres.secrets.dbes import SecretsDBE from oss.src.core.secrets.interfaces import SecretsDAOInterface - -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) from oss.src.core.secrets.dtos import CreateSecretDTO, UpdateSecretDTO from oss.src.dbs.postgres.secrets.mappings import ( @@ -17,8 +19,10 @@ class SecretsDAO(SecretsDAOInterface): - def __init__(self): - pass + def __init__(self, engine: TransactionsEngine = None): + if engine is None: + engine = get_transactions_engine() + self.engine = engine @staticmethod def _validate_scope(project_id: UUID | None, organization_id: UUID | None) -> None: @@ -48,7 +52,7 @@ async def create( organization_id=organization_id, secret_dto=create_secret_dto, ) - async with engine.core_session() as session: + async with self.engine.session() as session: session.add(secrets_dbe) await session.commit() @@ -61,7 +65,7 @@ async def get( project_id: UUID | None, organization_id: UUID | None, ): - async with engine.core_session() as session: + async with self.engine.session() as session: scope_filter = self._scope_filter(project_id, organization_id) stmt = select(SecretsDBE).filter_by( id=secret_id, @@ -77,7 +81,7 @@ async def get( return secrets_dto async def list(self, project_id: UUID | None, organization_id: UUID | None): - async with engine.core_session() as session: + async with self.engine.session() as session: scope_filter = self._scope_filter(project_id, organization_id) stmt = select(SecretsDBE).filter_by(**scope_filter) @@ -95,8 +99,9 @@ async def update( update_secret_dto: UpdateSecretDTO, project_id: UUID | None, organization_id: UUID | None, + user_id: UUID | None = None, ): - async with engine.core_session() as session: + async with self.engine.session() as session: scope_filter = self._scope_filter(project_id, organization_id) stmt = select(SecretsDBE).filter_by( id=secret_id, @@ -109,7 +114,9 @@ async def update( return None map_secrets_dto_to_dbe_update( - secrets_dbe=secrets_dbe, update_secret_dto=update_secret_dto + secrets_dbe=secrets_dbe, + update_secret_dto=update_secret_dto, + user_id=user_id, ) await session.commit() @@ -124,7 +131,7 @@ async def delete( project_id: UUID | None, organization_id: UUID | None, ): - async with engine.core_session() as session: + async with self.engine.session() as session: scope_filter = self._scope_filter(project_id, organization_id) stmt = select(SecretsDBE).filter_by( id=secret_id, diff --git a/api/oss/src/dbs/postgres/secrets/mappings.py b/api/oss/src/dbs/postgres/secrets/mappings.py index 14397c3194..4390f6a05b 100644 --- a/api/oss/src/dbs/postgres/secrets/mappings.py +++ b/api/oss/src/dbs/postgres/secrets/mappings.py @@ -1,5 +1,6 @@ import uuid import json +from datetime import datetime, timezone from oss.src.dbs.postgres.secrets.dbes import SecretsDBE from oss.src.core.secrets.dtos import ( @@ -30,8 +31,13 @@ def map_secrets_dto_to_dbe( def map_secrets_dto_to_dbe_update( - secrets_dbe: SecretsDBE, update_secret_dto: UpdateSecretDTO + secrets_dbe: SecretsDBE, + update_secret_dto: UpdateSecretDTO, + user_id=None, ) -> None: + secrets_dbe.updated_at = datetime.now(timezone.utc) + secrets_dbe.updated_by_id = user_id + if update_secret_dto.header: for key, value in update_secret_dto.header.model_dump( exclude_none=True diff --git a/api/oss/src/dbs/postgres/shared/dbas.py b/api/oss/src/dbs/postgres/shared/dbas.py index 846d0f99df..68a9cc5714 100644 --- a/api/oss/src/dbs/postgres/shared/dbas.py +++ b/api/oss/src/dbs/postgres/shared/dbas.py @@ -97,7 +97,6 @@ class LegacyLifecycleDBA: ) updated_at = Column( TIMESTAMP(timezone=True), - server_onupdate=func.current_timestamp(), nullable=True, ) updated_by_id = Column( @@ -116,7 +115,6 @@ class LifecycleDBA: ) updated_at = Column( TIMESTAMP(timezone=True), - server_onupdate=func.current_timestamp(), nullable=True, ) deleted_at = Column( diff --git a/api/oss/src/dbs/postgres/shared/engine.py b/api/oss/src/dbs/postgres/shared/engine.py index 67253fbed1..b37985ccb9 100644 --- a/api/oss/src/dbs/postgres/shared/engine.py +++ b/api/oss/src/dbs/postgres/shared/engine.py @@ -1,5 +1,5 @@ from asyncio import current_task -from typing import AsyncGenerator +from typing import AsyncGenerator, Optional from contextlib import asynccontextmanager from math import floor @@ -11,17 +11,14 @@ async_scoped_session, ) -from oss.src.dbs.postgres.shared.config import ( - POSTGRES_URI_CORE, - POSTGRES_URI_TRACING, -) +from oss.src.utils.env import env DATABASE_MEMORY = 32 * 1024 * 1024 * 1024 # 32 GB DATABASE_FACTOR = 8 * 1024 * 1024 * 1.15 # 8 MB + 15% overhead -DATABASE_MAX_CONNECTIONS = 5000 # 5000 connections +DATABASE_MAX_CONNECTIONS = 5000 MAX_CONNECTIONS = min(DATABASE_MEMORY / DATABASE_FACTOR, DATABASE_MAX_CONNECTIONS) -APP_CONNECTIONS = MAX_CONNECTIONS * 0.9 # reserve 10% for non-app connections +APP_CONNECTIONS = MAX_CONNECTIONS * 0.9 NOF_CONSUMERS = 2 * 4 # 2 engines x 4 containers NOF_CONNECTIONS = floor(APP_CONNECTIONS / NOF_CONSUMERS) POOL_SIZE = floor(NOF_CONNECTIONS * 0.25) @@ -29,98 +26,99 @@ POOL_RECYCLE = 30 * 60 # 30 minutes -class Engine: - def __init__(self) -> None: - self.postgres_uri_core = POSTGRES_URI_CORE +class TransactionsEngine: + """Postgres core DB — application data.""" - self.async_core_engine: AsyncEngine = create_async_engine( - url=self.postgres_uri_core, + def __init__(self) -> None: + self._engine: AsyncEngine = create_async_engine( + url=env.postgres.uri_core, pool_pre_ping=True, pool_recycle=POOL_RECYCLE, pool_size=POOL_SIZE, max_overflow=MAX_OVERFLOW, ) - self.async_core_session_maker = async_sessionmaker( + _session_maker = async_sessionmaker( autocommit=False, autoflush=False, class_=AsyncSession, expire_on_commit=False, - bind=self.async_core_engine, + bind=self._engine, ) - self.async_core_session = async_scoped_session( - session_factory=self.async_core_session_maker, + self._session = async_scoped_session( + session_factory=_session_maker, scopefunc=current_task, ) - self.postgres_uri_tracing = POSTGRES_URI_TRACING + async def close(self) -> None: + if self._engine is not None: + await self._engine.dispose() - self.async_tracing_engine: AsyncEngine = create_async_engine( - url=self.postgres_uri_tracing, + @asynccontextmanager + async def session(self) -> AsyncGenerator[AsyncSession, None]: + session: AsyncSession = self._session() + try: + yield session + await session.commit() + except Exception as e: + await session.rollback() + raise e + finally: + await session.close() + + +class AnalyticsEngine: + """Postgres tracing DB — observability data.""" + + def __init__(self) -> None: + self._engine: AsyncEngine = create_async_engine( + url=env.postgres.uri_tracing, pool_pre_ping=True, pool_recycle=POOL_RECYCLE, pool_size=POOL_SIZE, max_overflow=MAX_OVERFLOW, ) - self.async_tracing_session_maker = async_sessionmaker( + _session_maker = async_sessionmaker( autocommit=False, autoflush=False, class_=AsyncSession, expire_on_commit=False, - bind=self.async_tracing_engine, + bind=self._engine, ) - - self.async_tracing_session = async_scoped_session( - session_factory=self.async_tracing_session_maker, + self._session = async_scoped_session( + session_factory=_session_maker, scopefunc=current_task, ) - async def open(self): - raise NotImplementedError() - - async def close(self): - if self.async_core_engine is not None: - await self.async_core_engine.dispose() - - if self.async_tracing_engine is not None: - await self.async_tracing_engine.dispose() + async def close(self) -> None: + if self._engine is not None: + await self._engine.dispose() @asynccontextmanager - async def core_session(self) -> AsyncGenerator[AsyncSession, None]: - session: AsyncSession = self.async_core_session() - + async def session(self) -> AsyncGenerator[AsyncSession, None]: + session: AsyncSession = self._session() try: yield session await session.commit() - except Exception as e: await session.rollback() raise e - finally: await session.close() - @asynccontextmanager - async def tracing_session(self) -> AsyncGenerator[AsyncSession, None]: - session: AsyncSession = self.async_tracing_session() - - try: - yield session - await session.commit() - - except Exception as e: - await session.rollback() - raise e - - finally: - await session.close() - ### LEGACY CODE ### +_transactions_engine: Optional[TransactionsEngine] = None +_analytics_engine: Optional[AnalyticsEngine] = None - async def init_db(self): - self.open() - async def close_db(self): - self.close() +def get_transactions_engine() -> TransactionsEngine: + global _transactions_engine + if _transactions_engine is None: + _transactions_engine = TransactionsEngine() + return _transactions_engine -engine = Engine() +def get_analytics_engine() -> AnalyticsEngine: + global _analytics_engine + if _analytics_engine is None: + _analytics_engine = AnalyticsEngine() + return _analytics_engine diff --git a/api/oss/src/dbs/postgres/tools/dao.py b/api/oss/src/dbs/postgres/tools/dao.py index f94e87f273..c3cefe279c 100644 --- a/api/oss/src/dbs/postgres/tools/dao.py +++ b/api/oss/src/dbs/postgres/tools/dao.py @@ -16,7 +16,10 @@ ToolConnectionCreate, ) -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) from oss.src.dbs.postgres.tools.dbes import ToolConnectionDBE from oss.src.dbs.postgres.tools.mappings import ( map_connection_create_to_dbe, @@ -28,8 +31,16 @@ class ToolsDAO(ToolsDAOInterface): - def __init__(self, *, ToolConnectionDBE: type = ToolConnectionDBE): + def __init__( + self, + *, + ToolConnectionDBE: type = ToolConnectionDBE, + engine: TransactionsEngine = None, + ): self.ToolConnectionDBE = ToolConnectionDBE + if engine is None: + engine = get_transactions_engine() + self.engine = engine @suppress_exceptions(exclude=[EntityCreationConflict]) async def create_connection( @@ -49,7 +60,7 @@ async def create_connection( ) try: - async with engine.core_session() as session: + async with self.engine.session() as session: session.add(dbe) await session.commit() await session.refresh(dbe) @@ -80,7 +91,7 @@ async def get_connection( connection_id: UUID, ) -> Optional[ToolConnection]: """Fetch a connection by ID scoped to project_id. Returns None if not found.""" - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( select(self.ToolConnectionDBE) .filter(self.ToolConnectionDBE.project_id == project_id) @@ -109,7 +120,7 @@ async def update_connection( data_update: Optional[dict] = None, ) -> Optional[ToolConnection]: """Partially update flags and/or data for a connection. Returns updated DTO or None.""" - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( select(self.ToolConnectionDBE) .filter(self.ToolConnectionDBE.project_id == project_id) @@ -158,7 +169,7 @@ async def delete_connection( connection_id: UUID, ) -> bool: """Hard-delete a connection row. Returns True if a row was deleted.""" - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( delete(self.ToolConnectionDBE) .where(self.ToolConnectionDBE.project_id == project_id) @@ -181,7 +192,7 @@ async def query_connections( is_active: Optional[bool] = True, ) -> List[ToolConnection]: """List connections with optional filters. Defaults to active-only (is_active=True).""" - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.ToolConnectionDBE).filter( self.ToolConnectionDBE.project_id == project_id, ) @@ -215,7 +226,7 @@ async def activate_connection_by_provider_id( project_id: Optional[UUID] = None, ) -> Optional[ToolConnection]: """Set is_valid=True and is_active=True for the connection matching the provider ID.""" - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(self.ToolConnectionDBE).filter( self.ToolConnectionDBE.data["connected_account_id"].astext == provider_connection_id @@ -252,7 +263,7 @@ async def find_connection_by_provider_id( provider_connection_id: str, ) -> Optional[ToolConnection]: """Lookup any connection by provider-side connected_account_id (no project scope).""" - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = ( select(self.ToolConnectionDBE) .filter( diff --git a/api/oss/src/dbs/postgres/tracing/dao.py b/api/oss/src/dbs/postgres/tracing/dao.py index e2295cac8a..e10ce0f31d 100644 --- a/api/oss/src/dbs/postgres/tracing/dao.py +++ b/api/oss/src/dbs/postgres/tracing/dao.py @@ -30,7 +30,7 @@ ) from oss.src.dbs.postgres.shared.utils import apply_windowing -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import AnalyticsEngine, get_analytics_engine from oss.src.dbs.postgres.tracing.dbes import SpanDBE from oss.src.dbs.postgres.tracing.mappings import ( map_span_dbe_to_link_dto, @@ -67,8 +67,10 @@ class TracingDAO(TracingDAOInterface): - def __init__(self): - pass + def __init__(self, engine: AnalyticsEngine = None): + if engine is None: + engine = get_analytics_engine() + self.engine = engine ### SPANS @@ -85,7 +87,7 @@ async def ingest( if not span_dtos: return [] - async with engine.tracing_session() as session: + async with self.engine.session() as session: link_dtos: List[OTelLink] = [] for span_dto in span_dtos: @@ -158,7 +160,7 @@ async def query( # --------- try: - async with engine.tracing_session() as session: + async with self.engine.session() as session: # TIMEOUT await session.execute(TIMEOUT_STMT) # ------- @@ -413,7 +415,7 @@ async def analytics( # log.trace(str(statistics_stmt.compile(**DEBUG_ARGS)).replace("\n", " ")) # --------- - async with engine.tracing_session() as session: + async with self.engine.session() as session: await session.execute(TIMEOUT_STMT) rows = (await session.execute(select(statistics_stmt))).mappings().all() @@ -544,7 +546,7 @@ async def legacy_analytics( ) = parse_windowing(query.windowing) try: - async with engine.tracing_session() as session: + async with self.engine.session() as session: await session.execute(TIMEOUT_STMT) # BASE QUERY HELPERS @@ -747,7 +749,7 @@ async def fetch( if not trace_ids and not span_ids: return [] - async with engine.tracing_session() as session: + async with self.engine.session() as session: stmt = select(SpanDBE).filter(SpanDBE.project_id == project_id) if trace_ids: @@ -785,7 +787,7 @@ async def delete( if not trace_ids: return [] - async with engine.tracing_session() as session: + async with self.engine.session() as session: stmt = select(SpanDBE).filter( SpanDBE.project_id == project_id, SpanDBE.trace_id.in_(trace_ids), @@ -874,7 +876,7 @@ async def _query_by_group( realtime: If True, use last_active (mutable, shows recent activity but unstable cursors). If False/None, use first_active (immutable, stable cursors but doesn't reflect new activity). """ - async with engine.tracing_session() as session: + async with self.engine.session() as session: # TIMEOUT await session.execute(TIMEOUT_STMT) diff --git a/api/oss/src/dbs/postgres/users/dao.py b/api/oss/src/dbs/postgres/users/dao.py index 2fbe1c71b9..5da796a824 100644 --- a/api/oss/src/dbs/postgres/users/dao.py +++ b/api/oss/src/dbs/postgres/users/dao.py @@ -3,7 +3,10 @@ from sqlalchemy import select from sqlalchemy.exc import IntegrityError -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) from oss.src.dbs.postgres.users.dbes import UserIdentityDBE from oss.src.dbs.postgres.users.mappings import ( map_identity_dbe_to_dto, @@ -13,10 +16,15 @@ class IdentitiesDAO: + def __init__(self, engine: Optional[TransactionsEngine] = None): + if engine is None: + engine = get_transactions_engine() + self.engine = engine + async def create(self, dto: UserIdentityCreate) -> UserIdentity: identity_dbe = map_create_dto_to_dbe(dto) - async with engine.core_session() as session: + async with self.engine.session() as session: try: session.add(identity_dbe) await session.commit() @@ -37,7 +45,7 @@ async def create(self, dto: UserIdentityCreate) -> UserIdentity: async def get_by_method_subject( self, method: str, subject: str ) -> Optional[UserIdentity]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(UserIdentityDBE).filter_by( method=method, subject=subject, @@ -51,7 +59,7 @@ async def get_by_method_subject( return map_identity_dbe_to_dto(identity_dbe) async def list_by_user(self, user_id: UUID) -> List[UserIdentity]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(UserIdentityDBE).filter_by(user_id=user_id) result = await session.execute(stmt) identity_dbes = result.scalars().all() @@ -59,7 +67,7 @@ async def list_by_user(self, user_id: UUID) -> List[UserIdentity]: return [map_identity_dbe_to_dto(dbe) for dbe in identity_dbes] async def list_by_domain(self, domain: str) -> List[UserIdentity]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(UserIdentityDBE).filter_by(domain=domain) result = await session.execute(stmt) identity_dbes = result.scalars().all() diff --git a/api/oss/src/dbs/postgres/webhooks/dao.py b/api/oss/src/dbs/postgres/webhooks/dao.py index 47a31154af..6df48b8e4e 100644 --- a/api/oss/src/dbs/postgres/webhooks/dao.py +++ b/api/oss/src/dbs/postgres/webhooks/dao.py @@ -17,7 +17,10 @@ ) from oss.src.core.webhooks.interfaces import WebhooksDAOInterface -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) from oss.src.dbs.postgres.shared.utils import apply_windowing from oss.src.dbs.postgres.webhooks.dbes import ( WebhookSubscriptionDBE, @@ -33,8 +36,10 @@ class WebhooksDAO(WebhooksDAOInterface): - def __init__(self): - pass + def __init__(self, engine: TransactionsEngine = None): + if engine is None: + engine = get_transactions_engine() + self.engine = engine # --- SUBSCRIPTIONS ------------------------------------------------------ # @@ -57,7 +62,7 @@ async def create_subscription( secret_id=secret_id, ) - async with engine.core_session() as session: + async with self.engine.session() as session: session.add(subscription_dbe) await session.commit() @@ -75,7 +80,7 @@ async def fetch_subscription( # subscription_id: UUID, ) -> Optional[WebhookSubscription]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(WebhookSubscriptionDBE).where( WebhookSubscriptionDBE.project_id == project_id, WebhookSubscriptionDBE.id == subscription_id, @@ -102,7 +107,7 @@ async def edit_subscription( # secret_id: UUID | None = None, ) -> Optional[WebhookSubscription]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(WebhookSubscriptionDBE).where( WebhookSubscriptionDBE.id == subscription.id, WebhookSubscriptionDBE.project_id == project_id, @@ -140,7 +145,7 @@ async def delete_subscription( # subscription_id: UUID, ) -> bool: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(WebhookSubscriptionDBE).where( WebhookSubscriptionDBE.project_id == project_id, WebhookSubscriptionDBE.id == subscription_id, @@ -168,7 +173,7 @@ async def query_subscriptions( # windowing: Optional[Windowing] = None, ) -> List[WebhookSubscription]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(WebhookSubscriptionDBE).filter( WebhookSubscriptionDBE.project_id == project_id, ) @@ -234,7 +239,7 @@ async def create_delivery( delivery=delivery, ) - async with engine.core_session() as session: + async with self.engine.session() as session: values = { c.name: getattr(delivery_dbe, c.name) for c in WebhookDeliveryDBE.__table__.columns @@ -275,7 +280,7 @@ async def fetch_delivery( # delivery_id: UUID, ) -> Optional[WebhookDelivery]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(WebhookDeliveryDBE).where( WebhookDeliveryDBE.project_id == project_id, WebhookDeliveryDBE.id == delivery_id, @@ -301,7 +306,7 @@ async def query_deliveries( # windowing: Optional[Windowing] = None, ) -> List[WebhookDelivery]: - async with engine.core_session() as session: + async with self.engine.session() as session: stmt = select(WebhookDeliveryDBE).filter( WebhookDeliveryDBE.project_id == project_id, ) diff --git a/api/oss/src/dbs/redis/shared/engine.py b/api/oss/src/dbs/redis/shared/engine.py new file mode 100644 index 0000000000..59eeeba976 --- /dev/null +++ b/api/oss/src/dbs/redis/shared/engine.py @@ -0,0 +1,131 @@ +from typing import TYPE_CHECKING, Optional + +from oss.src.utils.env import env + +if TYPE_CHECKING: + from redis.asyncio import Redis + + +class CacheEngine: + """Redis volatile — caching (short socket timeout). + + Lazily opens a single client and delegates unknown attributes to it, so + callers use `cache_engine.get(...)`, `cache_engine.scan(...)`, etc. directly + (no separate accessor). Distributed locks use `LockEngine` instead — it has + a longer socket timeout suited to blocking lock operations. + """ + + # 0.5s — cache ops are fast; fail quickly rather than stall a request. + _SOCKET_TIMEOUT = 0.5 + + def __init__(self) -> None: + from redis.asyncio import Redis + + self._r: Optional[Redis] = None + + def _client(self) -> "Redis": + if self._r is None: + from redis.asyncio import Redis + + self._r = Redis.from_url( + url=env.redis.uri_volatile, + decode_responses=False, + socket_timeout=self._SOCKET_TIMEOUT, + ) + return self._r + + def __getattr__(self, name: str): + # Only reached for attributes not found normally (so `_r`, `_client`, + # `close`, etc. are unaffected). Proxies redis client methods. + return getattr(self._client(), name) + + async def close(self) -> None: + if self._r is not None: + await self._r.close() + self._r = None + + +class LockEngine: + """Redis volatile — distributed locks (long socket timeout). + + Same volatile Redis as `CacheEngine`, but a separate client with a longer + socket timeout because lock operations may block. Delegates unknown + attributes to its lazy client, so callers use `lock_engine.set(...)`, + `lock_engine.eval(...)`, etc. directly. + """ + + # 30s — lock ops may block (e.g. waiting to acquire); don't time out early. + _SOCKET_TIMEOUT = 30 + + def __init__(self) -> None: + from redis.asyncio import Redis + + self._r: Optional[Redis] = None + + def _client(self) -> "Redis": + if self._r is None: + from redis.asyncio import Redis + + self._r = Redis.from_url( + url=env.redis.uri_volatile, + decode_responses=False, + socket_timeout=self._SOCKET_TIMEOUT, + ) + return self._r + + def __getattr__(self, name: str): + return getattr(self._client(), name) + + async def close(self) -> None: + if self._r is not None: + await self._r.close() + self._r = None + + +class StreamsEngine: + """Redis durable — persistent streams for tracing/events.""" + + def __init__(self) -> None: + from redis.asyncio import Redis + + self._redis: Optional[Redis] = None + + def get_redis(self) -> "Redis": + if self._redis is None: + from redis.asyncio import Redis + + if not env.redis.uri_durable: + raise RuntimeError("REDIS_URI_DURABLE is required for streams.") + self._redis = Redis.from_url(env.redis.uri_durable, decode_responses=False) + return self._redis + + async def close(self) -> None: + if self._redis is not None: + await self._redis.close() + self._redis = None + + +_cache_engine: Optional[CacheEngine] = None +_lock_engine: Optional[LockEngine] = None +_streams_engine: Optional[StreamsEngine] = None + + +def get_cache_engine() -> CacheEngine: + global _cache_engine + if _cache_engine is None: + _cache_engine = CacheEngine() + return _cache_engine + + +def get_lock_engine() -> LockEngine: + global _lock_engine + if _lock_engine is None: + _lock_engine = LockEngine() + return _lock_engine + + +def get_streams_engine() -> StreamsEngine: + global _streams_engine + if _streams_engine is None: + _streams_engine = StreamsEngine() + return _streams_engine diff --git a/api/oss/src/middlewares/__init__.py b/api/oss/src/middlewares/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/oss/src/services/analytics_service.py b/api/oss/src/middlewares/analytics.py similarity index 88% rename from api/oss/src/services/analytics_service.py rename to api/oss/src/middlewares/analytics.py index 77b08fe8c7..48acc8f90e 100644 --- a/api/oss/src/services/analytics_service.py +++ b/api/oss/src/middlewares/analytics.py @@ -3,12 +3,12 @@ from datetime import datetime from typing import Callable, Optional -import posthog from fastapi import Request from oss.src.utils.caching import get_cache, set_cache from oss.src.utils.common import is_oss from oss.src.utils.env import env from oss.src.utils.logging import get_module_logger +from oss.src.utils.lazy import _load_posthog log = get_module_logger(__name__) @@ -44,15 +44,6 @@ } -# Initialize PostHog only if enabled -if env.posthog.enabled: - posthog.api_key = env.posthog.api_key - posthog.host = env.posthog.api_url - log.info("✓ PostHog enabled") -else: - log.warn("✗ PostHog disabled") - - async def _set_activation_property( distinct_id: str, property_name: str, @@ -63,7 +54,11 @@ async def _set_activation_property( Uses caching to ensure the property is only set once per user. Uses PostHog's $set_once to ensure idempotency. """ - if not distinct_id or not env.posthog.enabled: + if not distinct_id: + return + + posthog = _load_posthog() + if posthog is None: return project_id = getattr(request.state, "project_id", None) @@ -120,7 +115,11 @@ def capture_oss_deployment_created(user_email: str, organization_id: str): No-op if PostHog is not configured. """ - if is_oss() and env.posthog.enabled: + if is_oss(): + posthog = _load_posthog() + if posthog is None: + return + try: posthog.capture( distinct_id=user_email, @@ -246,29 +245,33 @@ async def analytics_middleware(request: Request, call_next: Callable): except Exception: # pylint: disable=bare-except pass - if distinct_id and env.posthog.api_key: - properties["$set"] = {"email": distinct_id} - - posthog.capture( - distinct_id=distinct_id, - event=event_name, - properties=properties or {}, - ) - - # Check if this is an activation event - if event_name in ACTIVATION_EVENTS: - property_name, allowed_auth_methods = ACTIVATION_EVENTS[event_name] - - # Check if auth method is allowed for this activation - if ( - allowed_auth_methods is None - or auth_method in allowed_auth_methods - ): - await _set_activation_property( - distinct_id=distinct_id, - property_name=property_name, - request=request, - ) + if distinct_id: + posthog = _load_posthog() + if posthog is not None: + properties["$set"] = {"email": distinct_id} + + posthog.capture( + distinct_id=distinct_id, + event=event_name, + properties=properties or {}, + ) + + # Check if this is an activation event + if event_name in ACTIVATION_EVENTS: + property_name, allowed_auth_methods = ACTIVATION_EVENTS[ + event_name + ] + + # Check if auth method is allowed for this activation + if ( + allowed_auth_methods is None + or auth_method in allowed_auth_methods + ): + await _set_activation_property( + distinct_id=distinct_id, + property_name=property_name, + request=request, + ) except Exception as e: log.error(f"❌ Error capturing event in PostHog: {e}") diff --git a/api/oss/src/services/auth_service.py b/api/oss/src/middlewares/auth.py similarity index 99% rename from api/oss/src/services/auth_service.py rename to api/oss/src/middlewares/auth.py index 2470262875..e86b4d5f02 100644 --- a/api/oss/src/services/auth_service.py +++ b/api/oss/src/middlewares/auth.py @@ -27,7 +27,7 @@ from oss.src.utils.common import is_ee from oss.src.services import db_manager from oss.src.services import api_key_service -from oss.src.services.exceptions import ( +from oss.src.utils.exceptions import ( UnauthorizedException, InternalServerErrorException, GatewayTimeoutException, @@ -127,7 +127,7 @@ def _log_bearer_auth_denied( ) -async def authentication_middleware(request: Request, call_next): +async def auth_middleware(request: Request, call_next): """ Middleware function for authentication. @@ -438,7 +438,7 @@ def _deny(stage: str, reason: str, exc: Optional[Exception] = None): log.error("Timeout: get_user_from_supertokens()") raise GatewayTimeoutException( - detail="Failed to reach auth provider. Please try again later.", + message="Failed to reach auth provider. Please try again later.", ) from e if not user_info: diff --git a/api/oss/src/models/api/workspace_models.py b/api/oss/src/models/api/workspace_models.py index f5b8bf66e8..8200b01035 100644 --- a/api/oss/src/models/api/workspace_models.py +++ b/api/oss/src/models/api/workspace_models.py @@ -17,7 +17,7 @@ class InviteRequest(BaseModel): email: str # Role slugs are dynamic at runtime (env-overridable via AGENTA_ACCESS_ROLES); # validation against the effective workspace catalog happens at the API - # boundary via `ee.src.core.entitlements.controls.get_role`. + # boundary via `ee.src.core.access.controls.get_role`. roles: Optional[List[str]] = None diff --git a/api/oss/src/routers/api_key_router.py b/api/oss/src/routers/api_key_router.py index 55e73d4246..0febd547fd 100644 --- a/api/oss/src/routers/api_key_router.py +++ b/api/oss/src/routers/api_key_router.py @@ -7,8 +7,8 @@ from oss.src.models.api.api_models import ListAPIKeysResponse if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import check_action_access router = APIRouter() diff --git a/api/oss/src/routers/health_router.py b/api/oss/src/routers/health_router.py deleted file mode 100644 index 2288ef95ce..0000000000 --- a/api/oss/src/routers/health_router.py +++ /dev/null @@ -1,9 +0,0 @@ -from fastapi import status -from oss.src.utils.common import APIRouter - -router = APIRouter() - - -@router.get("/", status_code=status.HTTP_200_OK, operation_id="health_check") -def health_check(): - return {"status": "ok"} diff --git a/api/oss/src/routers/organization_router.py b/api/oss/src/routers/organization_router.py index 8cfde80121..1af04dfa5f 100644 --- a/api/oss/src/routers/organization_router.py +++ b/api/oss/src/routers/organization_router.py @@ -30,21 +30,21 @@ def _role_description(role: str) -> str: """ if not is_ee(): return "" - from ee.src.core.entitlements.controls import get_role_description + from ee.src.core.access.controls import get_role_description return get_role_description("workspace", role) or "" if is_ee(): - from ee.src.utils.permissions import check_action_access - from ee.src.models.shared_models import Permission + from ee.src.core.access.permissions.service import check_action_access + from ee.src.core.access.permissions.types import Permission from ee.src.services import db_manager_ee, workspace_manager - from ee.src.services.selectors import ( + from ee.src.services.db_manager_ee import ( get_user_org_and_workspace_id, ) from ee.src.services.organization_service import notify_org_admin_invitation - from ee.src.utils.entitlements import ( + from ee.src.core.access.entitlements.service import ( check_entitlements, scope_from, Tracker, diff --git a/api/oss/src/routers/permissions_router.py b/api/oss/src/routers/permissions_router.py deleted file mode 100644 index d2dbf40191..0000000000 --- a/api/oss/src/routers/permissions_router.py +++ /dev/null @@ -1,231 +0,0 @@ -from typing import Optional, Union -from uuid import UUID - -from fastapi.responses import JSONResponse -from fastapi import Query, HTTPException - -from oss.src.utils.logging import get_module_logger -from oss.src.utils.caching import get_cache, set_cache -from oss.src.utils.context import get_auth_context, get_auth_scope - -from oss.src.utils.common import is_ee, is_oss, APIRouter - -if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access - from ee.src.utils.entitlements import check_entitlements, Counter - - -router = APIRouter() - -log = get_module_logger(__name__) - - -class Allow(JSONResponse): - def __init__( - self, - credentials: Optional[str] = None, - ) -> None: - super().__init__( - status_code=200, - content={ - "effect": "allow", - "credentials": credentials, - }, - ) - - -class Deny(HTTPException): - def __init__(self) -> None: - super().__init__( - status_code=403, - detail="Forbidden", - ) - - -@router.get( - "/verify", - operation_id="verify_permissions", -) -async def verify_permissions( - action: Optional[str] = Query(None), - scope_type: Optional[str] = Query(None), - scope_id: Optional[UUID] = Query(None), - resource_type: Optional[str] = Query(None), - resource_id: Optional[UUID] = Query(None), -): - ctx = get_auth_context() - project_id = str(ctx.scope.project_id) - user_id = str(ctx.scope.user_id) - credentials_header = ctx.credentials.header()[1] - - cache_key = { - "action": action, - "scope_type": scope_type, - "scope_id": scope_id, - "resource_type": resource_type, - "resource_id": resource_id, - } - - try: - if is_oss(): - return Allow(credentials_header) - - if not action or not resource_type: - log.warn("Missing required parameters: action, resource_type") - raise Deny() - - allow = await get_cache( - project_id=project_id, - user_id=user_id, - namespace="verify_permissions", - key=cache_key, - ) - # allow = None - - if allow == "allow": - return Allow(credentials_header) - if allow == "deny": - log.warn("Permission denied") - raise Deny() - - # CHECK PERMISSION 1/3: SCOPE - allow_scope = await check_scope_access( - scope_type=scope_type, - scope_id=scope_id, - ) - - if not allow_scope: - log.warn("Scope access denied") - await set_cache( - project_id=project_id, - user_id=user_id, - namespace="verify_permissions", - key=cache_key, - value="deny", - ) - raise Deny() - - # CHECK PERMISSION 1/2: ACTION - allow_action = await check_action_access( - project_id=project_id, - user_uid=user_id, - permission=Permission(action), - ) - - if not allow_action: - log.warn("Action access denied") - await set_cache( - project_id=project_id, - user_id=user_id, - namespace="verify_permissions", - key=cache_key, - value="deny", - ) - raise Deny() - - # CHECK PERMISSION 3/3: RESOURCE - allow_resource = await check_resource_access( - resource_type=resource_type, - ) - - if isinstance(allow_resource, bool): - if allow_resource is False: - log.warn("Resource access denied") - await set_cache( - project_id=project_id, - user_id=user_id, - namespace="verify_permissions", - key=cache_key, - value="deny", - ) - raise Deny() - - if allow_resource is True: - await set_cache( - project_id=project_id, - user_id=user_id, - namespace="verify_permissions", - key=cache_key, - value="allow", - ) - return Allow(credentials_header) - - elif isinstance(allow_resource, int): - if allow_resource <= 0: - log.warn("Resource access denied") - await set_cache( - project_id=project_id, - user_id=user_id, - namespace="verify_permissions", - key=cache_key, - value="deny", - ) - raise Deny() - else: - return Allow(credentials_header) - - # else: - log.warn("Resource access denied") - await set_cache( - project_id=project_id, - user_id=user_id, - namespace="verify_permissions", - key=cache_key, - value="deny", - ) - raise Deny() - - except Exception as exc: # pylint: disable=bare-except - log.warn(exc) - await set_cache( - project_id=project_id, - user_id=user_id, - namespace="verify_permissions", - key=cache_key, - value="deny", - ) - raise Deny() from exc - - -async def check_scope_access( - scope_type: Optional[str] = None, - scope_id: Optional[UUID] = None, -) -> bool: - auth_scope = get_auth_scope() - - allow_scope = False - - if scope_type == "project": - allow_scope = str(auth_scope.project_id) == str(scope_id) - elif scope_type == "workspace": - allow_scope = str(auth_scope.workspace_id) == str(scope_id) - elif not scope_type and not scope_id: - allow_scope = True - - return allow_scope - - -async def check_resource_access( - resource_type: Optional[str] = None, -) -> Union[bool, int]: - allow_resource = False - - if resource_type == "service": - allow_resource = True - - if resource_type == "local_secrets": - check, meter, _ = await check_entitlements( # type: ignore - key=Counter.CREDITS_CONSUMED, # type: ignore - delta=1, - ) - - if not check: - return False - - if not meter or not meter.value: - return False - - return meter.value - - return allow_resource diff --git a/api/oss/src/routers/user_profile.py b/api/oss/src/routers/user_profile.py index 13b7b96579..4b92dd232e 100644 --- a/api/oss/src/routers/user_profile.py +++ b/api/oss/src/routers/user_profile.py @@ -11,8 +11,8 @@ if is_ee(): - from ee.src.models.shared_models import Permission - from ee.src.utils.permissions import check_action_access + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import check_action_access router = APIRouter() diff --git a/api/oss/src/routers/workspace_router.py b/api/oss/src/routers/workspace_router.py index 64a574d315..546112b28f 100644 --- a/api/oss/src/routers/workspace_router.py +++ b/api/oss/src/routers/workspace_router.py @@ -9,13 +9,13 @@ from oss.src.models.api.workspace_models import Workspace if is_ee(): - from ee.src.utils.permissions import check_rbac_permission - from ee.src.models.shared_models import WorkspaceRole - from ee.src.services.selectors import get_user_org_and_workspace_id + from ee.src.core.access.permissions.service import check_rbac_permission + from ee.src.core.access.permissions.types import RequiredRole + from ee.src.services.db_manager_ee import get_user_org_and_workspace_id from ee.src.services import db_manager_ee, workspace_manager - from ee.src.core.entitlements.controls import get_roles + from ee.src.core.access.controls import get_roles - from ee.src.utils.entitlements import ( + from ee.src.core.access.entitlements.service import ( check_entitlements, scope_from, Gauge, @@ -64,7 +64,7 @@ async def get_all_workspace_roles(request: Request) -> List[Dict[str, str]]: Returns a list of all available workspace roles. Returns: - List[WorkspaceRoleResponse]: A list of WorkspaceRole objects representing the available workspace roles. + List[WorkspaceRoleResponse]: A list of DefaultRole objects representing the available workspace roles. Raises: HTTPException: If an error occurs while retrieving the workspace roles. @@ -117,7 +117,7 @@ async def remove_user_from_workspace( has_permission = await check_rbac_permission( user_org_workspace_data=user_org_workspace_data, project_id=str(project.id), - role=WorkspaceRole.ADMIN, + role=RequiredRole.ADMIN, ) if not has_permission: return JSONResponse( diff --git a/api/oss/src/services/admin_manager.py b/api/oss/src/services/admin_manager.py index c0354302ad..20084ca908 100644 --- a/api/oss/src/services/admin_manager.py +++ b/api/oss/src/services/admin_manager.py @@ -10,7 +10,7 @@ from oss.src.utils.logging import get_module_logger -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import get_transactions_engine from oss.src.services import db_manager from oss.src.models.db_models import UserDB @@ -98,7 +98,7 @@ class ProjectRequest(BaseModel): # Role slugs are dynamic at runtime (env-overridable via AGENTA_ACCESS_ROLES); # validation against the effective per-scope catalog happens at the handler -# boundary via `ee.src.core.entitlements.controls.get_role`. +# boundary via `ee.src.core.access.controls.get_role`. class OrganizationMembershipRequest(BaseModel): role: str is_demo: bool @@ -132,7 +132,9 @@ async def legacy_create_organization( return_org_wrk: bool = False, return_org_wrk_prj: bool = False, ) -> Union[OrganizationDB, WorkspaceDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: create_org_data = payload.model_dump(exclude_unset=True) create_org_data.pop("is_demo", None) @@ -235,7 +237,9 @@ async def user_exists(user_email: str) -> bool: async def check_user( request: UserRequest, ) -> Optional[UserRequest]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(UserDB).filter_by( email=request.email, @@ -252,7 +256,9 @@ async def check_user( async def create_user( request: UserRequest, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: user_db = UserDB( # id=uuid7() # use default # @@ -279,7 +285,9 @@ async def create_organization( request: OrganizationRequest, created_by_id: uuid.UUID, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: organization_db = OrganizationDB( name=request.name, description=request.description, @@ -305,7 +313,9 @@ async def create_organization( async def create_workspace( request: WorkspaceRequest, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: workspace_db = WorkspaceDB( # id=uuid7() # use default # @@ -334,7 +344,9 @@ async def create_workspace( async def create_project( request: ProjectRequest, ) -> Reference: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: project_db = ProjectDB( # id=uuid7() # use default # diff --git a/api/oss/src/services/api_key_service.py b/api/oss/src/services/api_key_service.py index f8d78af6c7..2398d8425e 100644 --- a/api/oss/src/services/api_key_service.py +++ b/api/oss/src/services/api_key_service.py @@ -13,9 +13,7 @@ from oss.src.utils.logging import get_module_logger from oss.src.models.db_models import APIKeyDB, UserDB -from oss.src.dbs.postgres.shared.engine import engine - -# from oss.src.utils.redis_utils import redis_connection +from oss.src.dbs.postgres.shared.engine import get_transactions_engine log = get_module_logger(__name__) @@ -38,7 +36,9 @@ async def _generate_unique_prefix(): # Define the characters to use for the prefix alphabet = string.ascii_letters + string.digits - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: while True: # Generate a random 8-character prefix prefix = "".join(secrets.choice(alphabet) for _ in range(8)) @@ -82,7 +82,9 @@ async def create_api_key( # get rate limit from env rate_limit = 0 - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: # Create an APIKeyDB instance with the prefix, hashed API key, and user_id api_key = APIKeyDB( prefix=prefix, @@ -115,7 +117,9 @@ async def is_valid_api_key(key: str): - The API Key object if the API key is valid, False otherwise. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: # Check if the API key is valid (not blacklisted and not expired) result = await session.execute( select(APIKeyDB) @@ -136,37 +140,6 @@ async def is_valid_api_key(key: str): return api_key -# async def check_rate_limit(api_key_obj: APIKeyDB, cache_key: str): -# """ -# Checks if an API key has exceeded its rate limit. - -# Args: -# - key: The API key to be checked. - -# Returns: -# - True if the API key has exceeded its rate limit, False otherwise. -# """ - -# if api_key_obj.rate_limit > 0: -# # Check rate limiting in Redis -# r = redis_connection() -# if r is not None: -# api_requests_within_minute = r.get(cache_key) -# if api_requests_within_minute is None: -# # Initialize the count in Redis with an initial value of 1 and a one-minute TTL -# r.setex(cache_key, 60, 1) -# else: -# # Check if requests made within the last minute exceed the rate limit -# count_within_minute = int(api_requests_within_minute.decode("utf-8")) -# if count_within_minute > api_key_obj.rate_limit: -# return True - -# # increment the apikey usage count in redis -# r.incr(cache_key) - -# return False - - async def use_api_key(key: str) -> Union[APIKeyDB, bool]: """ Validates and checks the rate limit of an API key. @@ -234,7 +207,9 @@ async def list_api_keys(user_id: str, project_id: str) -> List[APIKeyDB]: List[APIKeyDB]: A list of APIKeyDB objects associated with the user, sorted by most recently created first. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(APIKeyDB) .filter_by( @@ -260,7 +235,9 @@ async def delete_api_key(user_id: str, key_prefix: str): KeyError: If the API key does not exist or does not belong to the user. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(APIKeyDB).filter_by( created_by_id=uuid.UUID(user_id), prefix=key_prefix diff --git a/api/oss/src/services/db_manager.py b/api/oss/src/services/db_manager.py index e4ba740a80..98f9661fd4 100644 --- a/api/oss/src/services/db_manager.py +++ b/api/oss/src/services/db_manager.py @@ -17,10 +17,13 @@ from supertokens_python.asyncio import delete_user as delete_user_from_supertokens from oss.src.utils.logging import get_module_logger -from oss.src.services import user_service, analytics_service +from oss.src.services import user_service +from oss.src.middlewares import analytics from oss.src.utils.common import is_ee from oss.src.utils.env import env -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import ( + get_transactions_engine, +) from oss.src.utils.helpers import get_slug_from_name_and_id from oss.src.dbs.postgres.blobs.dao import BlobsDAO @@ -64,7 +67,8 @@ async def fetch_project_by_id( project_id: str, ) -> Optional[ProjectDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + async with engine.session() as session: project = ( ( await session.execute( @@ -91,7 +95,8 @@ async def fetch_projects_by_workspace( List[ProjectDB]: Projects scoped to the workspace. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + async with engine.session() as session: result = await session.execute( select(ProjectDB) .filter(ProjectDB.workspace_id == uuid.UUID(workspace_id)) @@ -103,7 +108,8 @@ async def fetch_projects_by_workspace( async def fetch_workspace_by_id( workspace_id: str, ) -> Optional[WorkspaceDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + async with engine.session() as session: workspace = ( ( await session.execute( @@ -122,7 +128,9 @@ async def fetch_workspace_by_id( async def fetch_organization_by_id( organization_id: str, ) -> Optional[OrganizationDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: organization = ( ( await session.execute( @@ -311,7 +319,9 @@ async def get_user(user_uid: str) -> UserDB: UserDB: instance of user """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: # NOTE: Backward Compatibility # --------------------------- # Previously, the user_id field in the api_keys collection in MongoDB used the @@ -331,7 +341,9 @@ async def get_user(user_uid: str) -> UserDB: async def is_first_user_signup() -> bool: """Check if this is the first user signing up (no users exist yet).""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: total_users = ( await session.scalar(select(func.count()).select_from(UserDB)) or 0 ) @@ -401,7 +413,9 @@ async def setup_oss_organization_for_first_user( # org with SELECT ... FOR UPDATE inside a single transaction — the # second caller blocks until the first commits, then sees the # workspace and skips the insert. No schema change required. - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: await session.execute( select(OrganizationDB.id).filter_by(id=organization_db.id).with_for_update() ) @@ -451,7 +465,7 @@ async def setup_oss_organization_for_first_user( user_id=user_id, ) - analytics_service.capture_oss_deployment_created( + analytics.capture_oss_deployment_created( user_email=user_email, organization_id=str(organization_db.id), ) @@ -462,7 +476,9 @@ async def setup_oss_organization_for_first_user( async def check_if_user_invitation_exists(email: str, organization_id: str): """Check if a user invitation with the given email and organization_id exists.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB) .join(ProjectDB, InvitationDB.project_id == ProjectDB.id) @@ -593,7 +609,9 @@ async def get_default_workspace_id_oss() -> str: orgs) cannot shadow the real singleton workspace and steer auth scope resolution to the wrong tenant. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(WorkspaceDB) .join(OrganizationDB, WorkspaceDB.organization_id == OrganizationDB.id) @@ -635,7 +653,9 @@ async def create_organization( EE keeps the previous behavior (one org per signup, slug left NULL). """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: # For bootstrap scenario, use a placeholder UUID if not provided _owner_id = owner_id or uuid.uuid4() _created_by_id = created_by_id or _owner_id @@ -705,7 +725,9 @@ async def create_workspace(name: str, organization_id: str): WorkspaceDB: instance of workspace """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: workspace_db = WorkspaceDB( name=name, organization_id=uuid.UUID(organization_id), @@ -735,7 +757,9 @@ async def update_organization(organization_id: str, values_to_update: Dict[str, values_to_update (Dict[str, Any]): The values to update in the organization """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(OrganizationDB).filter_by(id=uuid.UUID(organization_id)) ) @@ -773,7 +797,9 @@ async def create_or_update_default_project(values_to_update: Dict[str, Any]): "create_or_update_default_project requires 'organization_id' in values_to_update" ) - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(ProjectDB).filter_by( organization_id=organization_id, @@ -803,7 +829,9 @@ async def get_organizations() -> List[OrganizationDB]: List: A list of organizations. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(OrganizationDB)) organizations = result.scalars().all() return organizations @@ -820,7 +848,9 @@ async def get_organization_by_id(organization_id: str) -> OrganizationDB: OrganizationDB: The organization object if found, None otherwise. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(OrganizationDB).filter_by(id=uuid.UUID(organization_id)) ) @@ -839,7 +869,9 @@ async def get_organization_by_slug(organization_slug: str) -> OrganizationDB: OrganizationDB: The organization object if found, None otherwise. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(OrganizationDB).filter_by(slug=organization_slug) ) @@ -858,7 +890,9 @@ async def get_organization_owner(organization_id: str): UserDB: The owner of the organization if found, None otherwise. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(OrganizationDB).filter_by(id=uuid.UUID(organization_id)) ) @@ -883,7 +917,9 @@ async def get_user_organizations(user_id: str) -> List[OrganizationDB]: if is_ee(): from ee.src.models.db_models import OrganizationMemberDB - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: # Query organizations through organization_members table result = await session.execute( select(OrganizationDB) @@ -912,7 +948,9 @@ async def get_workspace(workspace_id: str) -> WorkspaceDB: Workspace: The retrieved workspace. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: query = select(WorkspaceDB).filter_by(id=uuid.UUID(workspace_id)) result = await session.execute(query) @@ -928,7 +966,9 @@ async def get_workspaces() -> List[WorkspaceDB]: List: A list of workspaces. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(WorkspaceDB)) workspaces = result.scalars().all() return workspaces @@ -954,7 +994,9 @@ async def remove_user_from_workspace(project_id: str, email: str): if not project: raise NoResultFound(f"Project with ID {project_id} not found") - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: if user: await session.delete(user) @@ -1000,7 +1042,9 @@ async def get_user_with_id(user_id: str) -> UserDB: Exception: If an error occurs while getting the user from the database. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(UserDB).filter_by(id=uuid.UUID(user_id))) user = result.scalars().first() if user is None: @@ -1012,7 +1056,9 @@ async def get_user_with_id(user_id: str) -> UserDB: async def update_user_username(user_id: str, username: str) -> UserDB: """Update a user's username.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(UserDB).filter_by(id=uuid.UUID(user_id))) user = result.scalars().first() if user is None: @@ -1047,7 +1093,9 @@ async def get_user_with_email(email: str): if "@" not in email: raise Exception("Please provide a valid email address") - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(UserDB).filter_by(email=email)) user = result.scalars().first() return user @@ -1085,7 +1133,9 @@ async def create_user_invitation_to_organization( if not project: raise NoResultFound(f"Project with ID {project_id} not found") - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: invitation = InvitationDB( token=token, email=email, @@ -1121,7 +1171,9 @@ async def get_project_by_id(project_id: str) -> ProjectDB: str: The retrieve project or None """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: project_query = await session.execute( select(ProjectDB) .options(joinedload(ProjectDB.organization).load_only(OrganizationDB.name)) @@ -1145,7 +1197,9 @@ async def get_default_project_id_from_workspace( Union[str, Exception]: The default project ID or an exception error message. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: project_query = await session.execute( select(ProjectDB) .where( @@ -1243,7 +1297,8 @@ async def _create( if session is not None: return await _create(session) - async with engine.core_session() as new_session: + engine = get_transactions_engine() + async with engine.session() as new_session: return await _create(new_session) @@ -1255,7 +1310,9 @@ async def delete_project(project_id: str) -> None: project_id (str): Identifier of project to delete. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: project = await session.get(ProjectDB, uuid.UUID(project_id)) if project is None: raise NoResultFound(f"Project with ID {project_id} not found") @@ -1286,7 +1343,9 @@ async def set_default_project(project_id: str) -> ProjectDB: ProjectDB: Updated project. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: project = await session.get(ProjectDB, uuid.UUID(project_id)) if project is None: raise NoResultFound(f"Project with ID {project_id} not found") @@ -1329,7 +1388,9 @@ async def update_project_name(project_id: str, *, project_name: str) -> ProjectD if not project_name.strip(): raise ValueError("Project name cannot be empty") - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: project = await session.get(ProjectDB, uuid.UUID(project_id)) if project is None: raise NoResultFound(f"Project with ID {project_id} not found") @@ -1360,7 +1421,9 @@ async def get_project_invitation_by_email(project_id: str, email: str) -> Invita InvitationDB: invitation object """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB).filter_by( project_id=uuid.UUID(project_id), email=email @@ -1376,7 +1439,9 @@ async def get_project_invitation_by_organization_and_email( ) -> Optional[InvitationDB]: """Get an invitation by organization and email, regardless of project.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB) .join(ProjectDB, InvitationDB.project_id == ProjectDB.id) @@ -1399,7 +1464,9 @@ async def get_project_invitations(project_id: str) -> InvitationDB: List[InvitationDB]: invitation objects """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB).filter_by(project_id=uuid.UUID(project_id)) ) @@ -1419,7 +1486,9 @@ async def update_invitation(invitation_id: str, values_to_update: dict) -> bool: bool: True if the invitation was successfully updated, False otherwise. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB).filter_by(id=uuid.UUID(invitation_id)) ) @@ -1460,7 +1529,9 @@ async def delete_invitation(invitation_id: str) -> bool: bool: True if the invitation was successfully deleted, False otherwise. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB).filter_by(id=uuid.UUID(invitation_id)) ) @@ -1512,7 +1583,9 @@ async def get_project_by_organization_id(organization_id: str): ProjectDB: project object """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(ProjectDB).filter_by(organization_id=uuid.UUID(organization_id)) ) @@ -1527,8 +1600,9 @@ async def get_default_project_by_organization_id(organization_id: str): so callers that depend on the OSS singleton invariant don't accidentally pick up an ephemeral per-account project. """ + engine = get_transactions_engine() - async with engine.core_session() as session: + async with engine.session() as session: result = await session.execute( select(ProjectDB).filter_by( organization_id=uuid.UUID(organization_id), @@ -1552,7 +1626,9 @@ async def get_project_invitation_by_token_and_email( InvitationDB: invitation object """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB).filter_by( project_id=uuid.UUID(project_id), token=token, email=email @@ -1569,7 +1645,9 @@ async def get_project_invitation_by_organization_token_and_email( ) -> Optional[InvitationDB]: """Get an invitation by organization, token, and email, regardless of project.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(InvitationDB) .join(ProjectDB, InvitationDB.project_id == ProjectDB.id) @@ -1596,7 +1674,9 @@ async def get_user_api_key_by_prefix( The user api key by prefix. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(APIKeyDB).filter_by( prefix=api_key_prefix, created_by_id=uuid.UUID(user_id) @@ -1612,56 +1692,74 @@ async def get_user_api_key_by_prefix( async def admin_get_user_by_id(user_id: uuid.UUID) -> Optional[UserDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(UserDB).filter_by(id=user_id)) return result.scalars().first() async def admin_get_user_by_email(email: str) -> Optional[UserDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(UserDB).filter_by(email=email)) return result.scalars().first() async def admin_get_org_by_id(org_id: uuid.UUID) -> Optional[OrganizationDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(OrganizationDB).filter_by(id=org_id)) return result.scalars().first() async def admin_get_org_by_slug(slug: str) -> Optional[OrganizationDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(OrganizationDB).filter_by(slug=slug)) return result.scalars().first() async def admin_get_workspace_by_id(ws_id: uuid.UUID) -> Optional[WorkspaceDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(WorkspaceDB).filter_by(id=ws_id)) return result.scalars().first() async def admin_get_project_by_id(proj_id: uuid.UUID) -> Optional[ProjectDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(ProjectDB).filter_by(id=proj_id)) return result.scalars().first() async def admin_get_api_key_by_id(key_id: uuid.UUID) -> Optional[APIKeyDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(APIKeyDB).filter_by(id=key_id)) return result.scalars().first() async def admin_get_api_key_by_prefix(prefix: str) -> Optional[APIKeyDB]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(APIKeyDB).filter_by(prefix=prefix)) return result.scalars().first() async def admin_get_orgs_owned_by_user(user_id: uuid.UUID) -> List[OrganizationDB]: """Return orgs where user is owner OR creator (both carry RESTRICT FK).""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(OrganizationDB).where( or_( @@ -1676,7 +1774,9 @@ async def admin_get_orgs_owned_by_user(user_id: uuid.UUID) -> List[OrganizationD async def admin_get_workspace_ids_for_orgs( org_ids: List[uuid.UUID], ) -> List[uuid.UUID]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(WorkspaceDB.id).where(WorkspaceDB.organization_id.in_(org_ids)) ) @@ -1686,7 +1786,9 @@ async def admin_get_workspace_ids_for_orgs( async def admin_get_project_ids_for_orgs( org_ids: List[uuid.UUID], ) -> List[uuid.UUID]: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute( select(ProjectDB.id).where(ProjectDB.organization_id.in_(org_ids)) ) @@ -1700,7 +1802,9 @@ async def admin_get_or_create_user( existing = await admin_get_user_by_email(email) if existing: return existing - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: user_db = UserDB( uid=str(uuid.uuid4()), username=username or email.split("@")[0], @@ -1732,7 +1836,9 @@ async def admin_create_organization( On EE behavior is unchanged: a new row is inserted with the supplied ``name``/``slug``. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: if not is_ee(): stmt = ( pg_insert(OrganizationDB) @@ -1788,7 +1894,9 @@ async def admin_create_workspace( On EE behavior is unchanged: a fresh workspace row is always inserted. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: if not is_ee(): await session.execute( select(OrganizationDB.id).filter_by(id=org_id).with_for_update() @@ -1834,7 +1942,9 @@ async def admin_create_project( *, is_default: bool = False, ) -> ProjectDB: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: proj_db = ProjectDB( project_name=name, is_default=is_default, @@ -1849,31 +1959,41 @@ async def admin_create_project( async def admin_delete_user(user_id: uuid.UUID) -> None: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: await session.execute(delete(UserDB).where(UserDB.id == user_id)) await session.commit() async def admin_delete_organization(org_id: uuid.UUID) -> None: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: await session.execute(delete(OrganizationDB).where(OrganizationDB.id == org_id)) await session.commit() async def admin_delete_workspace(ws_id: uuid.UUID) -> None: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: await session.execute(delete(WorkspaceDB).where(WorkspaceDB.id == ws_id)) await session.commit() async def admin_delete_project(proj_id: uuid.UUID) -> None: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: await session.execute(delete(ProjectDB).where(ProjectDB.id == proj_id)) await session.commit() async def admin_delete_api_key(key_id: uuid.UUID) -> None: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: await session.execute(delete(APIKeyDB).where(APIKeyDB.id == key_id)) await session.commit() @@ -1886,7 +2006,9 @@ async def admin_delete_accounts_batch( user_ids: List[uuid.UUID], ) -> None: """Delete a batch of entities atomically, in dependency order.""" - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: for proj_id in project_ids: await session.execute(delete(ProjectDB).where(ProjectDB.id == proj_id)) for ws_id in workspace_ids: @@ -1928,7 +2050,9 @@ async def admin_transfer_org_ownership_batch( of that user does not destroy orgs now owned by the target. """ now = datetime.now(timezone.utc) - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: for org_id in org_ids: await session.execute( update(OrganizationDB) diff --git a/api/oss/src/services/email_service.py b/api/oss/src/services/email_service.py deleted file mode 100644 index 8ad0540996..0000000000 --- a/api/oss/src/services/email_service.py +++ /dev/null @@ -1,73 +0,0 @@ -import os - -import sendgrid -from sendgrid.helpers.mail import Mail - -from fastapi import HTTPException - -from oss.src.utils.env import env -from oss.src.utils.logging import get_logger - -log = get_logger(__name__) - -# Initialize SendGrid only if enabled -if env.sendgrid.enabled: - sg = sendgrid.SendGridAPIClient(api_key=env.sendgrid.api_key) - log.info("✓ SendGrid enabled") -else: - sg = None - if env.sendgrid.api_key and not env.sendgrid.from_address: - log.warn("✗ SendGrid disabled: missing sender email address") - else: - log.warn("✗ SendGrid disabled") - - -def read_email_template(template_file_path): - """ - Function to read the HTML template from the file - """ - - # Get the absolute path to the template file - script_directory = os.path.dirname(os.path.abspath(__file__)) - absolute_template_file_path = os.path.join(script_directory, template_file_path) - - with open(absolute_template_file_path, "r") as template_file: - return template_file.read() - - -async def send_email( - to_email: str, subject: str, html_content: str, from_email: str -) -> bool: - """ - Send an email to a user. - - Args: - to_email (str): The email address to send the email to. - subject (str): The subject of the email. - html_content (str): The HTML content of the email. - from_email (str): The email address to send the email from. - - Returns: - bool: True if the email was sent successfully, False otherwise. - - Raises: - HTTPException: If there is an error sending the email. - """ - - # No-op if SendGrid is disabled - if not env.sendgrid.enabled: - log.info(f"[SENDGRID] Email disabled - would send '{subject}' to {to_email}") - return True - - message = Mail( - from_email=from_email, - to_emails=to_email, - subject=subject, - html_content=html_content, - ) - - try: - sg.send(message) - return True - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) diff --git a/api/oss/src/services/exceptions.py b/api/oss/src/services/exceptions.py deleted file mode 100644 index 37c62f1292..0000000000 --- a/api/oss/src/services/exceptions.py +++ /dev/null @@ -1,94 +0,0 @@ -from http import HTTPStatus - -from fastapi import HTTPException - - -def code_to_phrase(status_code: int) -> str: - try: - return HTTPStatus(status_code).phrase - except ValueError: - return "Unknown Status Code" - - -class BadRequestException(HTTPException): - def __init__( - self, - code: int = 400, - detail: str = "Bad Request", - ): - self.code = code - self.detail = detail - - super().__init__(self.code, self.detail) - - -class UnauthorizedException(HTTPException): - def __init__( - self, - code: int = 401, - detail: str = "Unauthorized", - ): - self.code = code - self.detail = detail - - super().__init__(self.code, self.detail) - - -class ForbiddenException(HTTPException): - def __init__( - self, - code: int = 403, - detail: str = "Fordidden", - ): - self.code = code - self.detail = detail - - super().__init__(self.code, self.detail) - - -class UnprocessableContentException(HTTPException): - def __init__( - self, - code: int = 422, - detail: str = "Unprocessable Content", - ): - self.code = code - self.detail = detail - - super().__init__(self.code, self.detail) - - -class TooManyRequestsException(HTTPException): - def __init__( - self, - code: int = 429, - detail: str = "Too Many Requests", - ): - self.code = code - self.detail = detail - - super().__init__(self.code, self.detail) - - -class InternalServerErrorException(HTTPException): - def __init__( - self, - code: int = 500, - detail: str = "Internal Server Error", - ): - self.code = code - self.detail = detail - - super().__init__(self.code, self.detail) - - -class GatewayTimeoutException(HTTPException): - def __init__( - self, - code: int = 504, - detail: str = "Gateway Timeout", - ): - self.code = code - self.detail = detail - - super().__init__(self.code, self.detail) diff --git a/api/oss/src/services/llm_apps_service.py b/api/oss/src/services/llm_apps_service.py deleted file mode 100644 index f898ab3b4b..0000000000 --- a/api/oss/src/services/llm_apps_service.py +++ /dev/null @@ -1,905 +0,0 @@ -import json -import asyncio -import shlex -import traceback -import aiohttp -from datetime import datetime -from typing import Any, Dict, List, Optional - -from agenta.sdk.models.shared import Reference -from oss.src.utils.logging import get_module_logger -from oss.src.services.auth_service import sign_secret_token -from oss.src.services.db_manager import get_project_by_id -from oss.src.models.shared_models import InvokationResult, Result, Error - -log = get_module_logger(__name__) - - -def get_nested_value(d: dict, keys: list, default=None): - """ - Helper function to safely retrieve nested values. - """ - try: - for key in keys: - if isinstance(d, dict): - d = d.get(key, default) - else: - return default - return d - except Exception as e: - log.error(f"Error accessing nested value: {e}") - return default - - -def extract_result_from_response(response: dict): - # Initialize default values - value = None - latency = None - cost = None - tokens = None - - try: - # Validate input - if not isinstance(response, dict): - raise ValueError("The response must be a dictionary.") - - # Handle version 3.0 response - if response.get("version") == "3.0": - value = response - # Ensure 'data' is a dictionary or convert it to a string - if not isinstance(value.get("data"), dict): - value["data"] = str(value.get("data")) - - if "tree" in response: - trace_tree = response.get("tree", {}).get("nodes", [])[0] - - duration_ms = get_nested_value( - trace_tree, ["metrics", "acc", "duration", "total"] - ) - if duration_ms: - duration_seconds = duration_ms / 1000 - else: - start_time = get_nested_value(trace_tree, ["time", "start"]) - end_time = get_nested_value(trace_tree, ["time", "end"]) - - if start_time and end_time: - duration_seconds = ( - datetime.fromisoformat(end_time) - - datetime.fromisoformat(start_time) - ).total_seconds() - else: - duration_seconds = None - - latency = duration_seconds - cost = get_nested_value( - trace_tree, ["metrics", "acc", "costs", "total"] - ) - tokens = get_nested_value( - trace_tree, ["metrics", "acc", "tokens", "total"] - ) - - # Handle version 2.0 response - elif response.get("version") == "2.0": - value = response - if not isinstance(value.get("data"), dict): - value["data"] = str(value.get("data")) - - if "trace" in response: - latency = response["trace"].get("latency", None) - cost = response["trace"].get("cost", None) - tokens = response["trace"].get("tokens", None) - - # Handle generic response (neither 2.0 nor 3.0) - else: - value = {"data": str(response.get("message", ""))} - latency = response.get("latency", None) - cost = response.get("cost", None) - tokens = response.get("tokens", None) - - # Determine the type of 'value' (either 'text' or 'object') - kind = "text" if isinstance(value, str) else "object" - - except ValueError as ve: - log.error(f"Input validation error: {ve}") - value = {"error": str(ve)} - kind = "error" - - except KeyError as ke: - log.error(f"Missing key: {ke}") - value = {"error": f"Missing key: {ke}"} - kind = "error" - - except TypeError as te: - log.error(f"Type error: {te}") - value = {"error": f"Type error: {te}"} - kind = "error" - - except Exception as e: - log.error(f"Unexpected error: {e}") - value = {"error": f"Unexpected error: {e}"} - kind = "error" - - return value, kind, cost, tokens, latency - - -def _parse_legacy_chat_messages(datapoint: Any) -> list[Any]: - # Legacy rows may store chat history under either `messages` or `chat`. - raw_messages = datapoint.get("messages") or datapoint.get("chat", "[]") - - if isinstance(raw_messages, list): - return raw_messages - - if isinstance(raw_messages, str): - try: - return json.loads(raw_messages) if raw_messages else [] - except (json.JSONDecodeError, TypeError): - log.warn(f"Failed to parse messages data, using empty list: {raw_messages}") - return [] - - log.warn(f"Unexpected format for messages data, using empty list: {raw_messages}") - return [] - - -def _extract_input_keys(parameters: Any) -> List[str]: - input_keys: List[str] = [] - - def visit(value: Any) -> None: - if isinstance(value, dict): - for key, nested_value in value.items(): - if key == "input_keys" and isinstance(nested_value, list): - for item in nested_value: - if isinstance(item, str) and item not in input_keys: - input_keys.append(item) - continue - visit(nested_value) - return - - if isinstance(value, list): - for item in value: - visit(item) - - visit(parameters) - return input_keys - - -def _extract_input_names_from_schemas(schemas: Any) -> List[str]: - if hasattr(schemas, "model_dump"): - schemas = schemas.model_dump(mode="json", exclude_none=True) - - if not isinstance(schemas, dict): - return [] - - inputs_schema = schemas.get("inputs") or {} - if not isinstance(inputs_schema, dict): - return [] - - properties = inputs_schema.get("properties") or {} - if not isinstance(properties, dict): - return [] - - return [key for key in properties.keys() if key != "messages"] - - -def parse_legacy_inputs( - datapoint: Any, - parameters: Any, - schemas: Any = None, - is_chat: Optional[bool] = None, -) -> Dict: - if not isinstance(datapoint, dict): - return {} - - datapoint_keys = list(datapoint.keys()) - input_keys = _extract_input_keys(parameters) - if input_keys: - inputs = {key: datapoint.get(key, None) for key in input_keys} - log.debug( - "parse_legacy_inputs: filtered testcase row using parameter input_keys", - input_keys=input_keys, - datapoint_keys=datapoint_keys, - ) - else: - schema_input_names = _extract_input_names_from_schemas(schemas) - if schema_input_names: - inputs = {key: datapoint.get(key, None) for key in schema_input_names} - log.debug( - "parse_legacy_inputs: filtered testcase row using input schema properties", - schema_input_names=schema_input_names, - datapoint_keys=datapoint_keys, - ) - else: - inputs = dict(datapoint) - log.warning( - "parse_legacy_inputs: falling back to full testcase row because no input_keys or input schema properties were available", - datapoint_keys=datapoint_keys, - ) - - if is_chat: - inputs["messages"] = _parse_legacy_chat_messages(datapoint) - - return inputs - - -def _format_curl_request( - *, - url: str, - headers: Dict[str, str], - json_body: Dict[str, Any], -) -> str: - # Keep any future debug curl output safe by redacting sensitive headers. - redacted_headers = { - key: "[REDACTED]" - if key.lower() in {"authorization", "proxy-authorization", "cookie"} - else value - for key, value in headers.items() - } - parts = ["curl", "-X", "POST", shlex.quote(url)] - - for key, value in redacted_headers.items(): - parts.extend(["-H", shlex.quote(f"{key}: {value}")]) - - parts.extend( - [ - "--data-raw", - shlex.quote(json.dumps(json_body, ensure_ascii=False)), - ] - ) - - return " ".join(parts) - - -def _deep_serialize(obj: Any) -> Any: - """Recursively serialize objects to JSON-compatible types.""" - from uuid import UUID - - if isinstance(obj, UUID): - return str(obj) - elif isinstance(obj, dict): - return {k: _deep_serialize(v) for k, v in obj.items()} - elif isinstance(obj, (list, tuple)): - return [_deep_serialize(item) for item in obj] - elif isinstance(obj, Reference): - return obj.model_dump(exclude_none=True) - return obj - - -def build_invoke_request( - *, - inputs: Dict[str, Any], - parameters: Dict[str, Any], - references: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - request = { - "data": { - "parameters": parameters, - "inputs": inputs, - } - } - - if references: - request["references"] = _deep_serialize(references) - - return request - - -def _extract_error_details( - app_response: Any, - *, - fallback_message: str, - fallback_stacktrace: str, -) -> tuple[str, str]: - if isinstance(app_response, dict): - detail = app_response.get("detail") - if isinstance(detail, dict): - return ( - detail.get("error", fallback_message), - detail.get("message") or detail.get("traceback") or fallback_stacktrace, - ) - if isinstance(detail, str): - return detail, fallback_stacktrace - - status = app_response.get("status") - if isinstance(status, dict): - return ( - status.get("message", fallback_message), - status.get("stacktrace") or fallback_stacktrace, - ) - - if isinstance(app_response, str) and app_response: - return app_response, fallback_stacktrace - - return fallback_message, fallback_stacktrace - - -async def invoke_app( - uri: str, - datapoint: Any, - parameters: Dict, - schemas: Optional[Dict[str, Any]], - openapi_is_chat: Optional[bool], - user_id: str, - project_id: str, - scenario_id: Optional[str] = None, - **kwargs, -) -> InvokationResult: - """ - Invoke an application for one testcase row. - - Args: - uri (str): The URI of the app to invoke. - datapoint (Any): The testcase row data to send as `data.inputs`. - parameters (Dict): The application parameters to send as `data.parameters`. - - Returns: - InvokationResult: The output of the app. - - Raises: - aiohttp.ClientError: If the POST request fails. - """ - - url = f"{uri}/invoke" - - inputs = parse_legacy_inputs( - datapoint, - parameters, - schemas=schemas, - is_chat=openapi_is_chat, - ) - request_body = build_invoke_request( - inputs=inputs, - parameters=parameters, - references=kwargs.get("references"), - ) - - project = await get_project_by_id( - project_id=project_id, - ) - - secret_token = await sign_secret_token( - user_id=str(user_id), - project_id=str(project_id), - workspace_id=str(project.workspace_id), - organization_id=str(project.organization_id), - ) - - headers = {} - if secret_token: - headers = {"Authorization": f"Secret {secret_token}"} - headers["ngrok-skip-browser-warning"] = "1" - - async with aiohttp.ClientSession() as client: - app_response = {} - - try: - log.info( - "Invoking application...", - scenario_id=scenario_id, - testcase_id=( - datapoint["testcase_id"] if "testcase_id" in datapoint else None - ), - url=url, - ) - response = await client.post( - url, - json=request_body, - headers=headers, - timeout=900, - ) - app_response = await response.json() - response.raise_for_status() - - ( - value, - kind, - cost, - tokens, - latency, - ) = extract_result_from_response(app_response) - - trace_id = app_response.get("trace_id", None) - span_id = app_response.get("span_id", None) - - log.info( - "Invoked application. ", - scenario_id=scenario_id, - trace_id=trace_id, - ) - - return InvokationResult( - result=Result( - type=kind, - value=value, - error=None, - ), - latency=latency, - cost=cost, - tokens=tokens, - trace_id=trace_id, - span_id=span_id, - ) - - except aiohttp.ClientResponseError as e: - log.error( - "Application request failed", - scenario_id=scenario_id, - testcase_id=( - datapoint["testcase_id"] if "testcase_id" in datapoint else None - ), - url=url, - status_code=e.status, - ) - error_message, stacktrace = _extract_error_details( - app_response, - fallback_message=f"HTTP error {e.status}: {e.message}", - fallback_stacktrace="".join( - traceback.format_exception_only(type(e), e) - ), - ) - log.error(f"HTTP error occurred during request: {error_message}") - except aiohttp.ServerTimeoutError as e: - error_message = "Request timed out" - stacktrace = "".join(traceback.format_exception_only(type(e), e)) - log.error(error_message) - except aiohttp.ClientConnectionError as e: - error_message = f"Connection error: {str(e)}" - stacktrace = "".join(traceback.format_exception_only(type(e), e)) - log.error(error_message) - except json.JSONDecodeError as e: - error_message = "Failed to decode JSON from response" - stacktrace = "".join(traceback.format_exception_only(type(e), e)) - log.error(error_message) - except Exception as e: - error_message = f"Unexpected error: {str(e)}" - stacktrace = "".join(traceback.format_exception_only(type(e), e)) - log.error(error_message) - - return InvokationResult( - result=Result( - type="error", - error=Error( - message=error_message, - stacktrace=stacktrace, - ), - ) - ) - - -async def run_with_retry( - uri: str, - input_data: Any, - parameters: Dict, - schemas: Optional[Dict[str, Any]], - max_retry_count: int, - retry_delay: int, - openapi_is_chat: Optional[bool], - user_id: str, - project_id: str, - scenario_id: Optional[str] = None, - **kwargs, -) -> InvokationResult: - """ - Runs the specified app with retry mechanism. - - Args: - uri (str): The URI of the app. - input_data (Any): The input data for the app. - parameters (Dict): The parameters for the app. - max_retry_count (int): The maximum number of retries. - retry_delay (int): The delay between retries in seconds. - openapi_is_chat (Optional[bool]): Whether the app is chat, if detected. - - Returns: - InvokationResult: The invokation result. - - """ - - if "references" in kwargs and "testcase_id" in input_data: - # Ensure references dict is fully serialized before modifying - references = kwargs["references"] - if isinstance(references, dict): - references = { - k: (v.model_dump(exclude_none=True) if isinstance(v, Reference) else v) - for k, v in references.items() - } - testcase_id = input_data["testcase_id"] - references["testcase"] = {"id": str(testcase_id) if testcase_id else None} - kwargs["references"] = references - - # references = kwargs.get("references", None) - # links = kwargs.get("links", None) - # hash_id = make_hash_id(references=references, links=links) - - retries = 0 - last_exception = None - while retries < max_retry_count: - try: - result = await invoke_app( - uri, - input_data, - parameters, - schemas, - openapi_is_chat, - user_id, - project_id, - scenario_id, - **kwargs, - ) - return result - except aiohttp.ClientError as e: - last_exception = e - log.error(f"Error in evaluation. Retrying in {retry_delay} seconds:", e) - await asyncio.sleep(retry_delay) - retries += 1 - except Exception as e: - last_exception = e - log.warn( - f"Error processing datapoint: {input_data}.", - exc_info=True, - ) - log.warn("".join(traceback.format_exception_only(type(e), e))) - retries += 1 - - # If max retries is reached or an exception that isn't in the second block, - # update & return the last exception - log.warn("Max retries reached") - exception_message = ( - "Max retries reached" - if retries == max_retry_count - else f"Error processing {input_data} datapoint" - ) - - return InvokationResult( - result=Result( - type="error", - value=None, - error=Error(message=exception_message, stacktrace=str(last_exception)), - ) - ) - - -async def batch_invoke( - uri: str, - testset_data: List[Dict], - *, - rate_limit_config: Dict, - user_id: str, - project_id: str, - parameters: Optional[Dict] = None, - scenarios: Optional[List[Dict]] = None, - revision: Optional[Any] = None, - schemas: Optional[Dict[str, Any]] = None, - is_chat: Optional[bool] = None, - **kwargs, -) -> List[InvokationResult]: - """ - Invokes the LLm apps in batches, processing the testset data. - - Args: - uri (str): The URI of the LLm app. - testset_data (List[Dict]): The testset data to be processed. - parameters (Dict): The parameters for the LLm app. - rate_limit_config (Dict): The rate limit configuration. - - Returns: - List[InvokationResult]: The list of app outputs after running all batches. - """ - ( - effective_parameters, - effective_schemas, - effective_is_chat, - ) = _extract_batch_invoke_metadata( - revision=revision, - parameters=parameters, - schemas=schemas, - is_chat=is_chat, - ) - log.debug( - "batch_invoke: resolved invocation metadata", - parameter_keys=( - list(effective_parameters.keys()) - if isinstance(effective_parameters, dict) - else None - ), - schema_input_keys=_extract_input_names_from_schemas(effective_schemas), - is_chat=effective_is_chat, - batch_size=len(testset_data), - ) - - batch_size = rate_limit_config[ - "batch_size" - ] # Number of testset to make in each batch - max_retries = rate_limit_config[ - "max_retries" - ] # Maximum number of times to retry the failed llm call - retry_delay = rate_limit_config[ - "retry_delay" - ] # Delay before retrying the failed llm call (in seconds) - delay_between_batches = rate_limit_config[ - "delay_between_batches" - ] # Delay between batches (in seconds) - - list_of_app_outputs: List[ - InvokationResult - ] = [] # Outputs after running all batches - - project = await get_project_by_id( - project_id=project_id, - ) - - secret_token = await sign_secret_token( - user_id=str(user_id), - project_id=str(project_id), - workspace_id=str(project.workspace_id), - organization_id=str(project.organization_id), - ) - - headers = {} - if secret_token: - headers = {"Authorization": f"Secret {secret_token}"} - headers["ngrok-skip-browser-warning"] = "1" - - schema_parameters, openapi_is_chat = get_parameters_from_schemas( - schemas=effective_schemas, - is_chat=effective_is_chat, - ) - - if not schema_parameters: - max_recursive_depth = 5 - runtime_prefix = uri - route_path = "" - - while max_recursive_depth > 0 and not schema_parameters: - try: - ( - schema_parameters, - openapi_is_chat, - ) = await get_parameters_from_inspect( - runtime_prefix, - route_path, - headers, - ) - except Exception: # pylint: disable=broad-exception-caught - schema_parameters = None - openapi_is_chat = None - - if not schema_parameters: - max_recursive_depth -= 1 - if not runtime_prefix.endswith("/"): - route_path = "/" + runtime_prefix.split("/")[-1] + route_path - runtime_prefix = "/".join(runtime_prefix.split("/")[:-1]) - else: - route_path = "" - runtime_prefix = runtime_prefix[:-1] - - if not schema_parameters: - schema_parameters, openapi_is_chat = await get_parameters_from_inspect( - runtime_prefix, - route_path, - headers, - ) - - # 🆕 Rewritten loop instead of recursion - for start_idx in range(0, len(testset_data), batch_size): - tasks = [] - - end_idx = min(start_idx + batch_size, len(testset_data)) - for index in range(start_idx, end_idx): - task = asyncio.ensure_future( - run_with_retry( - uri, - testset_data[index], - effective_parameters, - effective_schemas, - max_retries, - retry_delay, - openapi_is_chat, - user_id, - project_id, - scenarios[index].get("id") if scenarios else None, - **kwargs, - ) - ) - tasks.append(task) - - results = await asyncio.gather(*tasks) - - for result in results: - list_of_app_outputs.append(result) - - # Delay between batches if more to come - if end_idx < len(testset_data): - await asyncio.sleep(delay_between_batches) - - return list_of_app_outputs - - -def _to_json_dict(value: Any) -> Dict[str, Any]: - if hasattr(value, "model_dump"): - value = value.model_dump(mode="json", exclude_none=True) - - return value if isinstance(value, dict) else {} - - -def _extract_batch_invoke_metadata( - *, - revision: Optional[Any], - parameters: Optional[Dict[str, Any]], - schemas: Optional[Dict[str, Any]], - is_chat: Optional[bool], -) -> tuple[Dict[str, Any], Optional[Dict[str, Any]], Optional[bool]]: - revision_dict = _to_json_dict(revision) - revision_data = _to_json_dict(revision_dict.get("data")) - revision_flags = _to_json_dict(revision_dict.get("flags")) - - effective_parameters = parameters - if effective_parameters is None: - revision_parameters = revision_data.get("parameters") - effective_parameters = ( - revision_parameters if isinstance(revision_parameters, dict) else {} - ) - - effective_schemas = schemas - if effective_schemas is None: - revision_schemas = revision_data.get("schemas") - effective_schemas = ( - revision_schemas if isinstance(revision_schemas, dict) else None - ) - - effective_is_chat = is_chat - if effective_is_chat is None and "is_chat" in revision_flags: - effective_is_chat = bool(revision_flags["is_chat"]) - - return effective_parameters or {}, effective_schemas, effective_is_chat - - -def get_parameters_from_schemas( - schemas: Optional[Dict[str, Any]], - is_chat: Optional[bool] = None, -) -> tuple[List[Dict[str, Any]], Optional[bool]]: - if hasattr(schemas, "model_dump"): - schemas = schemas.model_dump(mode="json", exclude_none=True) - - if not isinstance(schemas, dict) or not schemas: - return [], is_chat - - parameters_schema = schemas.get("parameters") or {} - inputs_schema = schemas.get("inputs") or {} - - parameter_properties = ( - parameters_schema.get("properties", {}) - if isinstance(parameters_schema, dict) - else {} - ) - input_properties = ( - inputs_schema.get("properties", {}) if isinstance(inputs_schema, dict) else {} - ) - - parameters: List[Dict[str, Any]] = [] - - if isinstance(parameters_schema, dict): - parameters.append( - { - "name": "ag_config", - "type": "dict", - "default": list(parameter_properties.keys()), - } - ) - - input_names: List[str] = [] - has_messages = False - - for name, schema in input_properties.items(): - if not isinstance(schema, dict): - continue - - is_messages_field = name == "messages" or schema.get("x-ag-type-ref") in { - "messages", - "message", - } - - if is_messages_field: - has_messages = True - parameters.append( - { - "name": name, - "type": "messages", - "default": schema.get("default", []), - } - ) - continue - - if schema.get("x-ag-type") == "file_url": - parameters.append( - { - "name": name, - "type": "file_url", - "default": schema.get("default", ""), - } - ) - continue - - input_names.append(name) - - inferred_is_chat = is_chat if is_chat is not None else has_messages - - parameters.append( - { - "name": "inputs", - "type": "dict", - "default": input_names, - } - ) - - if inferred_is_chat and not has_messages: - parameters.append( - { - "name": "messages", - "type": "messages", - "default": [], - } - ) - - return parameters, inferred_is_chat - - -async def get_parameters_from_inspect( - runtime_prefix: str, - route_path: str, - headers: Optional[Dict[str, str]], -) -> tuple[List[Dict], Optional[bool]]: - """ - Read runtime inspect output for an LLM app and derive the UI parameter list. - """ - inspect_url = _build_inspect_url( - runtime_prefix=runtime_prefix, - route_path=route_path, - ) - payload = await _post_json_to_uri( - uri=inspect_url, - headers=headers, - body={}, - ) - - revision = (payload.get("data") or {}).get("revision") or {} - revision_data = revision.get("data") or {} - schemas = revision_data.get("schemas") - flags = payload.get("flags") - - is_chat = None - if isinstance(flags, dict) and "is_chat" in flags: - is_chat = bool(flags["is_chat"]) - - return get_parameters_from_schemas( - schemas=schemas, - is_chat=is_chat, - ) - - -def _build_inspect_url( - *, - runtime_prefix: str, - route_path: str, -) -> str: - runtime_prefix = runtime_prefix.rstrip("/") - route_path = route_path.strip("/") - - if route_path: - return f"{runtime_prefix}/{route_path}/inspect" - - return f"{runtime_prefix}/inspect" - - -async def _post_json_to_uri( - uri: str, - headers: Optional[Dict[str, str]], - body: Dict[str, Any], -): - if headers is None: - headers = {} - headers["ngrok-skip-browser-warning"] = "1" - - async with aiohttp.ClientSession() as client: - resp = await client.post(uri, headers=headers, json=body, timeout=5) - resp_text = await resp.text() - json_data = json.loads(resp_text) - return json_data diff --git a/api/oss/src/services/organization_service.py b/api/oss/src/services/organization_service.py index 18a1031949..f6d0fb9bf8 100644 --- a/api/oss/src/services/organization_service.py +++ b/api/oss/src/services/organization_service.py @@ -6,7 +6,8 @@ from oss.src.utils.env import env from oss.src.models.db_models import UserDB -from oss.src.services import db_manager, email_service +from oss.src.services import db_manager +from oss.src.utils import emailing from oss.src.models.api.workspace_models import InviteRequest, ResendInviteRequest @@ -158,26 +159,17 @@ async def send_invitation_email( if not env.sendgrid.enabled: return invite_link - html_template = email_service.read_email_template("./templates/send_email.html") - html_content = html_template.format( - username_placeholder=user.username, - action_placeholder="invited you to join", - workspace_placeholder="their organization", + await emailing.send_email( + to_email=email, + subject=f"{user.username} invited you to join their organization", + username=user.username, + action="invited you to join", + workspace="their organization", call_to_action=( "Click the link below to accept the invitation:


" f'Accept Invitation' ), ) - - if not env.sendgrid.from_address: - raise ValueError("Sendgrid requires a sender email address to work.") - - await email_service.send_email( - from_email=env.sendgrid.from_address, - to_email=email, - subject=f"{user.username} invited you to join their organization", - html_content=html_content, - ) return True diff --git a/api/oss/src/services/templates/send_email.html b/api/oss/src/services/templates/send_email.html deleted file mode 100644 index 7d124ffd8a..0000000000 --- a/api/oss/src/services/templates/send_email.html +++ /dev/null @@ -1,7 +0,0 @@ -

Hello,

-

- {username_placeholder} has {action_placeholder} {workspace_placeholder} on - Agenta. -

-

{call_to_action}

-

Thank you for using Agenta!

diff --git a/api/oss/src/services/user_service.py b/api/oss/src/services/user_service.py index d254510e72..8255f4430a 100644 --- a/api/oss/src/services/user_service.py +++ b/api/oss/src/services/user_service.py @@ -5,9 +5,10 @@ from oss.src.utils.env import env from oss.src.models.db_models import UserDB from oss.src.utils.logging import get_module_logger -from oss.src.dbs.postgres.shared.engine import engine +from oss.src.dbs.postgres.shared.engine import get_transactions_engine from oss.src.models.api.user_models import UserUpdate -from oss.src.services import db_manager, email_service +from oss.src.services import db_manager +from oss.src.utils import emailing log = get_module_logger(__name__) @@ -36,7 +37,9 @@ async def delete_user(user_id: str) -> None: Raises: NoResultFound: If user with the given ID is not found. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(UserDB).filter_by(id=user_id)) user = result.scalars().first() @@ -68,7 +71,9 @@ async def create_new_user(payload: dict) -> UserDB: # Attempt to create new user try: - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: user = UserDB(**payload) session.add(user) @@ -110,7 +115,9 @@ async def update_user(user_uid: str, payload: UserUpdate) -> UserDB: NoResultFound: User with session id xxxx not found. """ - async with engine.core_session() as session: + engine = get_transactions_engine() + + async with engine.session() as session: result = await session.execute(select(UserDB).filter_by(uid=user_uid)) user = result.scalars().first() @@ -151,20 +158,11 @@ async def generate_user_password_reset_link(user_id: str, admin_user_id: str): if not env.sendgrid.api_key: return password_reset_link - html_template = email_service.read_email_template("./templates/send_email.html") - html_content = html_template.format( - username_placeholder=admin_user.username, - action_placeholder="requested a password reset for you in their workspace", - workspace_placeholder="", - call_to_action=f"""

Click the link below to reset your password:


Reset Password""", - ) - - if not env.sendgrid.from_address: - raise ValueError("Sendgrid requires a sender email address to work.") - - await email_service.send_email( - from_email=env.sendgrid.from_address, + await emailing.send_email( to_email=user.email, subject=f"{admin_user.username} requested a password reset for you in their workspace", - html_content=html_content, + username=admin_user.username, + action="requested a password reset for you in their workspace", + workspace="", + call_to_action=f"""

Click the link below to reset your password:


Reset Password""", ) diff --git a/api/oss/src/tasks/asyncio/events/worker.py b/api/oss/src/tasks/asyncio/events/worker.py index f21e71cd7b..598f43fd1a 100644 --- a/api/oss/src/tasks/asyncio/events/worker.py +++ b/api/oss/src/tasks/asyncio/events/worker.py @@ -14,8 +14,8 @@ log = get_module_logger(__name__) if is_ee(): - from ee.src.utils.entitlements import check_entitlements, scope_from - from ee.src.core.entitlements.types import Counter + from ee.src.core.access.entitlements.service import check_entitlements, scope_from + from ee.src.core.access.entitlements.types import Counter if TYPE_CHECKING: from oss.src.tasks.asyncio.webhooks.dispatcher import WebhooksDispatcher @@ -267,10 +267,11 @@ async def process_batch( if is_ee() and org_id and not org_allowed.get(org_id, True): continue - total_ingested += await self.service.ingest( - project_id=project_batch["project_id"], - events=[msg.to_event() for msg in project_batch["events"]], - ) + if is_ee(): + total_ingested += await self.service.ingest( + project_id=project_batch["project_id"], + events=[msg.to_event() for msg in project_batch["events"]], + ) allowed_batches.append(project_batch) return total_ingested, processed_ids, allowed_batches diff --git a/api/oss/src/tasks/asyncio/tracing/worker.py b/api/oss/src/tasks/asyncio/tracing/worker.py index 5a745bfa9f..b93a414262 100644 --- a/api/oss/src/tasks/asyncio/tracing/worker.py +++ b/api/oss/src/tasks/asyncio/tracing/worker.py @@ -21,7 +21,11 @@ log = get_module_logger(__name__) if is_ee(): - from ee.src.utils.entitlements import check_entitlements, scope_from, Counter + from ee.src.core.access.entitlements.service import ( + check_entitlements, + scope_from, + Counter, + ) class TracingWorker: diff --git a/api/oss/src/tasks/asyncio/webhooks/dispatcher.py b/api/oss/src/tasks/asyncio/webhooks/dispatcher.py index b217735756..c0ff2570fc 100644 --- a/api/oss/src/tasks/asyncio/webhooks/dispatcher.py +++ b/api/oss/src/tasks/asyncio/webhooks/dispatcher.py @@ -106,12 +106,6 @@ async def _get_subscriptions( ) if cached is not None: - log.info( - "[WEBHOOKS DISPATCHER] Subscriptions cache hit", - project_id=str(project_id), - count=len(cached), - ) - decrypted = [] for sub in cached: @@ -136,12 +130,6 @@ async def _get_subscriptions( subscription=WebhookSubscriptionQuery(), ) - log.info( - "[WEBHOOKS DISPATCHER] Subscriptions loaded from DB", - project_id=str(project_id), - count=len(subscriptions), - ) - # Resolve secrets (vault reads happen only on cache miss) result: List[WebhookSubscription] = [] @@ -211,10 +199,6 @@ async def dispatch( continue if not subscriptions: - log.info( - "[WEBHOOKS DISPATCHER] No subscriptions for project", - project_id=str(project_id), - ) continue for msg in project_batch["events"]: diff --git a/api/oss/src/tasks/taskiq/evaluations/worker.py b/api/oss/src/tasks/taskiq/evaluations/worker.py index 4c70fa8d25..909eee2d0c 100644 --- a/api/oss/src/tasks/taskiq/evaluations/worker.py +++ b/api/oss/src/tasks/taskiq/evaluations/worker.py @@ -15,15 +15,13 @@ from oss.src.core.workflows.service import WorkflowsService from oss.src.core.evaluations.service import EvaluationsService -from oss.src.core.evaluations.tasks.legacy import ( - evaluate_batch_testset as evaluate_batch_testset_impl, - evaluate_batch_invocation as evaluate_batch_invocation_impl, - evaluate_batch_testcases as evaluate_batch_testcases_impl, - evaluate_batch_traces as evaluate_batch_traces_impl, -) -from oss.src.core.evaluations.tasks.live import ( - evaluate_live_query as evaluate_live_query_impl, +from oss.src.core.evaluations.tasks.run import ( + EvaluationSliceSource, + run_from_source, + run_from_batch, + rerun, ) +from oss.src.core.evaluations.runtime.models import SliceProcessMode from oss.src.core.evaluations.runtime.locks import ( acquire_job_lock, release_job_lock, @@ -225,55 +223,12 @@ def _register_tasks(self): """Register all evaluation tasks with the broker.""" @self.broker.task( - task_name="evaluations.legacy.annotate", + task_name="evaluations.run_from_source.process", retry_on_error=False, max_retries=0, # Never retry - handle errors in application logic ) - async def evaluate_batch_testset( + async def process_run_from_source( *, - project_id: UUID, - user_id: UUID, - # - run_id: UUID, - context: Context = TaskiqDepends(), - ) -> Any: - """Legacy annotation task - wraps the existing annotate function.""" - log.info( - "[TASK] Starting evaluate_batch_testset", - project_id=str(project_id), - user_id=str(user_id), - ) - - result = await self._with_job_lock( - run_id, - job_id=context.message.task_id or str(uuid4()), - job_type="api", - allow_concurrency=False, - runner=lambda: evaluate_batch_testset_impl( - project_id=project_id, - user_id=user_id, - # - run_id=run_id, - # - tracing_service=self.tracing_service, - testsets_service=self.testsets_service, - queries_service=self.queries_service, - workflows_service=self.workflows_service, - applications_service=self.applications_service, - evaluations_service=self.evaluations_service, - # - simple_evaluators_service=self.simple_evaluators_service, - ), - ) - log.info("[TASK] Completed evaluate_batch_testset") - return result - - @self.broker.task( - task_name="evaluations.live.evaluate", - retry_on_error=False, - max_retries=0, # Never retry - handle errors in application logic - ) - async def evaluate_live_query( project_id: UUID, user_id: UUID, # @@ -283,8 +238,8 @@ async def evaluate_live_query( oldest: Optional[datetime] = None, context: Context = TaskiqDepends(), ) -> Any: - """Live evaluation task - evaluates traces against evaluators.""" - log.info("[TASK] Starting evaluate_live_query") + """Process one evaluation run using the unified topology dispatcher.""" + log.info("[TASK] Starting process_run_from_source") if newest is None: newest = datetime.now(timezone.utc) @@ -296,168 +251,107 @@ async def evaluate_live_query( job_id=context.message.task_id or str(uuid4()), job_type="api", allow_concurrency=False, - runner=lambda: evaluate_live_query_impl( + runner=lambda: run_from_source( project_id=project_id, user_id=user_id, - # run_id=run_id, - # newest=newest, oldest=oldest, - ), - ) - log.info("[TASK] Completed evaluate_live_query") - return result - - @self.broker.task( - task_name="evaluations.queries.batch", - retry_on_error=False, - max_retries=0, - ) - async def evaluate_batch_query( - *, - project_id: UUID, - user_id: UUID, - # - run_id: UUID, - context: Context = TaskiqDepends(), - ) -> Any: - """One-shot query evaluation task for non-live runs.""" - log.info("[TASK] Starting evaluate_batch_query") - - result = await self._with_job_lock( - run_id, - job_id=context.message.task_id or str(uuid4()), - job_type="api", - allow_concurrency=False, - runner=lambda: evaluate_live_query_impl( - project_id=project_id, - user_id=user_id, - # - run_id=run_id, - # - newest=None, - oldest=None, - # - use_windowing=True, - ), - ) - log.info("[TASK] Completed evaluate_batch_query") - return result - - @self.broker.task( - task_name="evaluations.invocations.batch", - retry_on_error=False, - max_retries=0, - ) - async def evaluate_batch_invocation( - *, - project_id: UUID, - user_id: UUID, - # - run_id: UUID, - context: Context = TaskiqDepends(), - ) -> Any: - log.info("[TASK] Starting evaluate_batch_invocation") - result = await self._with_job_lock( - run_id, - job_id=context.message.task_id or str(uuid4()), - job_type="api", - allow_concurrency=False, - runner=lambda: evaluate_batch_invocation_impl( - project_id=project_id, - user_id=user_id, - # - run_id=run_id, - # tracing_service=self.tracing_service, testsets_service=self.testsets_service, + queries_service=self.queries_service, + workflows_service=self.workflows_service, applications_service=self.applications_service, evaluations_service=self.evaluations_service, + simple_evaluators_service=self.simple_evaluators_service, ), ) - log.info("[TASK] Completed evaluate_batch_invocation") + log.info("[TASK] Completed process_run_from_source") return result @self.broker.task( - task_name="evaluations.traces.batch", + task_name="evaluations.run_from_batch.process", retry_on_error=False, max_retries=0, ) - async def evaluate_batch_traces( + async def process_run_from_batch( *, project_id: UUID, user_id: UUID, # run_id: UUID, - trace_ids: list[str], + source_kind: EvaluationSliceSource, + trace_ids: Optional[list[str]] = None, + testcase_ids: Optional[list[UUID]] = None, input_step_key: Optional[str] = None, context: Context = TaskiqDepends(), ) -> Any: - log.info("[TASK] Starting evaluate_batch_traces") + log.info("[TASK] Starting process_run_from_batch", source_kind=source_kind) result = await self._with_job_lock( run_id, job_id=context.message.task_id or str(uuid4()), job_type="api", allow_concurrency=True, - runner=lambda: evaluate_batch_traces_impl( + runner=lambda: run_from_batch( project_id=project_id, user_id=user_id, - # run_id=run_id, + source_kind=source_kind, trace_ids=trace_ids, + testcase_ids=testcase_ids, input_step_key=input_step_key, - # tracing_service=self.tracing_service, + testcases_service=self.testcases_service, workflows_service=self.workflows_service, + applications_service=self.applications_service, evaluations_service=self.evaluations_service, ), ) - log.info("[TASK] Completed evaluate_batch_traces") + log.info("[TASK] Completed process_run_from_batch", source_kind=source_kind) return result @self.broker.task( - task_name="evaluations.testcases.batch", + task_name="evaluations.rerun.process", retry_on_error=False, max_retries=0, ) - async def evaluate_batch_testcases( + async def process_rerun( *, project_id: UUID, user_id: UUID, # run_id: UUID, - testcase_ids: list[UUID], - input_step_key: Optional[str] = None, + scenario_ids: Optional[list[UUID]] = None, + step_keys: Optional[list[str]] = None, + repeat_idxs: Optional[list[int]] = None, + process_mode: SliceProcessMode = "fill-missing", context: Context = TaskiqDepends(), ) -> Any: - log.info("[TASK] Starting evaluate_batch_testcases") + log.info("[TASK] Starting process_rerun", run_id=str(run_id)) result = await self._with_job_lock( run_id, job_id=context.message.task_id or str(uuid4()), job_type="api", allow_concurrency=True, - runner=lambda: evaluate_batch_testcases_impl( + runner=lambda: rerun( project_id=project_id, user_id=user_id, - # run_id=run_id, - testcase_ids=testcase_ids, - input_step_key=input_step_key, - # + scenario_ids=scenario_ids, + step_keys=step_keys, + repeat_idxs=repeat_idxs, + process_mode=process_mode, tracing_service=self.tracing_service, testcases_service=self.testcases_service, workflows_service=self.workflows_service, + applications_service=self.applications_service, evaluations_service=self.evaluations_service, ), ) - log.info("[TASK] Completed evaluate_batch_testcases") + log.info("[TASK] Completed process_rerun", run_id=str(run_id)) return result # Store task references for external access - self.evaluate_batch_testset = evaluate_batch_testset - self.evaluate_live_query = evaluate_live_query - self.evaluate_batch_query = evaluate_batch_query - self.evaluate_batch_invocation = evaluate_batch_invocation - self.evaluate_batch_traces = evaluate_batch_traces - self.evaluate_batch_testcases = evaluate_batch_testcases + self.process_run_from_source = process_run_from_source + self.process_run_from_batch = process_run_from_batch + self.process_rerun = process_rerun diff --git a/api/oss/src/utils/caching.py b/api/oss/src/utils/caching.py index 5b90815722..b2c6f97b62 100644 --- a/api/oss/src/utils/caching.py +++ b/api/oss/src/utils/caching.py @@ -1,20 +1,17 @@ from typing import Any, Type, Optional, Union from random import random from asyncio import sleep -from uuid import uuid4 import orjson -# from cachetools import TTLCache -from redis.asyncio import Redis from pydantic import BaseModel from oss.src.utils.logging import get_module_logger from oss.src.utils.env import env +from oss.src.dbs.redis.shared.engine import get_cache_engine log = get_module_logger(__name__) -AGENTA_LOCK_TTL = 15 # 15 seconds AGENTA_CACHE_TTL = 5 * 60 # 5 minutes (Layer 2) [L2] AGENTA_CACHE_LOCAL_TTL = 15 # 15 seconds for local in-memory cache (Layer 1) [L1] @@ -26,7 +23,6 @@ AGENTA_CACHE_SCAN_BATCH_SIZE = 500 AGENTA_CACHE_DELETE_BATCH_SIZE = 1000 -AGENTA_LOCK_SOCKET_TIMEOUT = 2.0 # Locks should be more reliable than cache lookups CACHE_DEBUG = False CACHE_DEBUG_VALUE = False @@ -38,35 +34,7 @@ # Original L1 cache: # local_cache: TTLCache = TTLCache(maxsize=4096, ttl=AGENTA_CACHE_LOCAL_TTL) -# Use volatile Redis instance for caching (prefix-based separation) -# decode_responses=False: orjson operates on bytes for 3x performance vs json -r = Redis.from_url( - url=env.redis.uri_volatile, - decode_responses=False, - socket_timeout=0.5, # read/write timeout -) - -# Dedicated Redis client for distributed locks with a longer timeout. -r_lock = Redis.from_url( - url=env.redis.uri_volatile, - decode_responses=False, - socket_timeout=AGENTA_LOCK_SOCKET_TIMEOUT, -) - -# Ownership-safe lock scripts. Owner token must match to renew/release. -_LOCK_RENEW_IF_OWNER_SCRIPT = """ -if redis.call("GET", KEYS[1]) == ARGV[1] then - return redis.call("EXPIRE", KEYS[1], tonumber(ARGV[2])) -end -return 0 -""" - -_LOCK_RELEASE_IF_OWNER_SCRIPT = """ -if redis.call("GET", KEYS[1]) == ARGV[1] then - return redis.call("DEL", KEYS[1]) -end -return 0 -""" +_cache_engine = get_cache_engine() # HELPERS ---------------------------------------------------------------------- @@ -129,7 +97,7 @@ async def _scan(pattern: str) -> list[str]: keys: list[str] = [] while True: # TODO: Really ? - cursor, batch = await r.scan( + cursor, batch = await _cache_engine.scan( cursor=cursor, match=pattern, count=AGENTA_CACHE_SCAN_BATCH_SIZE, @@ -219,7 +187,7 @@ async def _try_get_and_maybe_renew( # return _deserialize(raw, model=model, is_list=is_list) # Layer 2: Check Redis (distributed, 5min TTL, ~1ms latency) - raw = await r.get(cache_name) + raw = await _cache_engine.get(cache_name) if raw is not None: if CACHE_DEBUG: @@ -240,7 +208,7 @@ async def _try_get_and_maybe_renew( name=cache_name, ) - await r.expire(cache_name, ttl) + await _cache_engine.expire(cache_name, ttl) else: if CACHE_DEBUG: log.debug( @@ -306,7 +274,7 @@ async def _maybe_retry_get( lock_name = f"lock::{cache_name}" lock_ex = int(lock_ttl * 1000) # convert seconds to milliseconds - got_lock = await r.set(lock_name, "1", nx=True, ex=lock_ex) + got_lock = await _cache_engine.set(lock_name, "1", nx=True, ex=lock_ex) if got_lock: if CACHE_DEBUG: @@ -379,7 +347,7 @@ async def set_cache( # # Original L1 write path: # local_cache[cache_name] = cache_value - await r.set(cache_name, cache_value, px=cache_px) + await _cache_engine.set(cache_name, cache_value, px=cache_px) if CACHE_DEBUG: log.debug( @@ -392,7 +360,7 @@ async def set_cache( lock_name = f"lock::{cache_name}" - check = await r.delete(lock_name) + check = await _cache_engine.delete(lock_name) if check: if CACHE_DEBUG: @@ -513,7 +481,7 @@ async def invalidate_cache( # # Original L1 invalidation path: # local_cache.pop(cache_name, None) - await r.delete(cache_name) + await _cache_engine.delete(cache_name) else: cache_name = _pack( @@ -560,7 +528,7 @@ async def invalidate_cache( redis_keys_deleted = 0 for i in range(0, len(keys), AGENTA_CACHE_DELETE_BATCH_SIZE): batch = keys[i : i + AGENTA_CACHE_DELETE_BATCH_SIZE] - deleted_count = await r.delete(*batch) + deleted_count = await _cache_engine.delete(*batch) redis_keys_deleted += deleted_count if CACHE_DEBUG: @@ -589,220 +557,3 @@ async def invalidate_cache( log.warn(e) return None - - -async def acquire_lock( - namespace: str, - key: Optional[Union[str, dict]] = None, - project_id: Optional[str] = None, - user_id: Optional[str] = None, - ttl: int = AGENTA_LOCK_TTL, - strict: bool = False, -) -> Optional[str]: - """Acquire a distributed lock using Redis SET NX (atomic check-and-set). - - This prevents race conditions in distributed systems by ensuring only one - process can acquire the lock at a time. - - Args: - namespace: Lock namespace (e.g., "account-creation", "task-processing") - key: Unique identifier for the lock (e.g., email, user_id, task_id) - project_id: Optional project scope - user_id: Optional user scope - ttl: Lock expiration time in seconds (default: 10). Auto-releases after TTL. - strict: If True, re-raise Redis errors instead of returning None. - - Returns: - Lock owner token if lock was acquired, None if lock is already held by another process. - - Example: - lock_owner = await acquire_lock(namespace="account-creation", key=email, ttl=10) - if not lock_owner: - # Another process has the lock - return - - try: - # Do work while holding the lock - await create_account(email) - finally: - # Always release the lock - await release_lock( - namespace="account-creation", - key=email, - owner=lock_owner, - ) - """ - try: - lock_key = _pack( - namespace=f"lock:{namespace}", - key=key, - project_id=project_id, - user_id=user_id, - ) - lock_owner = uuid4().hex - - # Atomic SET NX: Returns True if lock acquired, False if already held - acquired = await r_lock.set(lock_key, lock_owner, nx=True, ex=ttl) - - if acquired: - if CACHE_DEBUG: - log.debug( - "[lock] ACQUIRED", - key=lock_key, - ttl=ttl, - ) - return lock_owner - else: - if CACHE_DEBUG: - log.debug( - "[lock] BLOCKED", - key=lock_key, - ) - return None - - except Exception as e: - log.error( - f"[lock] ACQUIRE ERROR: namespace={namespace} key={key} error={e}", - exc_info=True, - ) - if strict: - raise - return None - - -async def renew_lock( - namespace: str, - key: Optional[Union[str, dict]] = None, - project_id: Optional[str] = None, - user_id: Optional[str] = None, - ttl: int = AGENTA_LOCK_TTL, - owner: Optional[str] = None, -) -> bool: - """Renew (extend) the TTL of an existing distributed lock. - - Use this to prevent lock expiration during long-running operations. - Only succeeds if the lock key still exists in Redis. If an owner token is - provided, renewal only succeeds when ownership matches. - - Args: - namespace: Lock namespace (same as used in acquire_lock) - key: Lock key (same as used in acquire_lock) - project_id: Optional project ID (same as used in acquire_lock) - user_id: Optional user ID (same as used in acquire_lock) - ttl: New expiration time in seconds - owner: Optional owner token returned by acquire_lock - - Returns: - True if lock was renewed, False if lock has already expired or on error - """ - try: - lock_key = _pack( - namespace=f"lock:{namespace}", - key=key, - project_id=project_id, - user_id=user_id, - ) - - if owner: - renewed = await r_lock.eval( - _LOCK_RENEW_IF_OWNER_SCRIPT, - 1, - lock_key, - owner, - str(ttl), - ) - else: - renewed = await r_lock.expire(lock_key, ttl) - - if renewed: - if CACHE_DEBUG: - log.debug( - "[lock] RENEWED", - key=lock_key, - ttl=ttl, - ) - return True - else: - log.warn( - f"[lock] RENEW FAILED (expired or lost ownership): namespace={namespace} key={key}" - ) - return False - - except Exception as e: - log.error( - f"[lock] RENEW ERROR: namespace={namespace} key={key} error={e}", - exc_info=True, - ) - return False - - -async def release_lock( - namespace: str, - key: Optional[Union[str, dict]] = None, - project_id: Optional[str] = None, - user_id: Optional[str] = None, - owner: Optional[str] = None, - strict: bool = False, -) -> bool: - """Release a distributed lock acquired with acquire_lock(). - - Args: - namespace: Lock namespace (same as used in acquire_lock) - key: Lock key (same as used in acquire_lock) - project_id: Optional project ID (same as used in acquire_lock) - user_id: Optional user ID (same as used in acquire_lock) - owner: Optional owner token returned by acquire_lock - strict: If True, re-raise Redis errors instead of returning False. - - Returns: - True if lock was released, False if already expired. - - Example: - lock_acquired = await acquire_lock(namespace="account-creation", key=email) - if lock_acquired: - try: - # ... critical section ... - finally: - await release_lock(namespace="account-creation", key=email) - """ - try: - lock_key = _pack( - namespace=f"lock:{namespace}", - key=key, - project_id=project_id, - user_id=user_id, - ) - - if owner: - deleted = await r_lock.eval( - _LOCK_RELEASE_IF_OWNER_SCRIPT, - 1, - lock_key, - owner, - ) - else: - deleted = await r_lock.delete(lock_key) - - if deleted: - if CACHE_DEBUG: - log.debug( - "[lock] RELEASED", - key=lock_key, - ) - return True - else: - if CACHE_DEBUG: - log.debug( - "[lock] ALREADY EXPIRED OR OWNED BY ANOTHER WORKER", - key=lock_key, - ) - return False - - except Exception as e: - log.error( - f"[lock] RELEASE ERROR: namespace={namespace} key={key} error={e}", - exc_info=True, - ) - if strict: - raise - return False diff --git a/api/oss/src/utils/emailing.py b/api/oss/src/utils/emailing.py new file mode 100644 index 0000000000..1195cb8680 --- /dev/null +++ b/api/oss/src/utils/emailing.py @@ -0,0 +1,136 @@ +import time + +import httpx + +from sendgrid.helpers.mail import Mail + +from oss.src.utils.env import env +from oss.src.utils.lazy import _load_sendgrid +from oss.src.utils.logging import get_module_logger + +log = get_module_logger(__name__) + + +# Shared invitation/notification email template. Inlined (rather than a file) +# since it is tiny and avoids per-send file I/O and path resolution. +_EMAIL_TEMPLATE = ( + "

Hello,

\n" + "

\n" + " {username_placeholder} has {action_placeholder} {workspace_placeholder} on\n" + " Agenta.\n" + "

\n" + "

{call_to_action}

\n" + "

Thank you for using Agenta!

\n" +) + + +def _render_email_template( + *, + username: str, + action: str, + workspace: str, + call_to_action: str, +) -> str: + """Render the shared invitation/notification email template.""" + + return _EMAIL_TEMPLATE.format( + username_placeholder=username, + action_placeholder=action, + workspace_placeholder=workspace, + call_to_action=call_to_action, + ) + + +async def send_email( + *, + to_email: str, + subject: str, + # + username: str, + action: str, + workspace: str, + call_to_action: str, + # + from_email: str = None, +) -> bool: + """ + Render the shared email template and send it via SendGrid. + + No-op (returns True) when SendGrid is unavailable (disabled or failed to + load). Callers that need to short-circuit on a disabled mailer before doing + other work should still gate on `env.sendgrid.enabled` themselves. + + Returns True if the email was sent (or skipped because mailing is disabled), + raises on a send failure or missing sender address. + """ + + sendgrid = _load_sendgrid() + if sendgrid is None: + log.info(f"[SENDGRID] Email disabled - would send '{subject}' to {to_email}") + return True + + sender = from_email or env.sendgrid.from_address + if not sender: + raise ValueError("Sendgrid requires a sender email address to work.") + + html_content = _render_email_template( + username=username, + action=action, + workspace=workspace, + call_to_action=call_to_action, + ) + + message = Mail( + from_email=sender, + to_emails=to_email, + subject=subject, + html_content=html_content, + ) + + sendgrid.send(message) + + return True + + +def add_contact(email: str, max_retries: int = 5, initial_delay: int = 1): + """ + Add a contact to the Loops audience, with retry and exponential backoff. + + No-op (returns None) when Loops is disabled (no API key configured). + + Args: + email (str): Email address of the contact to be added. + max_retries (int): Maximum number of retries in case of rate limiting. + initial_delay (int): Initial delay in seconds before retrying. + + Raises: + ConnectionError: If max retries reached and unable to connect to Loops API. + + Returns: + Optional[httpx.Response]: The Loops API response, or None when disabled. + """ + + if not env.loops.enabled: + log.info(f"[LOOPS] Disabled - would add contact {email}") + return None + + url = "https://app.loops.so/api/v1/contacts/create" + headers = {"Authorization": f"Bearer {env.loops.api_key}"} + data = {"email": email} + + retries = 0 + delay = initial_delay + + while retries < max_retries: + response = httpx.post(url, json=data, headers=headers, timeout=20) + + # 429 indicates rate limiting; back off and retry. + if response.status_code == 429: + log.warning(f"[LOOPS] Rate limit hit. Retrying in {delay} seconds...") + time.sleep(delay) + retries += 1 + delay *= 2 + else: + return response + + raise ConnectionError("Max retries reached. Unable to connect to Loops API.") diff --git a/api/oss/src/utils/env.py b/api/oss/src/utils/env.py index 89c6262bb2..88f6cb55b3 100644 --- a/api/oss/src/utils/env.py +++ b/api/oss/src/utils/env.py @@ -86,7 +86,7 @@ class AccessConfig(BaseModel): """Access controls (allow/block lists, plans + roles, default plan). JSON env vars are parsed here at startup. Schema validation happens in - ``ee.src.core.entitlements.controls``. + ``ee.src.core.access.controls``. `default_plan` lives here (not under `agenta`) because it's part of the access-controls surface: it selects which entry of the effective plan diff --git a/api/oss/src/utils/lazy.py b/api/oss/src/utils/lazy.py new file mode 100644 index 0000000000..15df32d5ad --- /dev/null +++ b/api/oss/src/utils/lazy.py @@ -0,0 +1,117 @@ +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: + import stripe + import posthog + from sendgrid import SendGridAPIClient + + +_stripe_module: Optional["stripe"] = None +_stripe_checked = False + +_posthog_module: Optional["posthog"] = None +_posthog_checked = False + +_sendgrid_client: Optional["SendGridAPIClient"] = None +_sendgrid_checked = False + + +def _load_stripe() -> Optional["stripe"]: + global _stripe_module, _stripe_checked + + if _stripe_checked: + return _stripe_module + + _stripe_checked = True + try: + from oss.src.utils.env import env + from oss.src.utils.logging import get_module_logger + + log = get_module_logger(__name__) + + # Gate on "enabled" here so the return value means "a usable, configured + # Stripe module": None signals "not available" (disabled OR import/config + # failure), and callers need a single `if stripe is None` check. + if not env.stripe.enabled: + log.warn("✗ Stripe disabled") + _stripe_module = None + return _stripe_module + + import stripe as _stripe + + _stripe.api_key = env.stripe.api_key + log.info("✓ Stripe enabled:", target=env.stripe.webhook_target) + + _stripe_module = _stripe + except Exception: + _stripe_module = None + + return _stripe_module + + +def _load_posthog() -> Optional["posthog"]: + global _posthog_module, _posthog_checked + + if _posthog_checked: + return _posthog_module + + _posthog_checked = True + try: + from oss.src.utils.env import env + from oss.src.utils.logging import get_module_logger + + log = get_module_logger(__name__) + + # Gate on "enabled" here so the return value means "a usable, configured + # PostHog module": None signals "not available" (disabled OR import/config + # failure), and callers need a single `if posthog is None` check. + if not env.posthog.enabled: + log.warn("✗ PostHog disabled") + _posthog_module = None + return _posthog_module + + import posthog as _posthog + + _posthog.api_key = env.posthog.api_key + _posthog.host = env.posthog.api_url + log.info("✓ PostHog enabled") + + _posthog_module = _posthog + except Exception: + _posthog_module = None + + return _posthog_module + + +def _load_sendgrid() -> Optional["SendGridAPIClient"]: + global _sendgrid_client, _sendgrid_checked + + if _sendgrid_checked: + return _sendgrid_client + + _sendgrid_checked = True + try: + from oss.src.utils.env import env + from oss.src.utils.logging import get_module_logger + + log = get_module_logger(__name__) + + # Gate on "enabled" here so the return value means "a usable, configured + # SendGrid client": None signals "not available" (disabled OR import/config + # failure), and callers need a single `if sg is None` check. + if not env.sendgrid.enabled: + if env.sendgrid.api_key and not env.sendgrid.from_address: + log.warn("✗ SendGrid disabled: missing sender email address") + else: + log.warn("✗ SendGrid disabled") + _sendgrid_client = None + return _sendgrid_client + + import sendgrid + + _sendgrid_client = sendgrid.SendGridAPIClient(api_key=env.sendgrid.api_key) + log.info("✓ SendGrid enabled") + except Exception: + _sendgrid_client = None + + return _sendgrid_client diff --git a/api/oss/src/utils/locking.py b/api/oss/src/utils/locking.py new file mode 100644 index 0000000000..cf0f5c1460 --- /dev/null +++ b/api/oss/src/utils/locking.py @@ -0,0 +1,292 @@ +"""Distributed locks backed by Redis (volatile). + +Split out of `caching.py`: caching and locking share the same volatile Redis but +use separate clients with different socket timeouts (locks may block, so they use +the longer-timeout `LockEngine`). Lock keys are namespaced the same way as cache +keys via `caching._pack`, so a lock and its related cache entries sort together. +""" + +from typing import Optional, Union +from uuid import uuid4 + +from oss.src.utils.logging import get_module_logger +from oss.src.utils.caching import _pack +from oss.src.dbs.redis.shared.engine import get_lock_engine + +log = get_module_logger(__name__) + +AGENTA_LOCK_TTL = 15 # 15 seconds + +LOCK_DEBUG = False + +_lock_engine = get_lock_engine() + + +# Ownership-safe lock scripts. Owner token must match to renew/release. +_LOCK_RENEW_IF_OWNER_SCRIPT = """ +if redis.call("GET", KEYS[1]) == ARGV[1] then + return redis.call("EXPIRE", KEYS[1], tonumber(ARGV[2])) +end +return 0 +""" + +_LOCK_RELEASE_IF_OWNER_SCRIPT = """ +if redis.call("GET", KEYS[1]) == ARGV[1] then + return redis.call("DEL", KEYS[1]) +end +return 0 +""" + + +# LOCK-STORE PRIMITIVES -------------------------------------------------------- +# +# Thin pass-throughs to the lock Redis client for callers that manage their own +# lock-adjacent keys (e.g. lock metadata, worker heartbeats) and need direct +# key/value access beyond acquire/renew/release. Keeping these here means such +# callers depend only on `locking` and never hold a `LockEngine` themselves. + + +async def set_key(key: str, value, *, ttl: Optional[int] = None) -> None: + """SET a lock-store key, optionally with a TTL (seconds).""" + await _lock_engine.set(key, value, ex=ttl) + + +async def get_key(key: str): + """GET a lock-store key (raw bytes, or None).""" + return await _lock_engine.get(key) + + +async def delete_key(key: str) -> None: + """DEL a lock-store key.""" + await _lock_engine.delete(key) + + +async def has_key(key: str) -> bool: + """EXISTS check for a lock-store key.""" + return bool(await _lock_engine.exists(key)) + + +async def scan_keys(*, match: str): + """SCAN over lock-store keys matching `match` (async iterator). + + Async generator — iterate with `async for`. Uses SCAN, never KEYS. + """ + async for key in _lock_engine.scan_iter(match=match): + yield key + + +async def acquire_lock( + namespace: str, + key: Optional[Union[str, dict]] = None, + project_id: Optional[str] = None, + user_id: Optional[str] = None, + ttl: int = AGENTA_LOCK_TTL, + strict: bool = False, +) -> Optional[str]: + """Acquire a distributed lock using Redis SET NX (atomic check-and-set). + + This prevents race conditions in distributed systems by ensuring only one + process can acquire the lock at a time. + + Args: + namespace: Lock namespace (e.g., "account-creation", "task-processing") + key: Unique identifier for the lock (e.g., email, user_id, task_id) + project_id: Optional project scope + user_id: Optional user scope + ttl: Lock expiration time in seconds (default: 15). Auto-releases after TTL. + strict: If True, re-raise Redis errors instead of returning None. + + Returns: + Lock owner token if lock was acquired, None if lock is already held by another process. + + Example: + lock_owner = await acquire_lock(namespace="account-creation", key=email, ttl=10) + if not lock_owner: + # Another process has the lock + return + + try: + # Do work while holding the lock + await create_account(email) + finally: + # Always release the lock + await release_lock( + namespace="account-creation", + key=email, + owner=lock_owner, + ) + """ + try: + lock_key = _pack( + namespace=f"lock:{namespace}", + key=key, + project_id=project_id, + user_id=user_id, + ) + lock_owner = uuid4().hex + + # Atomic SET NX: Returns True if lock acquired, False if already held + acquired = await _lock_engine.set(lock_key, lock_owner, nx=True, ex=ttl) + + if acquired: + if LOCK_DEBUG: + log.debug( + "[lock] ACQUIRED", + key=lock_key, + ttl=ttl, + ) + return lock_owner + else: + if LOCK_DEBUG: + log.debug( + "[lock] BLOCKED", + key=lock_key, + ) + return None + + except Exception as e: + log.error( + f"[lock] ACQUIRE ERROR: namespace={namespace} key={key} error={e}", + exc_info=True, + ) + if strict: + raise + return None + + +async def renew_lock( + namespace: str, + key: Optional[Union[str, dict]] = None, + project_id: Optional[str] = None, + user_id: Optional[str] = None, + ttl: int = AGENTA_LOCK_TTL, + owner: Optional[str] = None, +) -> bool: + """Renew (extend) the TTL of an existing distributed lock. + + Use this to prevent lock expiration during long-running operations. + Only succeeds if the lock key still exists in Redis. If an owner token is + provided, renewal only succeeds when ownership matches. + + Args: + namespace: Lock namespace (same as used in acquire_lock) + key: Lock key (same as used in acquire_lock) + project_id: Optional project ID (same as used in acquire_lock) + user_id: Optional user ID (same as used in acquire_lock) + ttl: New expiration time in seconds + owner: Optional owner token returned by acquire_lock + + Returns: + True if lock was renewed, False if lock has already expired or on error + """ + try: + lock_key = _pack( + namespace=f"lock:{namespace}", + key=key, + project_id=project_id, + user_id=user_id, + ) + + if owner: + renewed = await _lock_engine.eval( + _LOCK_RENEW_IF_OWNER_SCRIPT, + 1, + lock_key, + owner, + str(ttl), + ) + else: + renewed = await _lock_engine.expire(lock_key, ttl) + + if renewed: + if LOCK_DEBUG: + log.debug( + "[lock] RENEWED", + key=lock_key, + ttl=ttl, + ) + return True + else: + log.warn( + f"[lock] RENEW FAILED (expired or lost ownership): namespace={namespace} key={key}" + ) + return False + + except Exception as e: + log.error( + f"[lock] RENEW ERROR: namespace={namespace} key={key} error={e}", + exc_info=True, + ) + return False + + +async def release_lock( + namespace: str, + key: Optional[Union[str, dict]] = None, + project_id: Optional[str] = None, + user_id: Optional[str] = None, + owner: Optional[str] = None, + strict: bool = False, +) -> bool: + """Release a distributed lock acquired with acquire_lock(). + + Args: + namespace: Lock namespace (same as used in acquire_lock) + key: Lock key (same as used in acquire_lock) + project_id: Optional project ID (same as used in acquire_lock) + user_id: Optional user ID (same as used in acquire_lock) + owner: Optional owner token returned by acquire_lock + strict: If True, re-raise Redis errors instead of returning False. + + Returns: + True if lock was released, False if already expired. + + Example: + lock_acquired = await acquire_lock(namespace="account-creation", key=email) + if lock_acquired: + try: + # ... critical section ... + finally: + await release_lock(namespace="account-creation", key=email) + """ + try: + lock_key = _pack( + namespace=f"lock:{namespace}", + key=key, + project_id=project_id, + user_id=user_id, + ) + + if owner: + deleted = await _lock_engine.eval( + _LOCK_RELEASE_IF_OWNER_SCRIPT, + 1, + lock_key, + owner, + ) + else: + deleted = await _lock_engine.delete(lock_key) + + if deleted: + if LOCK_DEBUG: + log.debug( + "[lock] RELEASED", + key=lock_key, + ) + return True + else: + if LOCK_DEBUG: + log.debug( + "[lock] ALREADY EXPIRED OR OWNED BY ANOTHER WORKER", + key=lock_key, + ) + return False + + except Exception as e: + log.error( + f"[lock] RELEASE ERROR: namespace={namespace} key={key} error={e}", + exc_info=True, + ) + if strict: + raise + return False diff --git a/api/oss/tests/legacy/old_tests/unit/test_llm_apps_service.py b/api/oss/tests/legacy/old_tests/unit/test_llm_apps_service.py deleted file mode 100644 index 04d729ad0b..0000000000 --- a/api/oss/tests/legacy/old_tests/unit/test_llm_apps_service.py +++ /dev/null @@ -1,209 +0,0 @@ -import pytest -from unittest.mock import patch, AsyncMock -import aiohttp - -from oss.src.services.llm_apps_service import ( - batch_invoke, - InvokationResult, - Result, -) - - -@pytest.mark.asyncio -async def test_batch_invoke_success(): - """ - Test the successful invocation of batch_invoke function. - - This test mocks the get_parameters_from_openapi and invoke_app functions - to simulate successful invocations. It verifies that the batch_invoke - function correctly returns the expected results for the given test data. - """ - with ( - patch( - "src.services.llm_apps_service.get_parameters_from_openapi", - new_callable=AsyncMock, - ) as mock_get_parameters_from_openapi, - patch( - "src.services.llm_apps_service.invoke_app", new_callable=AsyncMock - ) as mock_invoke_app, - patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep, # noqa: F841 - ): - mock_get_parameters_from_openapi.return_value = [ - {"name": "param1", "type": "input"}, - {"name": "param2", "type": "input"}, - ] - - # Mock the response of invoke_app to always succeed - def invoke_app_side_effect( - uri, - datapoint, - parameters, - openapi_parameters, - user_id, - project_id, - ): - return InvokationResult( - result=Result(type="text", value="Success", error=None), - latency=0.1, - cost=0.01, - tokens=1, - ) - - mock_invoke_app.side_effect = invoke_app_side_effect - - uri = "http://example.com" - testset_data = [ - {"id": 1, "param1": "value1", "param2": "value2"}, - {"id": 2, "param1": "value1", "param2": "value2"}, - ] - parameters = {} - rate_limit_config = { - "batch_size": 10, - "max_retries": 3, - "retry_delay": 3, - "delay_between_batches": 5, - } - - results = await batch_invoke( - uri, - testset_data, - parameters=parameters, - rate_limit_config=rate_limit_config, - user_id="test_user", - project_id="test_project", - ) - - assert len(results) == 2 - assert results[0].result.type == "text" - assert results[0].result.value == "Success" - assert results[1].result.type == "text" - assert results[1].result.value == "Success" - - -@pytest.mark.asyncio -async def test_batch_invoke_retries_and_failure(): - """ - Test the batch_invoke function with retries and eventual failure. - - This test mocks the get_parameters_from_openapi and invoke_app functions - to simulate failures that trigger retries. It verifies that the batch_invoke - function correctly retries the specified number of times and returns an error - result after reaching the maximum retries. - """ - with ( - patch( - "src.services.llm_apps_service.get_parameters_from_openapi", - new_callable=AsyncMock, - ) as mock_get_parameters_from_openapi, - patch( - "src.services.llm_apps_service.invoke_app", new_callable=AsyncMock - ) as mock_invoke_app, - patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep, # noqa: F841 - ): - mock_get_parameters_from_openapi.return_value = [ - {"name": "param1", "type": "input"}, - {"name": "param2", "type": "input"}, - ] - - # Mock the response of invoke_app to always fail - def invoke_app_side_effect( - uri, - datapoint, - parameters, - openapi_parameters, - user_id, - project_id, - ): - raise aiohttp.ClientError("Test Error") - - mock_invoke_app.side_effect = invoke_app_side_effect - - uri = "http://example.com" - testset_data = [ - {"id": 1, "param1": "value1", "param2": "value2"}, - {"id": 2, "param1": "value1", "param2": "value2"}, - ] - parameters = {} - rate_limit_config = { - "batch_size": 10, - "max_retries": 3, - "retry_delay": 3, - "delay_between_batches": 5, - } - - results = await batch_invoke( - uri, - testset_data, - parameters=parameters, - rate_limit_config=rate_limit_config, - user_id="test_user", - project_id="test_project", - ) - - assert len(results) == 2 - assert results[0].result.type == "error" - assert results[0].result.error.message == "Max retries reached" - assert results[1].result.type == "error" - assert results[1].result.error.message == "Max retries reached" - - -@pytest.mark.asyncio -async def test_batch_invoke_generic_exception(): - """ - Test the batch_invoke function with a generic exception. - - This test mocks the get_parameters_from_openapi and invoke_app functions - to simulate a generic exception during invocation. It verifies that the - batch_invoke function correctly handles the exception and returns an error - result with the appropriate error message. - """ - with ( - patch( - "src.m_apps_service.get_parameters_from_openapi", - new_callable=AsyncMock, - ) as mock_get_parameters_from_openapi, - patch( - "src.services.llm_apps_service.invoke_app", new_callable=AsyncMock - ) as mock_invoke_app, - patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep, # noqa: F841 - ): - mock_get_parameters_from_openapi.return_value = [ - {"name": "param1", "type": "input"}, - {"name": "param2", "type": "input"}, - ] - - # Mock the response of invoke_app to raise a generic exception - def invoke_app_side_effect( - uri, - datapoint, - parameters, - openapi_parameters, - user_id, - project_id, - ): - raise Exception("Generic Error") - - mock_invoke_app.side_effect = invoke_app_side_effect - - uri = "http://example.com" - testset_data = [{"id": 1, "param1": "value1", "param2": "value2"}] - parameters = {} - rate_limit_config = { - "batch_size": 1, - "max_retries": 3, - "retry_delay": 1, - "delay_between_batches": 1, - } - - results = await batch_invoke( - uri, - testset_data, - parameters=parameters, - rate_limit_config=rate_limit_config, - user_id="test_user", - project_id="test_project", - ) - - assert len(results) == 1 - assert results[0].result.type == "error" - assert results[0].result.error.message == "Max retries reached" diff --git a/api/oss/tests/legacy/vault_router/conftest.py b/api/oss/tests/legacy/vault_router/conftest.py index aef2c39a57..ec4abb182f 100644 --- a/api/oss/tests/legacy/vault_router/conftest.py +++ b/api/oss/tests/legacy/vault_router/conftest.py @@ -6,7 +6,7 @@ AGENTA_HOST = os.environ.get("AGENTA_HOST", "http://localhost") -API_BASE_URL = f"{AGENTA_HOST}/api/vault/v1/" +API_BASE_URL = f"{AGENTA_HOST}/api/" @pytest.fixture diff --git a/api/oss/tests/pytest/acceptance/accounts/test_actions.py b/api/oss/tests/pytest/acceptance/accounts/test_actions.py index c94d1ab81d..a3a7d24ec7 100644 --- a/api/oss/tests/pytest/acceptance/accounts/test_actions.py +++ b/api/oss/tests/pytest/acceptance/accounts/test_actions.py @@ -8,6 +8,8 @@ """ from uuid import uuid4 +import os +import pytest # --------------------------------------------------------------------------- @@ -58,6 +60,10 @@ def _delete_account_by_email(admin_api, *, email): class TestResetPassword: + @pytest.mark.skipif( + not os.getenv("POSTHOG_API_KEY"), + reason="PostHog API key not configured", + ) def test_reset_password_for_existing_identity(self, admin_api): uid = uuid4().hex[:12] email = f"reset-{uid}@test.agenta.ai" diff --git a/api/oss/tests/pytest/acceptance/evaluations/_flow_helpers.py b/api/oss/tests/pytest/acceptance/evaluations/_flow_helpers.py new file mode 100644 index 0000000000..7dfde2067e --- /dev/null +++ b/api/oss/tests/pytest/acceptance/evaluations/_flow_helpers.py @@ -0,0 +1,254 @@ +""" +Shared helpers for end-to-end evaluation FLOW tests. + +These tests trigger a real evaluation run through the worker + services +container and poll for completion. They run WITHOUT any LLM or code sandbox by +using the `agenta:custom:mock:v0` workflow for both the application +(invocation) and the evaluator (annotation). The mock workflow returns +predefined results selected by `parameters = {"key": ..., "kwargs": {...}}`. + +See `sdks/python/agenta/sdk/engines/running/handlers.py::mock_v0` for the +selector vocabulary (echo / static / pass / fail / score / error / delay). +""" + +from uuid import uuid4 + +from utils.polling import wait_for_response + + +MOCK_URI = "agenta:custom:mock:v0" + + +# - resource creation --------------------------------------------------------- + + +def create_mock_application(authed_api, *, key="echo", kwargs=None) -> dict: + """Create a deterministic mock application (no LLM). Returns the application dict.""" + slug = uuid4().hex + response = authed_api( + "POST", + "/simple/applications/", + json={ + "application": { + "slug": f"application-{slug}", + "name": f"Mock Application {slug}", + "data": { + "uri": MOCK_URI, + "parameters": {"key": key, "kwargs": kwargs or {}}, + }, + } + }, + ) + assert response.status_code == 200, response.text + return response.json()["application"] + + +def create_mock_evaluator(authed_api, *, key="pass", kwargs=None) -> dict: + """Create a deterministic mock evaluator (no LLM). Returns the evaluator dict.""" + slug = uuid4().hex + response = authed_api( + "POST", + "/simple/evaluators/", + json={ + "evaluator": { + "slug": f"evaluator-{slug}", + "name": f"Mock Evaluator {slug}", + "data": { + "uri": MOCK_URI, + "parameters": {"key": key, "kwargs": kwargs or {}}, + }, + } + }, + ) + assert response.status_code == 200, response.text + return response.json()["evaluator"] + + +def create_query(authed_api, *, trace_type="invocation") -> dict: + """Create a simple query (filters over traces). Returns the query dict.""" + slug = uuid4().hex + response = authed_api( + "POST", + "/simple/queries/", + json={ + "query": { + "slug": f"query-{slug}", + "name": f"Query {slug}", + "data": { + "filtering": { + "operator": "and", + "conditions": [ + { + "field": "trace_type", + "operator": "is", + "value": trace_type, + } + ], + } + }, + } + }, + ) + assert response.status_code == 200, response.text + return response.json()["query"] + + +def create_testset(authed_api, *, testcases=None) -> dict: + """Create a simple testset. Returns the testset dict (with revision_id).""" + slug = uuid4().hex + if testcases is None: + testcases = [ + {"data": {"input": "hello", "expected": "world"}}, + {"data": {"input": "hola", "expected": "mundo"}}, + ] + response = authed_api( + "POST", + "/simple/testsets/", + json={ + "testset": { + "slug": f"testset-{slug}", + "name": f"Testset {slug}", + "data": {"testcases": testcases}, + } + }, + ) + assert response.status_code == 200, response.text + return response.json()["testset"] + + +# - triggering and waiting ----------------------------------------------------- + +# Terminal statuses for an evaluation run. +TERMINAL_STATUSES = {"success", "failure", "errors", "cancelled"} + + +def create_simple_evaluation(authed_api, *, name, data, flags=None) -> dict: + """Create (and auto-start) a simple evaluation. Returns the evaluation dict.""" + payload = {"name": name, "data": data} + if flags is not None: + payload["flags"] = flags + response = authed_api( + "POST", + "/simple/evaluations/", + json={"evaluation": payload}, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["count"] == 1, body + return body["evaluation"] + + +def wait_for_run_terminal(authed_api, run_id, *, max_retries=20): + """Poll the run until it reaches a terminal status. Returns the final response.""" + return wait_for_response( + authed_api, + "GET", + f"/evaluations/runs/{run_id}", + condition_fn=lambda r: ( + r.status_code == 200 + and (r.json().get("run") or {}).get("status") in TERMINAL_STATUSES + ), + max_retries=max_retries, + ) + + +def fetch_run(authed_api, run_id) -> dict: + """Return the run dict.""" + response = authed_api("GET", f"/evaluations/runs/{run_id}") + assert response.status_code == 200, response.text + return response.json().get("run") or {} + + +def query_scenarios(authed_api, run_id): + """Return the list of scenarios for a run.""" + response = authed_api( + "POST", + "/evaluations/scenarios/query", + json={"scenario": {"run_id": str(run_id)}}, + ) + assert response.status_code == 200, response.text + return response.json().get("scenarios", []) + + +def query_metrics(authed_api, run_id): + """Return the list of metrics for a run.""" + response = authed_api( + "POST", + "/evaluations/metrics/query", + json={"metrics": {"run_id": str(run_id)}}, + ) + assert response.status_code == 200, response.text + return response.json().get("metrics", []) + + +def wait_for_metrics( + authed_api, run_id, *, expected_count=1, condition=None, max_retries=20 +): + """Poll metrics until they satisfy a condition, then return them. + + A run can flip to a terminal status a beat before its metric rows are + written, and the worker writes them incrementally per scenario, so tests + poll here rather than reading once. By default waits for `expected_count` + rows; pass `condition` (a callable over the metrics list) to wait for a + settled shape (e.g. the global aggregate reaching its final count). + """ + if condition is None: + + def condition(metrics): + return len(metrics) >= expected_count + + wait_for_response( + authed_api, + "POST", + "/evaluations/metrics/query", + json={"metrics": {"run_id": str(run_id)}}, + condition_fn=lambda r: ( + r.status_code == 200 and condition(r.json().get("metrics", [])) + ), + max_retries=max_retries, + ) + return query_metrics(authed_api, run_id) + + +# - lifecycle mutations -------------------------------------------------------- + + +def start_evaluation(authed_api, evaluation_id): + """Re-start (re-dispatch) an existing simple evaluation.""" + response = authed_api("POST", f"/simple/evaluations/{evaluation_id}/start") + assert response.status_code == 200, response.text + return response.json().get("evaluation") or {} + + +def close_run(authed_api, run_id): + """Close (lock) a run.""" + response = authed_api("POST", f"/evaluations/runs/{run_id}/close") + assert response.status_code == 200, response.text + return response.json() + + +def open_run(authed_api, run_id): + """Open (unlock) a run.""" + response = authed_api("POST", f"/evaluations/runs/{run_id}/open") + assert response.status_code == 200, response.text + return response.json() + + +def fetch_default_queue(authed_api, run_id) -> dict: + """Return the run's default queue dict (or {} if none).""" + response = authed_api("GET", f"/evaluations/runs/{run_id}/queues/default") + assert response.status_code == 200, response.text + return response.json().get("queue") or {} + + +def create_queue(authed_api, run_id, *, is_default=False, name=None) -> dict: + """Create a queue on a run. Defaults to a NON-default queue. + + Returns the created queue dict. + """ + queue = {"run_id": run_id, "flags": {"is_default": is_default}} + if name: + queue["name"] = name + response = authed_api("POST", "/evaluations/queues/", json={"queues": [queue]}) + assert response.status_code == 200, response.text + return response.json()["queues"][0] diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_closed_run_guard.py b/api/oss/tests/pytest/acceptance/evaluations/test_closed_run_guard.py new file mode 100644 index 0000000000..470ce334bf --- /dev/null +++ b/api/oss/tests/pytest/acceptance/evaluations/test_closed_run_guard.py @@ -0,0 +1,112 @@ +""" +Closed-run mutation guard. + +Closing a run (`POST /runs/{id}/close`) locks it: subsequent content mutations +(edit run, create scenario/result, create/refresh metrics) raise +`EvaluationClosedConflict`, surfaced as HTTP 409. Opening the run again +(`POST /runs/{id}/open`) lifts the lock. + +Queue archive/unarchive is intentionally NOT blocked on a closed run — that is a +worklist action, covered by test_evaluation_flows_modify.py. +""" + +from uuid import uuid4 + + +def _create_run(authed_api, steps=None) -> dict: + payload = {"name": f"closed-guard-{uuid4()}"} + if steps is not None: + payload["data"] = {"steps": steps} + response = authed_api("POST", "/evaluations/runs/", json={"runs": [payload]}) + assert response.status_code == 200, response.text + return response.json()["runs"][0] + + +def _create_scenario(authed_api, run_id) -> str: + response = authed_api( + "POST", "/evaluations/scenarios/", json={"scenarios": [{"run_id": run_id}]} + ) + assert response.status_code == 200, response.text + return response.json()["scenarios"][0]["id"] + + +def _close(authed_api, run_id): + response = authed_api("POST", f"/evaluations/runs/{run_id}/close") + assert response.status_code == 200, response.text + + +def _open(authed_api, run_id): + response = authed_api("POST", f"/evaluations/runs/{run_id}/open") + assert response.status_code == 200, response.text + + +class TestClosedRunGuard: + def test_close_sets_is_closed_then_open_clears_it(self, authed_api): + run = _create_run(authed_api) + run_id = run["id"] + + _close(authed_api, run_id) + fetched = authed_api("GET", f"/evaluations/runs/{run_id}").json()["run"] + assert fetched["flags"]["is_closed"] is True + + _open(authed_api, run_id) + fetched = authed_api("GET", f"/evaluations/runs/{run_id}").json()["run"] + assert fetched["flags"]["is_closed"] is False + + def test_edit_closed_run_is_blocked(self, authed_api): + run = _create_run(authed_api) + run_id = run["id"] + _close(authed_api, run_id) + + response = authed_api( + "PATCH", + f"/evaluations/runs/{run_id}", + json={"run": {"id": run_id, "name": "renamed-after-close"}}, + ) + assert response.status_code == 409, response.text + + def test_create_scenario_in_closed_run_is_blocked(self, authed_api): + run = _create_run(authed_api) + run_id = run["id"] + _close(authed_api, run_id) + + response = authed_api( + "POST", "/evaluations/scenarios/", json={"scenarios": [{"run_id": run_id}]} + ) + assert response.status_code == 409, response.text + + def test_create_result_in_closed_run_is_blocked(self, authed_api): + run = _create_run(authed_api) + run_id = run["id"] + scenario_id = _create_scenario(authed_api, run_id) + _close(authed_api, run_id) + + response = authed_api( + "POST", + "/evaluations/results/", + json={ + "results": [ + { + "step_key": "input", + "repeat_idx": 0, + "scenario_id": scenario_id, + "run_id": run_id, + } + ] + }, + ) + assert response.status_code == 409, response.text + + def test_edit_allowed_again_after_open(self, authed_api): + run = _create_run(authed_api) + run_id = run["id"] + _close(authed_api, run_id) + _open(authed_api, run_id) + + response = authed_api( + "PATCH", + f"/evaluations/runs/{run_id}", + json={"run": {"id": run_id, "name": "renamed-after-open"}}, + ) + assert response.status_code == 200, response.text + assert response.json()["run"]["name"] == "renamed-after-open" diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_default_queue_lifecycle.py b/api/oss/tests/pytest/acceptance/evaluations/test_default_queue_lifecycle.py new file mode 100644 index 0000000000..f6ff4b152f --- /dev/null +++ b/api/oss/tests/pytest/acceptance/evaluations/test_default_queue_lifecycle.py @@ -0,0 +1,160 @@ +""" +Default-queue reconciliation state machine. + +`_reconcile_default_queue` runs after every create/edit of a run and brings the +default queue + `run.flags.is_queue` in line with the run graph: + + should_exist = EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS or has_human + + required + missing -> create default queue + required + archived -> unarchive default queue + required + active -> no-op + not-required + active -> archive default queue + +`is_queue == has_human AND an active default queue exists`. + +Driven through `/evaluations/runs/` (create + PATCH steps) — no worker needed; +reconciliation is synchronous in the create/edit path. +""" + +from uuid import uuid4 + + +def _input_step(): + return { + "key": "input", + "type": "input", + "origin": "custom", + "references": {"testset": {"id": str(uuid4())}}, + } + + +def _human_step(): + return { + "key": "annotation-human", + "type": "annotation", + "origin": "human", + "references": {"evaluator_revision": {"id": str(uuid4())}}, + "inputs": [{"key": "input"}], + } + + +def _auto_step(): + return { + "key": "annotation-auto", + "type": "annotation", + "origin": "auto", + "references": {"evaluator_revision": {"id": str(uuid4())}}, + "inputs": [{"key": "input"}], + } + + +def _create_run(authed_api, steps): + response = authed_api( + "POST", + "/evaluations/runs/", + json={"runs": [{"name": "dq-lifecycle", "data": {"steps": steps}}]}, + ) + assert response.status_code == 200, response.text + return response.json()["runs"][0] + + +def _patch_steps(authed_api, run, steps): + response = authed_api( + "PATCH", + f"/evaluations/runs/{run['id']}", + json={"run": {"id": run["id"], "name": run["name"], "data": {"steps": steps}}}, + ) + assert response.status_code == 200, response.text + return response.json()["run"] + + +def _default_queue(authed_api, run_id) -> dict: + response = authed_api("GET", f"/evaluations/runs/{run_id}/queues/default") + assert response.status_code == 200, response.text + return response.json().get("queue") or {} + + +class TestDefaultQueueLifecycle: + def test_required_missing_creates_default_queue(self, authed_api): + # required + missing -> CREATE. A run with a human step gets a default + # queue and is_queue=True at creation. + run = _create_run(authed_api, steps=[_input_step(), _human_step()]) + assert run["flags"]["has_human"] is True + assert run["flags"]["is_queue"] is True + + dq = _default_queue(authed_api, run["id"]) + assert dq.get("id"), dq + assert dq["flags"]["is_default"] is True + + def test_not_required_active_archives_default_queue(self, authed_api): + # not-required + active -> ARCHIVE. Dropping the human step removes queue + # eligibility; the default queue is archived and is_queue flips False. + run = _create_run(authed_api, steps=[_input_step(), _human_step()]) + assert _default_queue(authed_api, run["id"]).get("id") + + edited = _patch_steps(authed_api, run, steps=[_input_step(), _auto_step()]) + assert edited["flags"]["has_human"] is False + assert edited["flags"]["is_queue"] is False + # active default queue is gone (archived) + assert _default_queue(authed_api, run["id"]) == {} + + def test_required_archived_unarchives_default_queue(self, authed_api): + # required + archived -> UNARCHIVE. Re-adding a human step after the + # default queue was archived brings it back and re-sets is_queue=True. + run = _create_run(authed_api, steps=[_input_step(), _human_step()]) + # drop human -> archive + _patch_steps(authed_api, run, steps=[_input_step(), _auto_step()]) + assert _default_queue(authed_api, run["id"]) == {} + + # re-add human -> unarchive + re_added = _patch_steps(authed_api, run, steps=[_input_step(), _human_step()]) + assert re_added["flags"]["has_human"] is True + assert re_added["flags"]["is_queue"] is True + dq = _default_queue(authed_api, run["id"]) + assert dq.get("id"), dq + + def test_required_active_is_noop(self, authed_api): + # required + active -> NO-OP. Editing a run that keeps its human step + # (changing an unrelated step) keeps the SAME default queue active. + run = _create_run(authed_api, steps=[_input_step(), _human_step()]) + dq_before = _default_queue(authed_api, run["id"]) + assert dq_before.get("id") + + # edit graph but keep the human step (add an auto step alongside) + edited = _patch_steps( + authed_api, run, steps=[_input_step(), _human_step(), _auto_step()] + ) + assert edited["flags"]["is_queue"] is True + dq_after = _default_queue(authed_api, run["id"]) + assert dq_after.get("id") == dq_before["id"], (dq_before, dq_after) + + def test_non_human_run_has_no_default_queue(self, authed_api): + # auto-only run: not a queue, no default queue created. + run = _create_run(authed_api, steps=[_input_step(), _auto_step()]) + assert run["flags"]["has_human"] is False + assert run["flags"]["is_queue"] is False + assert _default_queue(authed_api, run["id"]) == {} + + def test_planted_is_queue_flag_is_reconciled_back(self, authed_api): + # is_queue is service-derived: an edit that tries to PLANT is_queue=True + # on a run that is not queue-eligible (no human step) must be reconciled + # back to False rather than persisted as-is. + run = _create_run(authed_api, steps=[_input_step(), _auto_step()]) + assert run["flags"]["is_queue"] is False + + response = authed_api( + "PATCH", + f"/evaluations/runs/{run['id']}", + json={ + "run": { + "id": run["id"], + "name": run["name"], + "flags": {"is_queue": True}, + "data": {"steps": [_input_step(), _auto_step()]}, + } + }, + ) + assert response.status_code == 200, response.text + assert response.json()["run"]["flags"]["is_queue"] is False + assert _default_queue(authed_api, run["id"]) == {} diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_default_queue_policy.py b/api/oss/tests/pytest/acceptance/evaluations/test_default_queue_policy.py new file mode 100644 index 0000000000..c4d87bfa1e --- /dev/null +++ b/api/oss/tests/pytest/acceptance/evaluations/test_default_queue_policy.py @@ -0,0 +1,195 @@ +""" +Default-queue policy + validation. + +A *default* queue (`flags.is_default=True`) is a system-managed worklist tied to +a run's human evaluators. It must not carry per-user/scenario/step/batch filters, +must not be demoted to a normal queue, and must not be hard-deleted (archive +instead). These are enforced in the service layer with typed domain exceptions +(`DefaultQueueDataInvalid` -> 422, `DefaultQueueDemotionForbidden` / +`DefaultQueueDeletionForbidden` -> 409). + +These tests drive the `/evaluations/queues/` HTTP surface directly — no worker. +""" + +from uuid import uuid4 + +import pytest + + +def _create_run(authed_api, name=None) -> str: + response = authed_api( + "POST", + "/evaluations/runs/", + json={"runs": [{"name": name or f"run-{uuid4()}"}]}, + ) + assert response.status_code == 200, response.text + return response.json()["runs"][0]["id"] + + +def _create_queue(authed_api, run_id, *, flags=None, data=None, name=None): + queue = {"run_id": run_id} + if name: + queue["name"] = name + if flags is not None: + queue["flags"] = flags + if data is not None: + queue["data"] = data + return authed_api("POST", "/evaluations/queues/", json={"queues": [queue]}) + + +class TestDefaultQueueDataValidation: + # A default queue may not carry scenario/step/assignment/batch filters. + + @pytest.mark.parametrize( + "field,value", + [ + ("user_ids", [[str(uuid4())]]), + ("scenario_ids", [str(uuid4())]), + ("step_keys", ["annotation"]), + ("batch_size", 5), + ("batch_offset", 0), + ], + ids=["user_ids", "scenario_ids", "step_keys", "batch_size", "batch_offset"], + ) + def test_default_queue_with_filter_field_is_rejected( + self, authed_api, field, value + ): + run_id = _create_run(authed_api) + response = _create_queue( + authed_api, + run_id, + flags={"is_default": True}, + data={field: value}, + ) + # DefaultQueueDataInvalid -> 422 + assert response.status_code == 422, response.text + + def test_non_default_queue_may_carry_filter_fields(self, authed_api): + run_id = _create_run(authed_api) + response = _create_queue( + authed_api, + run_id, + flags={"is_default": False}, + data={"step_keys": ["annotation"], "batch_size": 5}, + ) + assert response.status_code == 200, response.text + assert response.json()["count"] == 1 + + def test_default_queue_without_filters_is_accepted(self, authed_api): + run_id = _create_run(authed_api) + response = _create_queue( + authed_api, + run_id, + flags={"is_default": True}, + data={}, + ) + assert response.status_code == 200, response.text + assert response.json()["count"] == 1 + + +class TestDefaultQueueDemotionForbidden: + def _create_default_queue(self, authed_api): + run_id = _create_run(authed_api) + resp = _create_queue(authed_api, run_id, flags={"is_default": True}) + assert resp.status_code == 200, resp.text + return resp.json()["queues"][0]["id"] + + def test_demoting_default_queue_is_forbidden(self, authed_api): + queue_id = self._create_default_queue(authed_api) + response = authed_api( + "PATCH", + f"/evaluations/queues/{queue_id}", + json={"queue": {"id": queue_id, "flags": {"is_default": False}}}, + ) + # DefaultQueueDemotionForbidden -> 409 + assert response.status_code == 409, response.text + + def test_keeping_default_flag_on_edit_is_allowed(self, authed_api): + queue_id = self._create_default_queue(authed_api) + response = authed_api( + "PATCH", + f"/evaluations/queues/{queue_id}", + json={ + "queue": { + "id": queue_id, + "name": "renamed", + "flags": {"is_default": True}, + } + }, + ) + assert response.status_code == 200, response.text + assert response.json()["queue"]["name"] == "renamed" + + def test_promoting_normal_queue_to_default_is_allowed(self, authed_api): + run_id = _create_run(authed_api) + resp = _create_queue(authed_api, run_id, flags={"is_default": False}) + assert resp.status_code == 200, resp.text + queue_id = resp.json()["queues"][0]["id"] + response = authed_api( + "PATCH", + f"/evaluations/queues/{queue_id}", + json={"queue": {"id": queue_id, "flags": {"is_default": True}}}, + ) + assert response.status_code == 200, response.text + + +class TestDefaultQueueDeletionForbidden: + def test_hard_deleting_default_queue_is_forbidden(self, authed_api): + run_id = _create_run(authed_api) + resp = _create_queue(authed_api, run_id, flags={"is_default": True}) + assert resp.status_code == 200, resp.text + queue_id = resp.json()["queues"][0]["id"] + + response = authed_api("DELETE", f"/evaluations/queues/{queue_id}") + # DefaultQueueDeletionForbidden -> 409 + assert response.status_code == 409, response.text + + def test_hard_deleting_normal_queue_is_allowed(self, authed_api): + run_id = _create_run(authed_api) + resp = _create_queue(authed_api, run_id, flags={"is_default": False}) + assert resp.status_code == 200, resp.text + queue_id = resp.json()["queues"][0]["id"] + + response = authed_api("DELETE", f"/evaluations/queues/{queue_id}") + assert response.status_code == 200, response.text + assert response.json()["count"] == 1 + + def test_bulk_delete_including_default_queue_is_forbidden(self, authed_api): + run_id_1 = _create_run(authed_api) + run_id_2 = _create_run(authed_api) + normal = _create_queue(authed_api, run_id_1, flags={"is_default": False}) + default = _create_queue(authed_api, run_id_2, flags={"is_default": True}) + normal_id = normal.json()["queues"][0]["id"] + default_id = default.json()["queues"][0]["id"] + + response = authed_api( + "DELETE", + "/evaluations/queues/", + json={"queue_ids": [normal_id, default_id]}, + ) + assert response.status_code == 409, response.text + + +class TestDefaultQueueUniqueness: + # At most one default queue per run for the run's lifetime (active OR + # archived), enforced by the partial unique index + # ux_evaluation_queues_default_per_run; create_queue surfaces the unique + # violation as an EntityCreationConflict (409). + + def test_second_default_queue_for_same_run_is_rejected(self, authed_api): + run_id = _create_run(authed_api) + first = _create_queue(authed_api, run_id, flags={"is_default": True}) + assert first.status_code == 200, first.text + assert first.json()["count"] == 1 + + second = _create_queue(authed_api, run_id, flags={"is_default": True}) + # The second default queue is rejected as a creation conflict. + assert second.json().get("count", 0) == 0, second.text + + def test_default_queues_allowed_across_different_runs(self, authed_api): + run_id_1 = _create_run(authed_api) + run_id_2 = _create_run(authed_api) + first = _create_queue(authed_api, run_id_1, flags={"is_default": True}) + second = _create_queue(authed_api, run_id_2, flags={"is_default": True}) + assert first.json()["count"] == 1, first.text + assert second.json()["count"] == 1, second.text diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_modify.py b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_modify.py new file mode 100644 index 0000000000..8ccad17a4a --- /dev/null +++ b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_modify.py @@ -0,0 +1,55 @@ +""" +End-to-end evaluation FLOW tests for modify-after-run lifecycle: +run to completion, then mutate, then re-verify. + +Uses the deterministic LLM-free `agenta:custom:mock:v0` workflow. See +`_flow_helpers.py`. + +Covers: + - re-start a finished batch run -> it re-dispatches and finalizes again + (validates the finalization rule: a (re)dispatched run resets to running, then + the slice re-finalizes it). +""" + +from ._flow_helpers import ( + create_mock_application, + create_mock_evaluator, + create_testset, + create_simple_evaluation, + wait_for_run_terminal, + start_evaluation, +) + + +class TestEvaluationModifyFlows: + def test_restart_finished_batch_run_re_dispatches_and_finalizes_again( + self, authed_api + ): + # ARRANGE: run a batch eval to terminal success ------------------------ + testset = create_testset(authed_api) + application = create_mock_application(authed_api, key="echo") + evaluator = create_mock_evaluator(authed_api, key="pass") + evaluation = create_simple_evaluation( + authed_api, + name="flow-restart-finished", + data={ + "testset_steps": [testset["revision_id"]], + "application_steps": [application["revision_id"]], + "evaluator_steps": {evaluator["revision_id"]: "auto"}, + }, + ) + run_id = evaluation["id"] + first = wait_for_run_terminal(authed_api, run_id) + assert first.json()["run"]["status"] == "success" + # ---------------------------------------------------------------------- + + # ACT: re-start the finished run --------------------------------------- + start_evaluation(authed_api, run_id) + # ---------------------------------------------------------------------- + + # ASSERT: it re-dispatches and reaches a terminal status again --------- + # (activation resets status=RUNNING, then the slice + # finalizes it. We assert the end state is terminal again.) + final = wait_for_run_terminal(authed_api, run_id) + assert final.json()["run"]["status"] == "success", final.json()["run"] + # ---------------------------------------------------------------------- diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_run.py b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_run.py new file mode 100644 index 0000000000..c56780976b --- /dev/null +++ b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_run.py @@ -0,0 +1,147 @@ +""" +End-to-end evaluation FLOW tests: trigger a run, wait for the worker to finish, +and verify the run reached a terminal state with scenarios/metrics. + +These run through the real worker + services container with NO LLM and NO code +sandbox, using the deterministic `agenta:custom:mock:v0` workflow for both the +application and the evaluator. See `_flow_helpers.py`. + +Coverage spans the worker dispatch topologies (see +`api/oss/src/core/evaluations/runtime/topology.py`): + - batch_testset (testset -> mock app -> mock auto-evaluator) -> success + - batch_invocation (testset -> mock app) -> success + - live_query (live + query -> evaluator) -> stays running + - batch_query (query -> evaluator) -> xfail +""" + +import time + +from ._flow_helpers import ( + create_mock_application, + create_mock_evaluator, + create_query, + create_testset, + create_simple_evaluation, + wait_for_run_terminal, + fetch_run, + query_scenarios, +) + + +class TestEvaluationRunFlows: + def test_testset_to_mock_app_to_mock_evaluator_runs_to_success(self, authed_api): + # ARRANGE -------------------------------------------------------------- + testset = create_testset(authed_api) + application = create_mock_application(authed_api, key="echo") + evaluator = create_mock_evaluator(authed_api, key="pass") + # ---------------------------------------------------------------------- + + # ACT ------------------------------------------------------------------ + evaluation = create_simple_evaluation( + authed_api, + name="flow-testset-app-evaluator", + data={ + "testset_steps": [testset["revision_id"]], + "application_steps": [application["revision_id"]], + "evaluator_steps": {evaluator["revision_id"]: "auto"}, + }, + ) + run_id = evaluation["id"] + # ---------------------------------------------------------------------- + + # ASSERT --------------------------------------------------------------- + final = wait_for_run_terminal(authed_api, run_id) + run = final.json()["run"] + assert run["status"] == "success", run + # terminal run is no longer active + assert (run.get("flags") or {}).get("is_active") is False, run + + scenarios = query_scenarios(authed_api, run_id) + # the default testset has 2 testcases -> 2 scenarios + assert len(scenarios) == 2, scenarios + # ---------------------------------------------------------------------- + + def test_testset_to_mock_app_batch_invocation_runs_to_success(self, authed_api): + # batch_invocation: testset -> application, no evaluator. + # ARRANGE -------------------------------------------------------------- + testset = create_testset(authed_api) + application = create_mock_application(authed_api, key="echo") + # ---------------------------------------------------------------------- + + # ACT ------------------------------------------------------------------ + evaluation = create_simple_evaluation( + authed_api, + name="flow-testset-app-only", + data={ + "testset_steps": [testset["revision_id"]], + "application_steps": [application["revision_id"]], + }, + ) + run_id = evaluation["id"] + # ---------------------------------------------------------------------- + + # ASSERT --------------------------------------------------------------- + final = wait_for_run_terminal(authed_api, run_id) + run = final.json()["run"] + assert run["status"] == "success", run + assert (run.get("flags") or {}).get("is_active") is False, run + # ---------------------------------------------------------------------- + + def test_live_query_evaluation_stays_running_and_active(self, authed_api): + # live_query never finalizes via the slice (update_run_status=False); it + # stays running/active so the scheduler keeps polling. This guards that + # the finalization fix does NOT finalize live evals. + # ARRANGE -------------------------------------------------------------- + query = create_query(authed_api, trace_type="invocation") + evaluator = create_mock_evaluator(authed_api, key="pass") + # ---------------------------------------------------------------------- + + # ACT ------------------------------------------------------------------ + evaluation = create_simple_evaluation( + authed_api, + name="flow-live-query-evaluator", + data={ + "query_steps": [query["revision_id"]], + "evaluator_steps": {evaluator["revision_id"]: "auto"}, + }, + flags={"is_live": True}, + ) + run_id = evaluation["id"] + # ---------------------------------------------------------------------- + + # ASSERT --------------------------------------------------------------- + # give the worker a moment, then assert it has NOT been finalized + time.sleep(5) + run = fetch_run(authed_api, run_id) + assert run.get("status") == "running", run + assert (run.get("flags") or {}).get("is_active") is True, run + # ---------------------------------------------------------------------- + + def test_batch_query_to_evaluator_runs_to_success(self, authed_api): + # batch_query: query -> evaluator (no app, not live). Resolves traces via + # the query filter; with no matching traces in the test env it resolves + # zero items and still finalizes to success (batch query + # runs finalize, unlike live query runs). + # ARRANGE -------------------------------------------------------------- + query = create_query(authed_api, trace_type="invocation") + evaluator = create_mock_evaluator(authed_api, key="pass") + # ---------------------------------------------------------------------- + + # ACT ------------------------------------------------------------------ + evaluation = create_simple_evaluation( + authed_api, + name="flow-batch-query-evaluator", + data={ + "query_steps": [query["revision_id"]], + "evaluator_steps": {evaluator["revision_id"]: "auto"}, + }, + ) + run_id = evaluation["id"] + # ---------------------------------------------------------------------- + + # ASSERT --------------------------------------------------------------- + final = wait_for_run_terminal(authed_api, run_id) + run = final.json()["run"] + assert run["status"] == "success", run + assert (run.get("flags") or {}).get("is_active") is False, run + # ---------------------------------------------------------------------- diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_basics.py b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_basics.py index 93a3684dc0..1d602b133c 100644 --- a/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_basics.py +++ b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_basics.py @@ -43,7 +43,7 @@ def test_create_evaluation_metrics(self, authed_api): assert response["count"] == 1 # ---------------------------------------------------------------------- - def test_edit_evaluation_metrics(self, authed_api): + def test_upsert_evaluation_metrics(self, authed_api): # ARRANGE -------------------------------------------------------------- runs = [ {"name": "test_edit_evaluation_metrics"}, @@ -89,7 +89,7 @@ def test_edit_evaluation_metrics(self, authed_api): metrics[0]["data"]["boolean_metric"] = False response = authed_api( - "PATCH", + "POST", "/evaluations/metrics/", json={"metrics": metrics}, ) @@ -214,32 +214,25 @@ def test_fetch_evaluation_metric(self, authed_api): # ---------------------------------------------------------------------- # ACT ------------------------------------------------------------------ - # NOTE: GET /metrics/{id} does not exist, use POST /metrics/query response = authed_api( - "POST", - "/evaluations/metrics/query", - json={ - "metrics": { - "run_id": run_id, - }, - }, + "GET", + f"/evaluations/metrics/{metric['id']}", ) # ---------------------------------------------------------------------- # ASSERT --------------------------------------------------------------- assert response.status_code == 200 response = response.json() - assert response["count"] >= 1 - metric_ids = [m["id"] for m in response["metrics"]] - assert metric["id"] in metric_ids - matched = [m for m in response["metrics"] if m["id"] == metric["id"]][0] + assert response["count"] == 1 + matched = response["metrics"][0] + assert matched["id"] == metric["id"] assert matched["data"]["integer_metric"] == 42 assert matched["data"]["float_metric"] == 3.14 assert matched["data"]["string_metric"] == "test" assert matched["data"]["boolean_metric"] is True # ---------------------------------------------------------------------- - def test_edit_evaluation_metric(self, authed_api): + def test_upsert_evaluation_metric(self, authed_api): # ARRANGE -------------------------------------------------------------- runs = [ {"name": "test_edit_evaluation_metric"}, @@ -286,7 +279,7 @@ def test_edit_evaluation_metric(self, authed_api): metric["data"]["boolean_metric"] = False response = authed_api( - "PATCH", + "POST", "/evaluations/metrics/", json={"metrics": [metric]}, ) @@ -345,8 +338,7 @@ def test_delete_evaluation_metric(self, authed_api): # ACT ------------------------------------------------------------------ response = authed_api( "DELETE", - "/evaluations/metrics/", - json={"metrics_ids": [metric["id"]]}, + f"/evaluations/metrics/{metric['id']}", ) # ---------------------------------------------------------------------- @@ -360,8 +352,7 @@ def test_delete_evaluation_metric(self, authed_api): # ACT ------------------------------------------------------------------ response = authed_api( "DELETE", - "/evaluations/metrics/", - json={"metrics_ids": [metric["id"]]}, + f"/evaluations/metrics/{metric['id']}", ) # ---------------------------------------------------------------------- @@ -370,3 +361,45 @@ def test_delete_evaluation_metric(self, authed_api): response = response.json() assert response["count"] == 0 # ---------------------------------------------------------------------- + + def test_invalid_metric_shape_is_rejected(self, authed_api): + # A metric with BOTH scenario_id and timestamp set matches none of the + # three partial unique indexes (global/variational/temporal). It must be + # rejected (422), not silently dropped. + # ARRANGE -------------------------------------------------------------- + runs = [ + {"name": "test_invalid_metric_shape"}, + ] + + response = authed_api( + "POST", + "/evaluations/runs/", + json={"runs": runs}, + ) + + assert response.status_code == 200 + + run_id = response.json()["runs"][0]["id"] + + metrics = [ + { + "run_id": run_id, + "scenario_id": run_id, # any UUID; the shape check runs first + "timestamp": "2026-01-01T00:00:00+00:00", + "status": "success", + "data": {"integer_metric": 1}, + }, + ] + # ---------------------------------------------------------------------- + + # ACT ------------------------------------------------------------------ + response = authed_api( + "POST", + "/evaluations/metrics/", + json={"metrics": metrics}, + ) + # ---------------------------------------------------------------------- + + # ASSERT --------------------------------------------------------------- + assert response.status_code == 422, response.text + # ---------------------------------------------------------------------- diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_flow.py b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_flow.py new file mode 100644 index 0000000000..bc9034c81f --- /dev/null +++ b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_flow.py @@ -0,0 +1,230 @@ +""" +End-to-end metric VALUE tests as part of the evaluation flow. + +The other metric suites cover plumbing — CRUD (`_basics`), refresh dispatch +routing on bare runs (`_refresh`), and query filters (`_queries`). None of them +assert that a REAL run produces correctly COMPUTED metrics. These tests close +that gap: they run a deterministic flow (mock app + mock evaluator, no LLM), +then assert the computed metric values and the three metric scopes. + +Metric model recap (see `simplified-interface.md` and +`EvaluationsService._refresh_metrics`): + - metric `data` is keyed by step (`{step_key: {...aggregates...}}`) + - variational = per scenario (scenario_id set), across its repeats + - temporal = per interval (timestamp/interval set) + - global = whole run (scenario_id null) — used for batch evaluations + +Scope: BATCH path (variational + global). TEMPORAL metrics belong to LIVE evals, +re-evaluated by a cron that POSTs `/admin/evaluations/runs/refresh` over a narrow +(~1 min) window. Asserting them from a test depends on trace-ingestion timing vs. +that window, which is too brittle to pin here — temporal is left to the live +infrastructure; the live path's "stays running/active" is covered in +`test_evaluation_flows_run.py`. +""" + +from ._flow_helpers import ( + create_mock_application, + create_mock_evaluator, + create_testset, + create_simple_evaluation, + wait_for_run_terminal, + query_scenarios, + wait_for_metrics, +) + + +def _metric_for(metrics, *, scenario_id): + """Return the single metric row matching scenario_id (None = global).""" + matches = [m for m in metrics if m.get("scenario_id") == scenario_id] + return matches[0] if matches else None + + +# The evaluator score aggregate lives under the evaluator step, keyed by the +# canonical output path. Computed metrics are numeric distributions with +# count/mean/sum (see `_refresh_metrics` -> analytics buckets). +SCORE_PATH = "attributes.ag.data.outputs.score" + + +def _score_aggregate(metric): + """Return the evaluator score aggregate ({count, mean, sum, ...}) or None. + + Walks the step-keyed metric `data` for the evaluator step that carries the + score path, without coupling to the evaluator step's generated slug. + """ + for step_key, step_metrics in (metric.get("data") or {}).items(): + if SCORE_PATH in (step_metrics or {}): + return step_metrics[SCORE_PATH] + return None + + +def _refresh_global_score(authed_api, run_id, *, expect_count, attempts=8): + """Drive the whole-run metric refresh and return the global score aggregate. + + The refresh is synchronous, but the run can report "terminal" a beat before + every scenario's result cell is durably persisted under parallel load, so the + first refresh may aggregate fewer cells. Re-refresh (short, ~15s total) until + the global score reaches `expect_count`, then return it; fail fast otherwise. + """ + import time + + last = None + for attempt in range(attempts): + response = authed_api( + "POST", + "/evaluations/metrics/refresh", + json={"metrics": {"run_id": str(run_id)}}, + ) + assert response.status_code == 200, response.text + global_metric = _metric_for(response.json()["metrics"], scenario_id=None) + last = _score_aggregate(global_metric) if global_metric else None + if last and last.get("count") == expect_count: + return last + time.sleep(min(0.5 * (2**attempt), 4.0)) + raise AssertionError( + f"global score did not reach count={expect_count} (last={last})" + ) + + +class TestEvaluationMetricsFlow: + def test_batch_run_produces_global_and_variational_metrics(self, authed_api): + # A deterministic batch run: 2 testcases -> echo app -> score evaluator + # returning a fixed score. After the run finishes, the worker has + # computed metrics. Assert both the global (whole-run) metric and the + # per-scenario (variational) metrics exist, keyed by the evaluator step, + # carrying the score. + # ARRANGE -------------------------------------------------------------- + testset = create_testset(authed_api) # 2 testcases + application = create_mock_application(authed_api, key="echo") + evaluator = create_mock_evaluator( + authed_api, key="score", kwargs={"score": 0.7, "threshold": 0.5} + ) + # ---------------------------------------------------------------------- + + # ACT ------------------------------------------------------------------ + evaluation = create_simple_evaluation( + authed_api, + name="flow-metrics-batch", + data={ + "testset_steps": [testset["revision_id"]], + "application_steps": [application["revision_id"]], + "evaluator_steps": {evaluator["revision_id"]: "auto"}, + }, + ) + run_id = evaluation["id"] + # ---------------------------------------------------------------------- + + # ASSERT --------------------------------------------------------------- + final = wait_for_run_terminal(authed_api, run_id) + assert final.json()["run"]["status"] == "success", final.json() + + scenarios = query_scenarios(authed_api, run_id) + assert len(scenarios) == 2, scenarios + scenario_ids = {s["id"] for s in scenarios} + + # global metric: drive the whole-run refresh ourselves (synchronous), + # but POLL it — the run can report "terminal" a beat before BOTH + # scenarios' result cells are durably persisted under parallel load, so a + # single refresh may aggregate count=1. Poll the refresh until it sees + # both (count=2), capped to ~15s so a stuck env fails fast. + global_score = _refresh_global_score(authed_api, run_id, expect_count=2) + # the score evaluator returns 0.7 for both testcases. + assert global_score["count"] == 2, global_score + assert global_score["mean"] == 0.7, global_score + assert global_score["min"] == 0.7 and global_score["max"] == 0.7, global_score + + # variational metrics: one per scenario, written per-scenario during the + # run (so they are present once the run is terminal), scenario_id set, + # keyed by step. Each scenario has one repeat, so its score mean is 0.7. + metrics = wait_for_metrics( + authed_api, + run_id, + condition=lambda ms: ( + len([m for m in ms if m.get("scenario_id") in scenario_ids]) == 2 + ), + max_retries=6, + ) + variational = [m for m in metrics if m.get("scenario_id") in scenario_ids] + assert len(variational) == 2, f"expected 2 variational metrics: {variational}" + for metric in variational: + score = _score_aggregate(metric) + assert score is not None, f"variational metric has no score: {metric}" + assert score["mean"] == 0.7, (metric.get("scenario_id"), score) + # ---------------------------------------------------------------------- + + def test_refresh_recomputes_metrics_for_a_finished_run(self, authed_api): + # `refresh` is the standalone metrics op (decoupled from process). After + # a run finishes, re-invoking refresh over the run scope must recompute + # and return the same metrics (idempotent over a stable tensor). + # ARRANGE -------------------------------------------------------------- + testset = create_testset(authed_api) + application = create_mock_application(authed_api, key="echo") + evaluator = create_mock_evaluator( + authed_api, key="score", kwargs={"score": 1.0, "threshold": 0.5} + ) + evaluation = create_simple_evaluation( + authed_api, + name="flow-metrics-refresh", + data={ + "testset_steps": [testset["revision_id"]], + "application_steps": [application["revision_id"]], + "evaluator_steps": {evaluator["revision_id"]: "auto"}, + }, + ) + run_id = evaluation["id"] + wait_for_run_terminal(authed_api, run_id) + # ---------------------------------------------------------------------- + + # ACT ------------------------------------------------------------------ + response = authed_api( + "POST", + "/evaluations/metrics/refresh", + json={"metrics": {"run_id": str(run_id)}}, + ) + # ---------------------------------------------------------------------- + + # ASSERT --------------------------------------------------------------- + assert response.status_code == 200, response.text + body = response.json() + # refresh over a finished run returns the recomputed metric(s). + assert body["count"] >= 1, body + refreshed = body["metrics"] + # the recomputed global metric carries the evaluator score (mean 1.0). + global_metric = _metric_for(refreshed, scenario_id=None) + assert global_metric is not None, f"no global metric after refresh: {refreshed}" + score = _score_aggregate(global_metric) + assert score is not None and score["mean"] == 1.0, score + # ---------------------------------------------------------------------- + + def test_failing_evaluator_metrics_reflect_zero_score(self, authed_api): + # The computed metric must reflect the evaluator's actual output, not a + # constant. A `fail` evaluator scores 0.0 — the metric value must differ + # from the passing case, proving the value is computed, not hardcoded. + # ARRANGE -------------------------------------------------------------- + testset = create_testset(authed_api) + application = create_mock_application(authed_api, key="echo") + evaluator = create_mock_evaluator(authed_api, key="fail") + evaluation = create_simple_evaluation( + authed_api, + name="flow-metrics-fail", + data={ + "testset_steps": [testset["revision_id"]], + "application_steps": [application["revision_id"]], + "evaluator_steps": {evaluator["revision_id"]: "auto"}, + }, + ) + run_id = evaluation["id"] + # ---------------------------------------------------------------------- + + # ACT ------------------------------------------------------------------ + final = wait_for_run_terminal(authed_api, run_id) + # ---------------------------------------------------------------------- + + # ASSERT --------------------------------------------------------------- + assert final.json()["run"]["status"] == "success", final.json() + # drive + poll the global refresh (see batch test) until both scenarios + # are aggregated, rather than waiting for the worker's lagging refresh. + score = _refresh_global_score(authed_api, run_id, expect_count=2) + # the score aggregate is the failing value (mean 0.0), not 1 — proving + # the metric reflects the evaluator's real output, not a constant. + assert score["mean"] == 0.0 and score["max"] == 0.0, score + # ---------------------------------------------------------------------- diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_refresh.py b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_refresh.py new file mode 100644 index 0000000000..a6fcdccef2 --- /dev/null +++ b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_refresh.py @@ -0,0 +1,121 @@ +""" +Metrics refresh dispatch (`POST /evaluations/metrics/refresh`). + +`EvaluationsService.refresh_metrics` is a dispatcher over the request body: it +fans out to `_refresh_metrics` per run / scenario / timestamp depending on which +of `run_ids` / `run_id` / `scenario_ids` / `timestamps` are present, and returns +the union of the per-call metrics. These tests pin the dispatch contract and the +early-return paths that do not require traces, evaluator schemas, or the worker: + + - no `run_id`/`run_ids` -> [] (count 0) + - unknown `run_id` -> [] (run not found) + - unknown ids in `run_ids` -> [] (each run not found) + - run with no metrics-bearing steps -> [] (nothing to refresh) + +The trace-backed / schema-inference branches of `_refresh_metrics` are covered +by the flow tests that exercise a full run; here we lock the routing shape and +the "nothing to do" outcomes. +""" + +from uuid import uuid4 + + +def _create_run(authed_api, name=None) -> str: + response = authed_api( + "POST", + "/evaluations/runs/", + json={"runs": [{"name": name or f"run-{uuid4()}"}]}, + ) + assert response.status_code == 200, response.text + return response.json()["runs"][0]["id"] + + +def _refresh(authed_api, **metrics): + return authed_api( + "POST", + "/evaluations/metrics/refresh", + json={"metrics": metrics}, + ) + + +class TestRefreshMetricsDispatch: + def test_refresh_with_empty_body_returns_no_metrics(self, authed_api): + # No run_id and no run_ids -> the dispatcher short-circuits to []. + response = _refresh(authed_api) + assert response.status_code == 200, response.text + body = response.json() + assert body["count"] == 0 + assert body["metrics"] == [] + + def test_refresh_unknown_run_id_returns_no_metrics(self, authed_api): + # A syntactically valid but non-existent run resolves to no run, so + # _refresh_metrics returns [] rather than erroring. + response = _refresh(authed_api, run_id=str(uuid4())) + assert response.status_code == 200, response.text + body = response.json() + assert body["count"] == 0 + assert body["metrics"] == [] + + def test_refresh_unknown_run_ids_returns_no_metrics(self, authed_api): + # The run_ids branch loops per id; each unknown id contributes nothing. + response = _refresh(authed_api, run_ids=[str(uuid4()), str(uuid4())]) + assert response.status_code == 200, response.text + body = response.json() + assert body["count"] == 0 + assert body["metrics"] == [] + + def test_refresh_run_without_steps_returns_no_metrics(self, authed_api): + # A bare run (created with just a name) has no metrics-bearing steps, so + # there is nothing to refresh. + run_id = _create_run(authed_api) + response = _refresh(authed_api, run_id=run_id) + assert response.status_code == 200, response.text + body = response.json() + assert body["count"] == 0 + assert body["metrics"] == [] + + def test_refresh_scenario_ids_branch_returns_no_metrics(self, authed_api): + # With run_id set and scenario_ids present, the dispatcher loops the + # scenario_ids branch. Unknown scenarios on a bare run yield nothing. + run_id = _create_run(authed_api) + response = _refresh( + authed_api, + run_id=run_id, + scenario_ids=[str(uuid4()), str(uuid4())], + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["count"] == 0 + assert body["metrics"] == [] + + def test_refresh_timestamps_branch_returns_no_metrics(self, authed_api): + # With run_id set and timestamps present (and no scenario_ids), the + # dispatcher loops the timestamps branch. A bare run yields nothing. + run_id = _create_run(authed_api) + response = _refresh( + authed_api, + run_id=run_id, + timestamps=["2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z"], + interval=3600, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["count"] == 0 + assert body["metrics"] == [] + + def test_refresh_run_ids_takes_precedence_over_run_id(self, authed_api): + # When both run_ids and run_id are present, the dispatcher uses run_ids + # (the first branch). Two bare runs in run_ids both yield nothing, and a + # would-be conflicting run_id is ignored — count stays 0 either way, but + # this exercises the run_ids branch with real (empty) runs. + run_id_1 = _create_run(authed_api) + run_id_2 = _create_run(authed_api) + response = _refresh( + authed_api, + run_ids=[run_id_1, run_id_2], + run_id=str(uuid4()), + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["count"] == 0 + assert body["metrics"] == [] diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_step_removal.py b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_step_removal.py new file mode 100644 index 0000000000..ad93e77f1c --- /dev/null +++ b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_step_removal.py @@ -0,0 +1,179 @@ +from uuid import uuid4 + + +def _input_step(): + return { + "key": "input", + "type": "input", + "origin": "custom", + "references": {"testset": {"id": str(uuid4())}}, + } + + +def _human_annotation_step(): + return { + "key": "annotation", + "type": "annotation", + "origin": "human", + "references": {"evaluator_revision": {"id": str(uuid4())}}, + "inputs": [{"key": "input"}], + } + + +def _create_run(authed_api, steps): + response = authed_api( + "POST", + "/evaluations/runs/", + json={"runs": [{"name": "step removal run", "data": {"steps": steps}}]}, + ) + assert response.status_code == 200 + return response.json()["runs"][0] + + +def _create_scenario(authed_api, run_id): + response = authed_api( + "POST", + "/evaluations/scenarios/", + json={"scenarios": [{"run_id": run_id}]}, + ) + assert response.status_code == 200 + return response.json()["scenarios"][0] + + +def _create_result(authed_api, run_id, scenario_id, step_key): + response = authed_api( + "POST", + "/evaluations/results/", + json={ + "results": [ + { + "step_key": step_key, + "repeat_idx": 0, + "scenario_id": scenario_id, + "run_id": run_id, + } + ] + }, + ) + assert response.status_code == 200 + return response.json()["results"][0] + + +def _query_results(authed_api, **result): + response = authed_api( + "POST", + "/evaluations/results/query", + json={"result": result}, + ) + assert response.status_code == 200 + return response.json() + + +def _patch_steps(authed_api, run, steps): + response = authed_api( + "PATCH", + f"/evaluations/runs/{run['id']}", + json={ + "run": { + "id": run["id"], + "name": run["name"], + "data": {"steps": steps}, + } + }, + ) + assert response.status_code == 200 + return response.json() + + +class TestEvaluationStepRemoval: + def test_edit_dropping_a_non_input_step_prunes_only_its_cells(self, authed_api): + # ARRANGE -------------------------------------------------------------- + run = _create_run( + authed_api, + steps=[_input_step(), _human_annotation_step()], + ) + run_id = run["id"] + scenario = _create_scenario(authed_api, run_id) + scenario_id = scenario["id"] + + # Cells on both the input step and the annotation step. + _create_result(authed_api, run_id, scenario_id, "input") + _create_result(authed_api, run_id, scenario_id, "annotation") + + assert _query_results(authed_api, run_id=run_id)["count"] == 2 + # ---------------------------------------------------------------------- + + # ACT — edit the run, dropping the annotation step from the graph. + response = _patch_steps(authed_api, run, steps=[_input_step()]) + # ---------------------------------------------------------------------- + + # ASSERT --------------------------------------------------------------- + assert response["count"] == 1 + edited = response["run"] + assert [s["key"] for s in edited["data"]["steps"]] == ["input"] + # has_human dropped, queue eligibility gone. + assert edited["flags"]["has_human"] is False + assert edited["flags"]["is_queue"] is False + + # The annotation cell is pruned; the input cell survives. + remaining = _query_results(authed_api, run_id=run_id) + assert remaining["count"] == 1 + assert remaining["results"][0]["step_key"] == "input" + + # The scenario still has an input cell, so it is NOT orphaned. + scenarios = authed_api( + "POST", + "/evaluations/scenarios/query", + json={"scenario": {"run_id": run_id}}, + ).json() + assert scenarios["count"] == 1 + # ---------------------------------------------------------------------- + + def test_edit_dropping_the_input_step_prunes_orphan_scenarios(self, authed_api): + # ARRANGE -------------------------------------------------------------- + run = _create_run(authed_api, steps=[_input_step()]) + run_id = run["id"] + scenario = _create_scenario(authed_api, run_id) + scenario_id = scenario["id"] + + # The scenario's only cell is on the input step. + _create_result(authed_api, run_id, scenario_id, "input") + assert _query_results(authed_api, run_id=run_id)["count"] == 1 + # ---------------------------------------------------------------------- + + # ACT — drop the input step entirely (empty graph). + response = _patch_steps(authed_api, run, steps=[]) + # ---------------------------------------------------------------------- + + # ASSERT --------------------------------------------------------------- + assert response["count"] == 1 + assert response["run"]["data"]["steps"] in (None, []) + + # Cells pruned. + assert _query_results(authed_api, run_id=run_id)["count"] == 0 + + # The scenario was sourced only from the removed input step -> orphaned + # and removed. + scenarios = authed_api( + "POST", + "/evaluations/scenarios/query", + json={"scenario": {"run_id": run_id}}, + ).json() + assert scenarios["count"] == 0 + # ---------------------------------------------------------------------- + + def test_create_run_does_not_prune(self, authed_api): + # Create is "edit from an empty graph": the shared reconcile path runs, + # but prior_step_keys is empty so nothing is pruned and the graph is + # preserved verbatim. + run = _create_run( + authed_api, + steps=[_input_step(), _human_annotation_step()], + ) + assert [s["key"] for s in run["data"]["steps"]] == ["input", "annotation"] + assert run["flags"]["has_human"] is True + + # A human run gets an active default queue on create. + response = authed_api("GET", f"/evaluations/runs/{run['id']}/queues/default") + assert response.status_code == 200 + assert response.json()["count"] == 1 diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_steps_basics.py b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_steps_basics.py index b10691df1a..4b1979d614 100644 --- a/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_steps_basics.py +++ b/api/oss/tests/pytest/acceptance/evaluations/test_evaluation_steps_basics.py @@ -140,7 +140,7 @@ def test_fetch_evaluation_results(self, authed_api, mock_data): assert step_key_3 in step_keys # ---------------------------------------------------------------------- - def test_edit_evaluation_results(self, authed_api, mock_data): + def test_upsert_evaluation_results(self, authed_api, mock_data): # ARRANGE -------------------------------------------------------------- run_id = mock_data["runs"][0]["id"] scenario_id = mock_data["scenarios"][0]["id"] @@ -181,7 +181,6 @@ def test_edit_evaluation_results(self, authed_api, mock_data): assert response["count"] == 3 results = response["results"] - result_ids = [r["id"] for r in results] # ---------------------------------------------------------------------- # ACT ------------------------------------------------------------------ @@ -190,7 +189,7 @@ def test_edit_evaluation_results(self, authed_api, mock_data): results[2]["status"] = "cancelled" response = authed_api( - "PATCH", + "POST", "/evaluations/results/", json={"results": results}, ) @@ -200,10 +199,10 @@ def test_edit_evaluation_results(self, authed_api, mock_data): assert response.status_code == 200 response = response.json() assert response["count"] == 3 - patched = {r["id"]: r for r in response["results"]} - assert patched[result_ids[0]]["status"] == "success" - assert patched[result_ids[1]]["status"] == "failure" - assert patched[result_ids[2]]["status"] == "cancelled" + patched = {r["step_key"]: r for r in response["results"]} + assert patched[step_key_1]["status"] == "success" + assert patched[step_key_2]["status"] == "failure" + assert patched[step_key_3]["status"] == "cancelled" # ---------------------------------------------------------------------- def test_delete_evaluation_results(self, authed_api, mock_data): @@ -310,7 +309,7 @@ def test_fetch_evaluation_result(self, authed_api, mock_data): assert response["result"]["id"] == result_id # ---------------------------------------------------------------------- - def test_edit_evaluation_result(self, authed_api, mock_data): + def test_upsert_single_evaluation_result(self, authed_api, mock_data): # ARRANGE -------------------------------------------------------------- run_id = mock_data["runs"][0]["id"] authed_api("POST", f"/evaluations/runs/{run_id}/open") @@ -345,9 +344,9 @@ def test_edit_evaluation_result(self, authed_api, mock_data): result["status"] = "success" response = authed_api( - "PATCH", - f"/evaluations/results/{result_id}", - json={"result": result}, + "POST", + "/evaluations/results/", + json={"results": [result]}, ) # ---------------------------------------------------------------------- @@ -355,8 +354,8 @@ def test_edit_evaluation_result(self, authed_api, mock_data): assert response.status_code == 200 response = response.json() assert response["count"] == 1 - assert response["result"]["id"] == result_id - assert response["result"]["status"] == "success" + assert response["results"][0]["id"] == result_id + assert response["results"][0]["status"] == "success" # ---------------------------------------------------------------------- def test_delete_evaluation_result(self, authed_api, mock_data): diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_simple_evaluations_workflows.py b/api/oss/tests/pytest/acceptance/evaluations/test_simple_evaluations_workflows.py index 8f994024b1..a932a26c21 100644 --- a/api/oss/tests/pytest/acceptance/evaluations/test_simple_evaluations_workflows.py +++ b/api/oss/tests/pytest/acceptance/evaluations/test_simple_evaluations_workflows.py @@ -63,6 +63,44 @@ def _create_simple_evaluator(authed_api) -> dict: return response.json()["evaluator"] +def _create_simple_testset(authed_api) -> dict: + slug = uuid4().hex + response = authed_api( + "POST", + "/simple/testsets/", + json={ + "testset": { + "slug": f"testset-{slug}", + "name": f"Testset {slug}", + "data": { + "testcases": [ + {"data": {"input": "hello", "expected": "world"}}, + {"data": {"input": "hola", "expected": "mundo"}}, + ] + }, + } + }, + ) + assert response.status_code == 200 + return response.json()["testset"] + + +def _create_simple_application(authed_api) -> dict: + slug = uuid4().hex + response = authed_api( + "POST", + "/simple/applications/", + json={ + "application": { + "slug": f"application-{slug}", + "name": f"Application {slug}", + } + }, + ) + assert response.status_code == 200 + return response.json()["application"] + + class TestSimpleEvaluationsWorkflowReferences: def test_create_live_simple_evaluation_accepts_query_and_evaluator_revision_ids( self, authed_api @@ -124,3 +162,146 @@ def test_create_live_simple_evaluation_rejects_non_invocation_query_revision( body = response.json() assert body["count"] == 0 assert body.get("evaluation") is None + + def test_create_batch_inference_evaluation_preserves_flags_repeats_and_refs( + self, authed_api + ): + testset = _create_simple_testset(authed_api) + application = _create_simple_application(authed_api) + + response = authed_api( + "POST", + "/simple/evaluations/", + json={ + "evaluation": { + "name": "batch-inference-setup", + "flags": { + "is_cached": True, + "is_split": True, + }, + "data": { + "testset_steps": [testset["revision_id"]], + "application_steps": [application["revision_id"]], + "repeats": 3, + }, + } + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body["count"] == 1 + + evaluation = body["evaluation"] + assert evaluation["flags"]["is_cached"] is True + assert evaluation["flags"]["is_split"] is True + assert evaluation["data"]["repeats"] == 3 + assert set(evaluation["data"]["testset_steps"].keys()) == { + testset["revision_id"] + } + assert set(evaluation["data"]["application_steps"].keys()) == { + application["revision_id"] + } + + def test_create_batch_evaluation_preserves_application_evaluator_matrix( + self, authed_api + ): + testset = _create_simple_testset(authed_api) + application = _create_simple_application(authed_api) + evaluator = _create_simple_evaluator(authed_api) + + response = authed_api( + "POST", + "/simple/evaluations/", + json={ + "evaluation": { + "name": "batch-application-evaluator-setup", + "flags": { + "is_cached": False, + "is_split": False, + }, + "data": { + "testset_steps": {testset["revision_id"]: "custom"}, + "application_steps": {application["revision_id"]: "custom"}, + "evaluator_steps": {evaluator["revision_id"]: "auto"}, + "repeats": 2, + }, + } + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body["count"] == 1 + + evaluation = body["evaluation"] + assert evaluation["flags"]["is_cached"] is False + assert evaluation["flags"]["is_split"] is False + assert evaluation["data"]["repeats"] == 2 + assert evaluation["data"]["testset_steps"] == {testset["revision_id"]: "custom"} + assert evaluation["data"]["application_steps"] == { + application["revision_id"]: "custom" + } + assert evaluation["data"]["evaluator_steps"] == { + evaluator["revision_id"]: "auto" + } + + def test_create_testset_to_evaluator_evaluation_does_not_fail_setup( + self, authed_api + ): + testset = _create_simple_testset(authed_api) + evaluator = _create_simple_evaluator(authed_api) + + response = authed_api( + "POST", + "/simple/evaluations/", + json={ + "evaluation": { + "name": "testset-to-evaluator-setup", + "data": { + "testset_steps": [testset["revision_id"]], + "evaluator_steps": [evaluator["revision_id"]], + }, + } + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body["count"] == 1 + evaluation = body["evaluation"] + assert set(evaluation["data"]["testset_steps"].keys()) == { + testset["revision_id"] + } + assert set(evaluation["data"]["evaluator_steps"].keys()) == { + evaluator["revision_id"] + } + + def test_create_query_to_application_evaluation_does_not_fail_setup( + self, authed_api + ): + query = _create_simple_query(authed_api) + application = _create_simple_application(authed_api) + + response = authed_api( + "POST", + "/simple/evaluations/", + json={ + "evaluation": { + "name": "query-to-application-setup", + "data": { + "query_steps": [query["revision_id"]], + "application_steps": [application["revision_id"]], + }, + } + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body["count"] == 1 + evaluation = body["evaluation"] + assert set(evaluation["data"]["query_steps"].keys()) == {query["revision_id"]} + assert set(evaluation["data"]["application_steps"].keys()) == { + application["revision_id"] + } diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py b/api/oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py index b473190dd7..2e35212764 100644 --- a/api/oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py +++ b/api/oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py @@ -169,7 +169,7 @@ def test_create_simple_queue_from_queries(self, authed_api): data = response.json() assert data["count"] == 1 queue = data["queue"] - assert queue["data"]["kind"] == "traces" + assert queue["data"]["kind"] == "queries" assert queue["data"]["queries"] == [query_revision_id] def test_create_simple_queue_from_testsets(self, authed_api): @@ -194,9 +194,162 @@ def test_create_simple_queue_from_testsets(self, authed_api): data = response.json() assert data["count"] == 1 queue = data["queue"] - assert queue["data"]["kind"] == "testcases" + assert queue["data"]["kind"] == "testsets" assert queue["data"]["testsets"] == [testset_revision_id] + def test_create_source_backed_queue_preserves_repeats_and_assignments( + self, authed_api + ): + evaluator_revision_id = _create_evaluator(authed_api) + testset_revision_id = _create_testset(authed_api) + user_id_1 = str(uuid4()) + user_id_2 = str(uuid4()) + + response = authed_api( + "POST", + "/simple/queues/", + json={ + "queue": { + "name": "testset-backed-queue-with-repeats", + "data": { + "testsets": [testset_revision_id], + "evaluators": {evaluator_revision_id: "human"}, + "repeats": 2, + "assignments": [[user_id_1], [user_id_2]], + }, + } + }, + ) + + assert response.status_code == 200 + data = response.json() + assert data["count"] == 1 + queue = data["queue"] + assert queue["data"]["kind"] == "testsets" + assert queue["data"]["testsets"] == [testset_revision_id] + assert queue["data"]["repeats"] == 2 + assert queue["data"]["assignments"] == [[user_id_1], [user_id_2]] + + def test_source_backed_queue_with_bare_evaluator_list_is_human_queue( + self, authed_api + ): + # a source-backed queue created with a bare evaluator list (no + # explicit origin) must default the evaluator to origin="human", so the + # backing run becomes a real queue (has_human -> is_queue). Before the + # fix the source builder defaulted to "custom", leaving is_queue=False + # and the queue unparseable (empty response). + evaluator_revision_id = _create_evaluator(authed_api) + query_revision_id = _create_query(authed_api) + + response = authed_api( + "POST", + "/simple/queues/", + json={ + "queue": { + "name": "query-backed-human-queue", + "data": { + "queries": [query_revision_id], + "evaluators": [evaluator_revision_id], + }, + } + }, + ) + + assert response.status_code == 200 + data = response.json() + assert data["count"] == 1 + queue = data["queue"] + assert queue["data"]["kind"] == "queries" + run_id = queue["run_id"] + + # The backing run must be a real human queue. + run_response = authed_api("GET", f"/evaluations/runs/{run_id}") + assert run_response.status_code == 200 + run = run_response.json()["run"] + assert run["flags"]["has_human"] is True + assert run["flags"]["is_queue"] is True + + def test_simple_queue_rejects_evaluator_dicts_with_no_human(self, authed_api): + # Simple-queue constraint: a queue CAN contain non-human evaluators but + # CANNOT resolve to ZERO human evaluators. A human evaluator is one that + # is origin-less (defaults to human) or explicitly "human". So an + # explicit dict whose values are all non-human is rejected, regardless + # of whether they are auto, custom, or a mix of the two. The underlying + # evaluation run has no such restriction. + eval_a = _create_evaluator(authed_api) + eval_b = _create_evaluator(authed_api) + + non_human_evaluator_specs = [ + {eval_a: "auto"}, # all auto + {eval_a: "custom"}, # all custom + {eval_a: "auto", eval_b: "custom"}, # mix of non-human origins + ] + + for evaluators in non_human_evaluator_specs: + testset_revision_id = _create_testset(authed_api) + response = authed_api( + "POST", + "/simple/queues/", + json={ + "queue": { + "name": "testset-backed-no-human", + "data": { + "testsets": [testset_revision_id], + "evaluators": evaluators, + }, + } + }, + ) + + assert response.status_code == 422, ( + f"expected reject for evaluators={evaluators}, got {response.status_code}" + ) + assert "at least one human evaluator" in response.text + + def test_simple_queue_allows_human_mixed_with_non_human_evaluators( + self, authed_api + ): + # The "at least one human" rule allows a mix: a human alongside any + # non-human origin (auto or custom) is a valid human queue. The explicit + # non-human origin is honored on the underlying run. + for non_human_origin, expected_flag in ( + ("auto", "has_auto"), + ("custom", "has_custom"), + ): + human_evaluator_id = _create_evaluator(authed_api) + other_evaluator_id = _create_evaluator(authed_api) + testset_revision_id = _create_testset(authed_api) + + response = authed_api( + "POST", + "/simple/queues/", + json={ + "queue": { + "name": f"testset-backed-human-plus-{non_human_origin}", + "data": { + "testsets": [testset_revision_id], + "evaluators": { + human_evaluator_id: "human", + other_evaluator_id: non_human_origin, + }, + }, + } + }, + ) + + assert response.status_code == 200 + data = response.json() + assert data["count"] == 1 + run_id = data["queue"]["run_id"] + + run_response = authed_api("GET", f"/evaluations/runs/{run_id}") + assert run_response.status_code == 200 + run = run_response.json()["run"] + # Both origins honored; the human presence makes it a real queue. + assert run["flags"]["has_human"] is True + assert run["flags"][expected_flag] is True + assert run["flags"]["is_queue"] is True + def test_create_simple_queue_without_evaluators_returns_empty(self, authed_api): # ACT ------------------------------------------------------------------ response = authed_api( @@ -283,7 +436,7 @@ def test_add_testcases_rejects_testset_backed_source_queue(self, authed_api): == "Cannot add testcases directly to a source-backed queue. Create a direct testcases queue instead." ) - def test_create_simple_queue_rejects_kind_with_queries(self, authed_api): + def test_create_simple_queue_rejects_resolved_kind_with_queries(self, authed_api): evaluator_revision_id = _create_evaluator(authed_api) query_revision_id = _create_query(authed_api) @@ -302,11 +455,32 @@ def test_create_simple_queue_rejects_kind_with_queries(self, authed_api): ) assert response.status_code == 422 - assert ( - "simple queue source must not include kind alongside queries or testsets" - in response.text + assert "query-backed queues must use kind='queries'" in response.text + + def test_create_simple_queue_rejects_mixed_query_and_testset_sources( + self, authed_api + ): + evaluator_revision_id = _create_evaluator(authed_api) + query_revision_id = _create_query(authed_api) + testset_revision_id = _create_testset(authed_api) + + response = authed_api( + "POST", + "/simple/queues/", + json={ + "queue": { + "data": { + "queries": [query_revision_id], + "testsets": [testset_revision_id], + "evaluators": [evaluator_revision_id], + }, + } + }, ) + assert response.status_code == 422 + assert "simple queue source must be either queries or testsets" in response.text + def test_create_simple_queue_with_assignments(self, authed_api): # ARRANGE -------------------------------------------------------------- evaluator_revision_id = _create_evaluator(authed_api) diff --git a/api/oss/tests/pytest/acceptance/evaluations/test_tensor_slice_endpoints.py b/api/oss/tests/pytest/acceptance/evaluations/test_tensor_slice_endpoints.py new file mode 100644 index 0000000000..3c64e2f3bc --- /dev/null +++ b/api/oss/tests/pytest/acceptance/evaluations/test_tensor_slice_endpoints.py @@ -0,0 +1,433 @@ +"""Acceptance tests for the tensor slice HTTP endpoints (PR: unify eval loops). + +Covers the coordinate-addressed ops over EXISTING scenarios exposed on the +simple-evaluations router: + + POST /simple/evaluations/{id}/process -> dispatch re-execution (async, 202-ish) + POST /simple/evaluations/{id}/probe -> read result cells in a slice + POST /simple/evaluations/{id}/populate -> bulk-write result cells + +These assert the HTTP contract (routes exist, accept the TensorSlice-shaped +body, return the right envelope) without depending on live-LLM execution, which +is covered elsewhere and is inherently flaky. +""" + +from uuid import uuid4 + + +def _create_simple_evaluator(authed_api) -> dict: + slug = uuid4().hex + response = authed_api( + "POST", + "/simple/evaluators/", + json={ + "evaluator": { + "slug": f"evaluator-{slug}", + "name": f"Evaluator {slug}", + "data": { + "schemas": { + "outputs": { + "type": "object", + "properties": {"score": {"type": "number"}}, + "required": ["score"], + "additionalProperties": False, + } + } + }, + } + }, + ) + assert response.status_code == 200 + return response.json()["evaluator"] + + +def _create_simple_testset(authed_api) -> dict: + slug = uuid4().hex + response = authed_api( + "POST", + "/simple/testsets/", + json={ + "testset": { + "slug": f"testset-{slug}", + "name": f"Testset {slug}", + "data": { + "testcases": [ + {"data": {"input": "hello", "expected": "world"}}, + ] + }, + } + }, + ) + assert response.status_code == 200 + return response.json()["testset"] + + +def _create_testset_evaluator_evaluation(authed_api) -> dict: + """A run with a testset input + auto evaluator (no application/LLM).""" + testset = _create_simple_testset(authed_api) + evaluator = _create_simple_evaluator(authed_api) + response = authed_api( + "POST", + "/simple/evaluations/", + json={ + "evaluation": { + "name": f"tensor-slice-{uuid4().hex[:8]}", + "flags": {"is_cached": False, "is_split": False}, + "data": { + "testset_steps": {testset["revision_id"]: "custom"}, + "evaluator_steps": {evaluator["revision_id"]: "auto"}, + "repeats": 1, + }, + } + }, + ) + assert response.status_code == 200, response.text + return response.json()["evaluation"] + + +def _input_step_key(authed_api, run_id) -> str: + """Read the run's actual input step key (e.g. 'testset-'). + + Step keys are derived from the revision slug at build time, so they can't be + hardcoded — fetch the run and read its graph. + """ + response = authed_api("GET", f"/evaluations/runs/{run_id}") + assert response.status_code == 200, response.text + run = response.json().get("run") or {} + steps = (run.get("data") or {}).get("steps") or [] + input_steps = [s for s in steps if s.get("type") == "input"] + assert input_steps, f"run {run_id} has no input step: {steps}" + return input_steps[0]["key"] + + +class TestTensorSliceEndpoints: + def test_probe_returns_results_envelope_for_empty_slice(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + # Probe a coordinate that addresses nothing yet -> empty results envelope. + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/probe", + json={"scenario_ids": [str(uuid4())]}, + ) + + assert response.status_code == 200, response.text + body = response.json() + assert "count" in body + assert "results" in body + assert body["results"] == [] + assert body["count"] == 0 + + def test_probe_accepts_full_coordinate_body(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/probe", + json={ + "scenario_ids": [str(uuid4())], + "step_keys": ["evaluator-step"], + "repeat_idxs": [0], + }, + ) + + assert response.status_code == 200, response.text + body = response.json() + assert isinstance(body["results"], list) + + def test_probe_accepts_empty_body(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/probe", + json={}, + ) + + assert response.status_code == 200, response.text + assert "results" in response.json() + + def test_populate_writes_input_result_cells(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + scenario_id = str(uuid4()) + step_key = _input_step_key(authed_api, run_id) + + # Populate an input cell (the source identity) directly. + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/populate", + json={ + "results": [ + { + "run_id": run_id, + "scenario_id": scenario_id, + "step_key": step_key, + "repeat_idx": 0, + "status": "success", + } + ] + }, + ) + + assert response.status_code == 200, response.text + body = response.json() + assert "count" in body + assert "results" in body + assert body["count"] == len(body["results"]) + + def test_populate_accepts_empty_results(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/populate", + json={"results": []}, + ) + + assert response.status_code == 200, response.text + assert response.json()["results"] == [] + + def test_process_accepts_coordinate_slice_and_acknowledges(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + # Dispatch re-execution over a coordinate slice. Async dispatch -> the + # endpoint acknowledges acceptance with 202 and an empty body (it does + # not wait for execution). + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/process", + json={ + "scenario_ids": [str(uuid4())], + "step_keys": ["evaluator-step"], + # overwrite omitted -> defaults to False (fill-missing) + }, + ) + + assert response.status_code == 202, response.text + + def test_process_accepts_overwrite(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/process", + json={"overwrite": True}, + ) + + assert response.status_code == 202, response.text + + def test_process_round_trips_populate_then_probe(self, authed_api): + """add a scenario, populate its input cell, then probe it by coordinate.""" + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + step_key = _input_step_key(authed_api, run_id) + + # A result cell FKs to an existing scenario — you cannot populate into a + # scenario that does not exist. Add the scenario first via the height op + # (`/scenarios/add`), then populate its input cell. + created = authed_api( + "POST", + f"/simple/evaluations/{run_id}/scenarios/add", + json={"count": 1}, + ) + assert created.status_code == 200, created.text + scenario_id = created.json()["scenarios"][0]["id"] + + populate = authed_api( + "POST", + f"/simple/evaluations/{run_id}/populate", + json={ + "results": [ + { + "run_id": run_id, + "scenario_id": scenario_id, + "step_key": step_key, + "repeat_idx": 0, + "status": "success", + } + ] + }, + ) + assert populate.status_code == 200, populate.text + + probe = authed_api( + "POST", + f"/simple/evaluations/{run_id}/probe", + json={"scenario_ids": [scenario_id]}, + ) + assert probe.status_code == 200, probe.text + results = probe.json()["results"] + # the populated cell is now readable by coordinate + assert any( + str(r.get("scenario_id")) == scenario_id and r.get("step_key") == step_key + for r in results + ) + + +class TestGraphShapeEndpoints: + """The graph-shape ops: add/remove scenarios, add/remove steps, set repeats.""" + + def test_add_scenarios_returns_created_rows(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/scenarios/add", + json={"count": 3}, + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["count"] == 3 + assert len(body["scenarios"]) == 3 + + def test_add_scenarios_floors_timestamp_to_minute(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/scenarios/add", + json={"count": 1, "timestamp": "2026-06-02T14:23:47.500000Z"}, + ) + + assert response.status_code == 200, response.text + scenario = response.json()["scenarios"][0] + # floored to the minute (interval fixed at 1 minute server-side) + assert scenario["timestamp"].startswith("2026-06-02T14:23:00") + assert scenario["interval"] == 1 + + def test_remove_scenarios_returns_204(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + created = authed_api( + "POST", + f"/simple/evaluations/{run_id}/scenarios/add", + json={"count": 1}, + ) + assert created.status_code == 200, created.text + scenario_id = created.json()["scenarios"][0]["id"] + + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/scenarios/remove", + json={"scenario_ids": [scenario_id]}, + ) + + assert response.status_code == 204, response.text + + def test_prune_returns_204(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/prune", + json={"scenario_ids": [str(uuid4())]}, + ) + + assert response.status_code == 204, response.text + + def test_set_repeats_returns_run(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/repeats/set", + json={"repeats": 3}, + ) + + assert response.status_code == 200, response.text + run = response.json()["run"] + assert (run.get("data") or {}).get("repeats") == 3 + + +class TestSliceScoping: + """Path evaluation_id is enforced so a slice cannot reach another run.""" + + def test_populate_rejects_result_for_a_different_run(self, authed_api): + """UEL-017: a result whose run_id != path evaluation_id is rejected 400.""" + eval_a = _create_testset_evaluator_evaluation(authed_api) + eval_b = _create_testset_evaluator_evaluation(authed_api) + run_a, run_b = eval_a["id"], eval_b["id"] + step_key = _input_step_key(authed_api, run_b) + + # POST to run_a's populate, but the result targets run_b. + response = authed_api( + "POST", + f"/simple/evaluations/{run_a}/populate", + json={ + "results": [ + { + "run_id": run_b, + "scenario_id": str(uuid4()), + "step_key": step_key, + "repeat_idx": 0, + "status": "success", + } + ] + }, + ) + + assert response.status_code == 400, response.text + + def test_remove_scenarios_rejects_a_different_runs_scenario(self, authed_api): + """UEL-018: scenario_ids must belong to the path evaluation_id (else 400).""" + eval_a = _create_testset_evaluator_evaluation(authed_api) + eval_b = _create_testset_evaluator_evaluation(authed_api) + run_a, run_b = eval_a["id"], eval_b["id"] + + # A real scenario in run_b. + created = authed_api( + "POST", + f"/simple/evaluations/{run_b}/scenarios/add", + json={"count": 1}, + ) + assert created.status_code == 200, created.text + scenario_b = created.json()["scenarios"][0]["id"] + + # Try to delete run_b's scenario through run_a's path. + response = authed_api( + "POST", + f"/simple/evaluations/{run_a}/scenarios/remove", + json={"scenario_ids": [scenario_b]}, + ) + + assert response.status_code == 400, response.text + + # And the scenario still exists in run_b (was not deleted). + probe = authed_api( + "POST", + f"/simple/evaluations/{run_b}/probe", + json={"scenario_ids": [scenario_b]}, + ) + assert probe.status_code == 200, probe.text + + +class TestClosedRunReturns409: + """Write ops against a closed run return 409, not 500 (UEL-020).""" + + def test_add_scenarios_on_closed_run_returns_409(self, authed_api): + evaluation = _create_testset_evaluator_evaluation(authed_api) + run_id = evaluation["id"] + + closed = authed_api("POST", f"/evaluations/runs/{run_id}/close") + assert closed.status_code in (200, 204), closed.text + + response = authed_api( + "POST", + f"/simple/evaluations/{run_id}/scenarios/add", + json={"count": 1}, + ) + + assert response.status_code == 409, response.text diff --git a/api/oss/tests/pytest/acceptance/events/test_events_basics.py b/api/oss/tests/pytest/acceptance/events/test_events_basics.py deleted file mode 100644 index 3b3cfb81d6..0000000000 --- a/api/oss/tests/pytest/acceptance/events/test_events_basics.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Acceptance tests for the events query endpoint. - -Requires a running API. These tests verify the API contract (shape, status -codes, filtering) without making strong assumptions about how many events -exist in the system at query time — the events worker is a separate process. - -These exercise the *ungated* OSS contract, where a basic account may query -events freely. Under EE the same endpoint is gated on the AUDIT entitlement -(Hobby plan lacks it) and the VIEW_EVENTS permission, so a basic account is -correctly rejected with 403. That gated behaviour is covered by the EE suite -(ee/tests/pytest/acceptance/events/test_events_basics.py) using a business-plan -developer account, so this OSS suite is skipped on EE deployments. -""" - -import os - -import pytest -import requests - -from utils.constants import BASE_TIMEOUT - - -@pytest.fixture(scope="class") -def events_api(cls_account, ag_env): - if os.getenv("AGENTA_LICENSE") == "ee": - pytest.skip( - "Endpoint is plan/role-gated under EE; covered by the EE events suite." - ) - - credentials = cls_account["credentials"] - - def _request(method: str, endpoint: str, **kwargs): - headers = kwargs.pop("headers", {}) - headers.setdefault("Authorization", credentials) - return requests.request( - method=method, - url=f"{ag_env['api_url']}{endpoint}", - headers=headers, - timeout=BASE_TIMEOUT, - **kwargs, - ) - - yield _request - - -class TestEventsBasics: - def test_query_events_returns_valid_response(self, events_api): - """POST /events/query with an empty body returns a valid response.""" - # ACT ------------------------------------------------------------------ - response = events_api("POST", "/events/query", json={}) - # ---------------------------------------------------------------------- - - # ASSERT --------------------------------------------------------------- - assert response.status_code == 200 - body = response.json() - assert "count" in body - assert "events" in body - assert isinstance(body["events"], list) - assert body["count"] == len(body["events"]) - # ---------------------------------------------------------------------- - - def test_query_events_by_event_type(self, events_api): - """Filtering by event_type returns only matching events.""" - # ACT ------------------------------------------------------------------ - response = events_api( - "POST", - "/events/query", - json={ - "event": {"event_type": "environments.revisions.committed"}, - }, - ) - # ---------------------------------------------------------------------- - - # ASSERT --------------------------------------------------------------- - assert response.status_code == 200 - body = response.json() - assert "count" in body - assert "events" in body - # Every returned event must match the requested type - for event in body["events"]: - assert event["event_type"] == "environments.revisions.committed" - # ---------------------------------------------------------------------- - - def test_query_events_by_unknown_event_type(self, events_api): - """Filtering by UNKNOWN event_type returns only unknown events.""" - # ACT ------------------------------------------------------------------ - response = events_api( - "POST", - "/events/query", - json={ - "event": {"event_type": "unknown"}, - }, - ) - # ---------------------------------------------------------------------- - - # ASSERT --------------------------------------------------------------- - assert response.status_code == 200 - body = response.json() - for event in body["events"]: - assert event["event_type"] == "unknown" - # ---------------------------------------------------------------------- - - def test_query_events_with_windowing_limit(self, events_api): - """Windowing limit=1 returns at most 1 event.""" - # ACT ------------------------------------------------------------------ - response = events_api( - "POST", - "/events/query", - json={"windowing": {"limit": 1}}, - ) - # ---------------------------------------------------------------------- - - # ASSERT --------------------------------------------------------------- - assert response.status_code == 200 - body = response.json() - assert body["count"] <= 1 - assert len(body["events"]) <= 1 - # ---------------------------------------------------------------------- - - def test_query_events_invalid_event_type_rejected(self, events_api): - """Sending an unrecognised event_type value should be rejected (422).""" - # ACT ------------------------------------------------------------------ - response = events_api( - "POST", - "/events/query", - json={ - "event": {"event_type": "not.a.real.event.type"}, - }, - ) - # ---------------------------------------------------------------------- - - # ASSERT --------------------------------------------------------------- - assert response.status_code == 422 - # ---------------------------------------------------------------------- diff --git a/api/oss/tests/pytest/acceptance/loadables/test_loadable_strategies.py b/api/oss/tests/pytest/acceptance/loadables/test_loadable_strategies.py index 45f9c4b290..878a7fcc50 100644 --- a/api/oss/tests/pytest/acceptance/loadables/test_loadable_strategies.py +++ b/api/oss/tests/pytest/acceptance/loadables/test_loadable_strategies.py @@ -127,8 +127,20 @@ def mock_data(authed_api): # Ingestion is asynchronous; 202 Accepted assert response.status_code == 202, response.text - # Allow a moment for async ingestion to settle - time.sleep(1) + # Ingestion is asynchronous and its latency varies (especially under the + # parallel test run). Poll the trace store until all three ingested traces + # are queryable instead of relying on a fixed sleep, which is racy under + # load. Fast on the happy path; tolerant when ingestion is slow. + ids_param = ",".join(trace_ids) + deadline = time.time() + 15 + while True: + poll = authed_api("GET", f"/traces/?trace_ids={ids_param}") + visible = poll.json().get("traces") or poll.json().get("spans") or [] + if poll.status_code == 200 and len(visible) >= 3: + break + if time.time() >= deadline: + break + time.sleep(0.25) # -- Query revision (stores filtering + windowing) ---------------------- diff --git a/api/oss/tests/pytest/acceptance/workflows/test_workflow_revisions_basics.py b/api/oss/tests/pytest/acceptance/workflows/test_workflow_revisions_basics.py index 3cc70bb7f9..c911c42133 100644 --- a/api/oss/tests/pytest/acceptance/workflows/test_workflow_revisions_basics.py +++ b/api/oss/tests/pytest/acceptance/workflows/test_workflow_revisions_basics.py @@ -3,6 +3,36 @@ import pytest +def _commit_workflow_revision( + authed_api, + *, + workflow_id, + workflow_variant_id, + slug, + name, + description, + flags, + tags, + meta, +): + return authed_api( + "POST", + "/workflows/revisions/commit", + json={ + "workflow_revision": { + "slug": slug, + "name": name, + "description": description, + "flags": flags, + "tags": tags, + "meta": meta, + "workflow_id": workflow_id, + "workflow_variant_id": workflow_variant_id, + } + }, + ) + + @pytest.fixture(scope="class") def mock_data(authed_api): # ARRANGE -------------------------------------------------------------- @@ -139,33 +169,27 @@ def test_fetch_workflow_revision( # ARRANGE -------------------------------------------------------------- workflow_revision_slug = uuid4() - response = authed_api( - "POST", - "/workflows/revisions/commit", - json={ - "workflow_revision": { - "slug": f"workflow-revision-{workflow_revision_slug}", - "name": f"Workflow Revision {workflow_revision_slug}", - "description": "Workflow Revision Description", - "flags": { - "is_custom": False, - "is_evaluator": False, - "is_feedback": False, - }, - "tags": { - "tag1": "value1", - "tag2": "value2", - "tag3": "value3", - }, - "meta": { - "meta1": "value1", - "meta2": "value2", - "meta3": "value3", - }, - "data": {"parameters": {"key1": "value1"}}, - "workflow_id": mock_data["workflows"][0]["id"], - "workflow_variant_id": mock_data["workflow_variants"][0]["id"], - } + response = _commit_workflow_revision( + authed_api, + workflow_id=mock_data["workflows"][0]["id"], + workflow_variant_id=mock_data["workflow_variants"][0]["id"], + slug=f"workflow-revision-{workflow_revision_slug}", + name=f"Workflow Revision {workflow_revision_slug}", + description="Workflow Revision Description", + flags={ + "is_custom": False, + "is_evaluator": False, + "is_feedback": False, + }, + tags={ + "tag1": "value1", + "tag2": "value2", + "tag3": "value3", + }, + meta={ + "meta1": "value1", + "meta2": "value2", + "meta3": "value3", }, ) @@ -197,33 +221,27 @@ def test_edit_workflow_revision( # ARRANGE -------------------------------------------------------------- workflow_revision_slug = uuid4() - response = authed_api( - "POST", - "/workflows/revisions/commit", - json={ - "workflow_revision": { - "slug": f"workflow-revision-{workflow_revision_slug}", - "name": f"Workflow revision {workflow_revision_slug}", - "description": "Workflow revision Description", - "flags": { - "is_custom": False, - "is_evaluator": False, - "is_feedback": False, - }, - "tags": { - "tag1": "value1", - "tag2": "value2", - "tag3": "value3", - }, - "meta": { - "meta1": "value1", - "meta2": "value2", - "meta3": "value3", - }, - "data": {"parameters": {"key1": "value1"}}, - "workflow_id": mock_data["workflows"][0]["id"], - "workflow_variant_id": mock_data["workflow_variants"][0]["id"], - } + response = _commit_workflow_revision( + authed_api, + workflow_id=mock_data["workflows"][0]["id"], + workflow_variant_id=mock_data["workflow_variants"][0]["id"], + slug=f"workflow-revision-{workflow_revision_slug}", + name=f"Workflow revision {workflow_revision_slug}", + description="Workflow revision Description", + flags={ + "is_custom": False, + "is_evaluator": False, + "is_feedback": False, + }, + tags={ + "tag1": "value1", + "tag2": "value2", + "tag3": "value3", + }, + meta={ + "meta1": "value1", + "meta2": "value2", + "meta3": "value3", }, ) @@ -278,33 +296,27 @@ def test_archive_workflow_revision( # ARRANGE -------------------------------------------------------------- workflow_revision_slug = uuid4() - response = authed_api( - "POST", - "/workflows/revisions/commit", - json={ - "workflow_revision": { - "slug": f"workflow-revision-{workflow_revision_slug}", - "name": f"Workflow revision {workflow_revision_slug}", - "description": "Workflow revision Description", - "flags": { - "is_custom": False, - "is_evaluator": False, - "is_feedback": False, - }, - "tags": { - "tag1": "value1", - "tag2": "value2", - "tag3": "value3", - }, - "meta": { - "meta1": "value1", - "meta2": "value2", - "meta3": "value3", - }, - "data": {"parameters": {"key1": "value1"}}, - "workflow_id": mock_data["workflows"][0]["id"], - "workflow_variant_id": mock_data["workflow_variants"][0]["id"], - } + response = _commit_workflow_revision( + authed_api, + workflow_id=mock_data["workflows"][0]["id"], + workflow_variant_id=mock_data["workflow_variants"][0]["id"], + slug=f"workflow-revision-{workflow_revision_slug}", + name=f"Workflow revision {workflow_revision_slug}", + description="Workflow revision Description", + flags={ + "is_custom": False, + "is_evaluator": False, + "is_feedback": False, + }, + tags={ + "tag1": "value1", + "tag2": "value2", + "tag3": "value3", + }, + meta={ + "meta1": "value1", + "meta2": "value2", + "meta3": "value3", }, ) @@ -338,33 +350,27 @@ def test_unarchive_workflow_revision( # ARRANGE -------------------------------------------------------------- workflow_revision_slug = uuid4() - response = authed_api( - "POST", - "/workflows/revisions/commit", - json={ - "workflow_revision": { - "slug": f"workflow-revision-{workflow_revision_slug}", - "name": f"Workflow revision {workflow_revision_slug}", - "description": "Workflow revision Description", - "flags": { - "is_custom": False, - "is_evaluator": False, - "is_feedback": False, - }, - "tags": { - "tag1": "value1", - "tag2": "value2", - "tag3": "value3", - }, - "meta": { - "meta1": "value1", - "meta2": "value2", - "meta3": "value3", - }, - "data": {"parameters": {"key1": "value1"}}, - "workflow_id": mock_data["workflows"][0]["id"], - "workflow_variant_id": mock_data["workflow_variants"][0]["id"], - } + response = _commit_workflow_revision( + authed_api, + workflow_id=mock_data["workflows"][0]["id"], + workflow_variant_id=mock_data["workflow_variants"][0]["id"], + slug=f"workflow-revision-{workflow_revision_slug}", + name=f"Workflow revision {workflow_revision_slug}", + description="Workflow revision Description", + flags={ + "is_custom": False, + "is_evaluator": False, + "is_feedback": False, + }, + tags={ + "tag1": "value1", + "tag2": "value2", + "tag3": "value3", + }, + meta={ + "meta1": "value1", + "meta2": "value2", + "meta3": "value3", }, ) @@ -408,33 +414,27 @@ def test_commit_workflow_revision( # ARRANGE -------------------------------------------------------------- workflow_revision_slug = uuid4() - response = authed_api( - "POST", - "/workflows/revisions/commit", - json={ - "workflow_revision": { - "slug": f"workflow-revision-{workflow_revision_slug}", - "name": f"Workflow revision {workflow_revision_slug}", - "description": "Workflow revision Description", - "flags": { - "is_custom": False, - "is_evaluator": False, - "is_feedback": False, - }, - "tags": { - "tag1": "value1", - "tag2": "value2", - "tag3": "value3", - }, - "meta": { - "meta1": "value1", - "meta2": "value2", - "meta3": "value3", - }, - "data": {"parameters": {"key1": "value1"}}, - "workflow_id": mock_data["workflows"][0]["id"], - "workflow_variant_id": mock_data["workflow_variants"][0]["id"], - } + response = _commit_workflow_revision( + authed_api, + workflow_id=mock_data["workflows"][0]["id"], + workflow_variant_id=mock_data["workflow_variants"][0]["id"], + slug=f"workflow-revision-{workflow_revision_slug}", + name=f"Workflow revision {workflow_revision_slug}", + description="Workflow revision Description", + flags={ + "is_custom": False, + "is_evaluator": False, + "is_feedback": False, + }, + tags={ + "tag1": "value1", + "tag2": "value2", + "tag3": "value3", + }, + meta={ + "meta1": "value1", + "meta2": "value2", + "meta3": "value3", }, ) diff --git a/api/oss/tests/pytest/unit/auth/test_helper.py b/api/oss/tests/pytest/unit/auth/test_helper.py index f6b1cfb676..81b2eba7f8 100644 --- a/api/oss/tests/pytest/unit/auth/test_helper.py +++ b/api/oss/tests/pytest/unit/auth/test_helper.py @@ -5,7 +5,7 @@ from oss.src.core.auth import helper as auth_helper from oss.src.core.auth.supertokens import overrides -from oss.src.services.exceptions import UnauthorizedException +from oss.src.utils.exceptions import UnauthorizedException from supertokens_python.recipe.thirdparty.interfaces import SignInUpNotAllowed @@ -74,7 +74,7 @@ async def test_get_blocked_domains_accepts_string_posthog_payload(monkeypatch): allowed_domains=set(), ), ), - posthog=SimpleNamespace(enabled=True), + posthog=SimpleNamespace(enabled=True, api_key="posthog-key"), ), ) monkeypatch.setattr( @@ -88,9 +88,11 @@ async def test_get_blocked_domains_accepts_string_posthog_payload(monkeypatch): AsyncMock(), ) monkeypatch.setattr( - auth_helper.posthog, - "get_feature_flag_payload", - lambda feature_flag, distinct_id: "agenta.dev", + auth_helper, + "_load_posthog", + lambda: SimpleNamespace( + get_feature_flag_payload=lambda feature_flag, distinct_id: "agenta.dev", + ), ) blocked_domains = await auth_helper.get_blocked_domains() @@ -111,7 +113,7 @@ async def test_get_blocked_domains_splits_comma_separated_posthog_payload(monkey allowed_domains=set(), ), ), - posthog=SimpleNamespace(enabled=True), + posthog=SimpleNamespace(enabled=True, api_key="posthog-key"), ), ) monkeypatch.setattr( @@ -125,9 +127,13 @@ async def test_get_blocked_domains_splits_comma_separated_posthog_payload(monkey AsyncMock(), ) monkeypatch.setattr( - auth_helper.posthog, - "get_feature_flag_payload", - lambda feature_flag, distinct_id: "spica.asia, agenta.ai , yopmail.com", + auth_helper, + "_load_posthog", + lambda: SimpleNamespace( + get_feature_flag_payload=( + lambda feature_flag, distinct_id: "spica.asia, agenta.ai , yopmail.com" + ), + ), ) blocked_domains = await auth_helper.get_blocked_domains() @@ -135,6 +141,40 @@ async def test_get_blocked_domains_splits_comma_separated_posthog_payload(monkey assert blocked_domains == {"spica.asia", "agenta.ai", "yopmail.com"} +@pytest.mark.asyncio +async def test_get_blocked_emails_treats_posthog_errors_as_empty(monkeypatch): + monkeypatch.setattr( + auth_helper, + "env", + SimpleNamespace( + agenta=SimpleNamespace( + access=SimpleNamespace( + blocked_domains=set(), + blocked_emails=set(), + allowed_domains=set(), + ), + ), + posthog=SimpleNamespace(enabled=True), + ), + ) + monkeypatch.setattr( + auth_helper, + "get_cache", + AsyncMock(return_value=None), + ) + monkeypatch.setattr( + auth_helper, + "_load_posthog", + lambda: SimpleNamespace( + get_feature_flag_payload=lambda feature_flag, distinct_id: ( + _ for _ in () + ).throw(ValueError("API key is required")), + ), + ) + + assert await auth_helper.get_blocked_emails() == set() + + @pytest.mark.asyncio async def test_thirdparty_sign_in_up_checks_blocking_before_auth(monkeypatch): called = False diff --git a/api/oss/tests/pytest/unit/evaluations/test_cache_split_utils.py b/api/oss/tests/pytest/unit/evaluations/test_cache_split_utils.py index 7c0cef9d24..06ae1026e9 100644 --- a/api/oss/tests/pytest/unit/evaluations/test_cache_split_utils.py +++ b/api/oss/tests/pytest/unit/evaluations/test_cache_split_utils.py @@ -110,7 +110,7 @@ def test_repeat_and_fanout_planning_helpers_follow_split_rules(): assert ( effective_is_split( is_split=True, - is_queue=True, + has_traces=True, has_application_steps=True, has_evaluator_steps=True, ) diff --git a/api/oss/tests/pytest/unit/evaluations/test_live_evaluation_guards.py b/api/oss/tests/pytest/unit/evaluations/test_live_evaluation_guards.py index 2c4b8f032f..f6a3d2a703 100644 --- a/api/oss/tests/pytest/unit/evaluations/test_live_evaluation_guards.py +++ b/api/oss/tests/pytest/unit/evaluations/test_live_evaluation_guards.py @@ -8,129 +8,129 @@ import pytest -# Stub genson so importing oss.src.core.evaluations.tasks.live (which transitively -# pulls in services depending on it) does not require genson during unit tests. sys.modules.setdefault("genson", types.SimpleNamespace(SchemaBuilder=object)) -from oss.src.core.evaluators.dtos import EvaluatorRevision, EvaluatorRevisionData +from oss.src.core.evaluators.dtos import EvaluatorRevision +from oss.src.core.evaluations.runtime.sources import QueryRevisionTraceResolver +from oss.src.core.evaluations.service import SimpleEvaluationsService +from oss.src.core.evaluations.types import EvaluationRunDataStep from oss.src.core.queries.dtos import QueryRevision, QueryRevisionData from oss.src.core.shared.dtos import Reference @pytest.mark.asyncio -async def test_resolve_query_revisions_skips_revision_with_null_data(): - """A live evaluation must not run when the referenced query revision has no data. - - If the revision row exists but `data` is null, running it would issue an - unfiltered tracing query and silently evaluate every trace in the project. - """ - from oss.src.core.evaluations.tasks.live import _resolve_query_revisions - +async def test_query_revision_trace_resolver_skips_revision_with_null_data(): revision_id = uuid4() - revision_ref = Reference(id=revision_id) - - revision_without_data = QueryRevision( - id=revision_id, - slug="my-query", - data=None, - ) - - queries_service = SimpleNamespace( - fetch_query_revision=AsyncMock(return_value=revision_without_data), + resolver = QueryRevisionTraceResolver( + queries_service=SimpleNamespace( + fetch_query_revision=AsyncMock( + return_value=QueryRevision( + id=revision_id, + slug="my-query", + data=None, + ) + ) + ) ) - resolved = await _resolve_query_revisions( - queries_service=queries_service, + batch = await resolver.resolve( project_id=uuid4(), - query_revision_refs={"query-step": revision_ref}, + step=EvaluationRunDataStep( + key="query-step", + type="input", + origin="custom", + references={"query_revision": Reference(id=revision_id)}, + ), ) - assert resolved == {}, ( - "Revisions with null `data` must be skipped, not fabricated with empty data." - ) + assert batch is None @pytest.mark.asyncio -async def test_resolve_query_revisions_keeps_revision_with_real_data(): - from oss.src.core.evaluations.tasks.live import _resolve_query_revisions - +async def test_query_revision_trace_resolver_keeps_revision_with_trace_ids(): revision_id = uuid4() - revision_ref = Reference(id=revision_id) - - revision_with_data = QueryRevision( - id=revision_id, - slug="my-query", - data=QueryRevisionData(trace_ids=["00000000000000000000000000000001"]), - ) - - queries_service = SimpleNamespace( - fetch_query_revision=AsyncMock(return_value=revision_with_data), + resolver = QueryRevisionTraceResolver( + queries_service=SimpleNamespace( + fetch_query_revision=AsyncMock( + return_value=QueryRevision( + id=revision_id, + slug="my-query", + data=QueryRevisionData( + trace_ids=["00000000000000000000000000000001"] + ), + ) + ) + ) ) - resolved = await _resolve_query_revisions( - queries_service=queries_service, + batch = await resolver.resolve( project_id=uuid4(), - query_revision_refs={"query-step": revision_ref}, + step=EvaluationRunDataStep( + key="query-step", + type="input", + origin="custom", + references={"query_revision": Reference(id=revision_id)}, + ), ) - assert set(resolved.keys()) == {"query-step"} + assert batch is not None + assert batch.step_key == "query-step" + assert batch.trace_ids == ["00000000000000000000000000000001"] @pytest.mark.asyncio -async def test_resolve_evaluator_revisions_skips_revision_with_null_data(): - """A live evaluation must not invoke an evaluator workflow with no config. - - If the evaluator revision row exists but `data` is null, downstream code - would build a `WorkflowServiceRequest` with empty uri/url/script and call - `workflows_service.invoke_workflow` against a non-functional target. - """ - from oss.src.core.evaluations.tasks.live import _resolve_evaluator_revisions - +async def test_make_evaluation_run_data_rejects_live_query_revision_with_null_data(): revision_id = uuid4() - revision_ref = Reference(id=revision_id) - - revision_without_data = EvaluatorRevision( - id=revision_id, - slug="my-evaluator", - data=None, - ) - - evaluators_service = SimpleNamespace( - fetch_evaluator_revision=AsyncMock(return_value=revision_without_data), + service = SimpleNamespace( + queries_service=SimpleNamespace( + fetch_query_revision=AsyncMock( + return_value=QueryRevision( + id=revision_id, + slug="my-query", + data=None, + ) + ) + ), + testsets_service=SimpleNamespace(), + applications_service=SimpleNamespace(), + evaluators_service=SimpleNamespace(), ) - resolved = await _resolve_evaluator_revisions( - evaluators_service=evaluators_service, + run_data = await SimpleEvaluationsService._make_evaluation_run_data( + service, project_id=uuid4(), - evaluator_revision_refs={"evaluator-step": revision_ref}, + user_id=uuid4(), + query_steps=[revision_id], + is_live=True, ) - assert resolved == {}, ( - "Evaluator revisions with null `data` must be skipped, not fabricated with empty data." - ) + assert run_data is None @pytest.mark.asyncio -async def test_resolve_evaluator_revisions_keeps_revision_with_real_data(): - from oss.src.core.evaluations.tasks.live import _resolve_evaluator_revisions - +async def test_make_evaluation_run_data_rejects_evaluator_revision_with_null_data(): revision_id = uuid4() - revision_ref = Reference(id=revision_id) - - revision_with_data = EvaluatorRevision( - id=revision_id, - slug="my-evaluator", - data=EvaluatorRevisionData(uri="http://evaluator.local/invoke"), - ) - - evaluators_service = SimpleNamespace( - fetch_evaluator_revision=AsyncMock(return_value=revision_with_data), + service = SimpleNamespace( + queries_service=SimpleNamespace(), + testsets_service=SimpleNamespace(), + applications_service=SimpleNamespace(), + evaluators_service=SimpleNamespace( + fetch_evaluator_revision=AsyncMock( + return_value=EvaluatorRevision( + id=revision_id, + slug="my-evaluator", + data=None, + ) + ) + ), ) - resolved = await _resolve_evaluator_revisions( - evaluators_service=evaluators_service, + run_data = await SimpleEvaluationsService._make_evaluation_run_data( + service, project_id=uuid4(), - evaluator_revision_refs={"evaluator-step": revision_ref}, + user_id=uuid4(), + evaluator_steps=[revision_id], + is_live=False, ) - assert set(resolved.keys()) == {"evaluator-step"} + assert run_data is None diff --git a/api/oss/tests/pytest/unit/evaluations/test_query_eval_loops.py b/api/oss/tests/pytest/unit/evaluations/test_query_eval_loops.py index 4f88be8851..5ca3f411b4 100644 --- a/api/oss/tests/pytest/unit/evaluations/test_query_eval_loops.py +++ b/api/oss/tests/pytest/unit/evaluations/test_query_eval_loops.py @@ -24,7 +24,7 @@ SimpleQueueData, ) from oss.src.core.evaluations.service import SimpleQueuesService -from oss.src.core.evaluations.tasks import live as live_module +from oss.src.core.evaluations.tasks import run as run_module @pytest.mark.asyncio @@ -92,8 +92,8 @@ async def test_simple_queue_create_dispatches_each_query_source_with_step_key(): ) ), testsets_service=SimpleNamespace(fetch_testset_revision=AsyncMock()), - evaluate_batch_traces=AsyncMock(return_value=True), - evaluate_batch_testcases=AsyncMock(return_value=True), + dispatch_trace_slice=AsyncMock(return_value=True), + dispatch_testcase_slice=AsyncMock(return_value=True), ) service = SimpleQueuesService( @@ -116,9 +116,9 @@ async def test_simple_queue_create_dispatches_each_query_source_with_step_key(): assert created_queue is not None assert created_queue.data is not None - assert created_queue.data.kind == "traces" + assert created_queue.data.kind == "queries" assert created_queue.data.queries == [query_revision_id_1, query_revision_id_2] - assert simple_evaluations_service.evaluate_batch_traces.await_args_list == [ + assert simple_evaluations_service.dispatch_trace_slice.await_args_list == [ call( project_id=project_id, user_id=user_id, @@ -137,7 +137,7 @@ async def test_simple_queue_create_dispatches_each_query_source_with_step_key(): @pytest.mark.asyncio -async def test_evaluate_live_query_marks_human_steps_pending(monkeypatch): +async def test_run_query_source_marks_human_steps_pending(monkeypatch): project_id = uuid4() user_id = uuid4() run_id = uuid4() @@ -189,7 +189,7 @@ async def test_evaluate_live_query_marks_human_steps_pending(monkeypatch): create_scenarios = AsyncMock( return_value=[SimpleNamespace(id=scenario_id, tags=None, meta=None)] ) - create_results = AsyncMock(return_value=[SimpleNamespace(id=uuid4())]) + set_results = AsyncMock(return_value=[SimpleNamespace(id=uuid4())]) edit_scenario = AsyncMock( side_effect=lambda **kwargs: SimpleNamespace( id=kwargs["scenario"].id, @@ -197,74 +197,253 @@ async def test_evaluate_live_query_marks_human_steps_pending(monkeypatch): meta=kwargs["scenario"].meta, ) ) + edit_run = AsyncMock() refresh_metrics = AsyncMock() + query_results = AsyncMock(return_value=[]) + evaluations_service = SimpleNamespace( + fetch_run=fetch_run, + create_scenarios=create_scenarios, + set_results=set_results, + edit_scenario=edit_scenario, + edit_run=edit_run, + refresh_metrics=refresh_metrics, + query_results=query_results, + ) + queries_service = SimpleNamespace( + fetch_query_revision=AsyncMock( + return_value=SimpleNamespace( + id=query_revision_id, + slug="query-live", + data=SimpleNamespace(filtering=None, windowing=None), + ) + ) + ) + workflows_service = SimpleNamespace(invoke_workflow=AsyncMock()) + + # The query resolver hands the unified flow an already-hydrated trace; no + # re-fetch happens downstream (Option A seed path). monkeypatch.setattr( - live_module, - "evaluations_service", - SimpleNamespace( - fetch_run=fetch_run, - create_scenarios=create_scenarios, - create_results=create_results, - edit_scenario=edit_scenario, - refresh_metrics=refresh_metrics, + run_module, + "resolve_query_source_items", + AsyncMock( + return_value={ + "query-live": [ + run_module.ResolvedSourceItem( + kind="trace", + step_key="query-live", + trace_id="trace-live", + trace=trace, + ) + ] + } ), ) - monkeypatch.setattr( - live_module, - "queries_service", - SimpleNamespace( - fetch_query_revision=AsyncMock( - return_value=SimpleNamespace( - id=query_revision_id, - slug="query-live", - data=SimpleNamespace(filtering=None, windowing=None), - ) - ) + + # Live run (use_windowing=False) routes through the query seam directly. + await run_module._run_query_source( + project_id=project_id, + user_id=user_id, + run=run, + newest=None, + oldest=None, + use_windowing=False, + tracing_service=SimpleNamespace(), + queries_service=queries_service, + workflows_service=workflows_service, + applications_service=SimpleNamespace(), + evaluations_service=evaluations_service, + ) + + # The input cell is written under the query step key. The unified flow + # populates it once (mint+populate) and the SDK plan logs the input step + # again on execute — same as the direct-slice ingest path. We assert the + # query step key is present and the human step is logged PENDING, rather + # than pinning the exact call count. + logged_step_keys = { + result.step_key + for call_args in set_results.await_args_list + for result in call_args.kwargs["results"] + } + assert "query-live" in logged_step_keys + + # The human annotation step is PENDING (the backend never executes it). + scenario_statuses = { + call_args.kwargs["scenario"].status + for call_args in edit_scenario.await_args_list + } + assert all( + isinstance(call_args.kwargs["scenario"], EvaluationScenarioEdit) + for call_args in edit_scenario.await_args_list + ) + assert EvaluationStatus.PENDING in scenario_statuses + # Human-only live tick produced no auto results -> no metric refresh, and a + # live run is never finalized. + refresh_metrics.assert_not_awaited() + edit_run.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_run_query_source_skips_empty_query_results(monkeypatch): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + query_revision_id = uuid4() + evaluator_revision_id = uuid4() + run = EvaluationRun( + id=run_id, + flags=EvaluationRunFlags(has_queries=True, has_evaluators=True), + status=EvaluationStatus.RUNNING, + data=EvaluationRunData( + steps=[ + EvaluationRunDataStep( + key="query-main", + type="input", + origin="custom", + references={"query_revision": Reference(id=query_revision_id)}, + ), + EvaluationRunDataStep( + key="evaluator-auto", + type="annotation", + origin="auto", + references={ + "evaluator_revision": Reference(id=evaluator_revision_id) + }, + ), + ] ), ) + execute_bindings = AsyncMock() monkeypatch.setattr( - live_module, - "evaluators_service", - SimpleNamespace( - fetch_evaluator_revision=AsyncMock( - return_value=SimpleNamespace( - id=evaluator_revision_id, - slug="evaluator-human", - data=SimpleNamespace(), - ) - ) + run_module, + "resolve_query_source_items", + AsyncMock(return_value={"query-main": []}), + ) + monkeypatch.setattr(run_module, "_execute_bindings", execute_bindings) + + # Live run (use_windowing=False): an empty tick must not mint or execute. + await run_module._run_query_source( + project_id=project_id, + user_id=user_id, + run=run, + newest=None, + oldest=None, + use_windowing=False, + tracing_service=SimpleNamespace(), + queries_service=SimpleNamespace(), + workflows_service=SimpleNamespace(), + applications_service=SimpleNamespace(), + evaluations_service=SimpleNamespace(fetch_run=AsyncMock(return_value=run)), + ) + + execute_bindings.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_run_query_source_finalizes_batch_run_on_pre_slice_error( + monkeypatch, +): + """A pre-slice error in a batch run must finalize it to FAILURE (UEL-024). + + An exception raised before the slice processor runs (here, during source + resolution) never reaches the slice's own finalize, so the outer handler must + flip a batch (use_windowing=True) run to FAILURE + inactive instead of + leaving it stuck RUNNING. + """ + project_id, user_id, run_id = uuid4(), uuid4(), uuid4() + run = EvaluationRun( + id=run_id, + flags=EvaluationRunFlags(has_queries=True, has_evaluators=True, is_active=True), + status=EvaluationStatus.RUNNING, + data=EvaluationRunData( + steps=[ + EvaluationRunDataStep( + key="query-main", + type="input", + origin="custom", + references={"query_revision": Reference(id=uuid4())}, + ), + ] ), ) - monkeypatch.setattr( - live_module, - "tracing_service", - SimpleNamespace(query_traces=AsyncMock(return_value=[trace])), + edit_run = AsyncMock() + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + edit_run=edit_run, ) monkeypatch.setattr( - live_module, - "workflows_service", - SimpleNamespace(invoke_workflow=AsyncMock()), + run_module, + "resolve_query_source_items", + AsyncMock(side_effect=RuntimeError("boom: trace fetch failed")), + ) + + # Must not raise — the error is handled and the run finalized. + await run_module._run_query_source( + project_id=project_id, + user_id=user_id, + run=run, + newest=None, + oldest=None, + use_windowing=True, + tracing_service=SimpleNamespace(), + queries_service=SimpleNamespace(), + workflows_service=SimpleNamespace(), + applications_service=SimpleNamespace(), + evaluations_service=evaluations_service, + ) + + edit_run.assert_awaited_once() + _, kwargs = edit_run.await_args + edited = kwargs["run"] + assert edited.status == EvaluationStatus.FAILURE + assert edited.flags.is_active is False + + +@pytest.mark.asyncio +async def test_run_query_source_live_run_not_finalized_on_error(monkeypatch): + """A LIVE run (use_windowing=False) keeps ticking — it is NOT finalized.""" + project_id, user_id, run_id = uuid4(), uuid4(), uuid4() + run = EvaluationRun( + id=run_id, + flags=EvaluationRunFlags( + has_queries=True, has_evaluators=True, is_live=True, is_active=True + ), + status=EvaluationStatus.RUNNING, + data=EvaluationRunData( + steps=[ + EvaluationRunDataStep( + key="query-main", + type="input", + origin="custom", + references={"query_revision": Reference(id=uuid4())}, + ), + ] + ), + ) + edit_run = AsyncMock() + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + edit_run=edit_run, ) monkeypatch.setattr( - live_module, - "fetch_traces_by_hash", - AsyncMock(return_value=[]), + run_module, + "resolve_query_source_items", + AsyncMock(side_effect=RuntimeError("boom")), ) - await live_module.evaluate_live_query( + await run_module._run_query_source( project_id=project_id, user_id=user_id, - run_id=run_id, + run=run, + newest=None, + oldest=None, + use_windowing=False, + tracing_service=SimpleNamespace(), + queries_service=SimpleNamespace(), + workflows_service=SimpleNamespace(), + applications_service=SimpleNamespace(), + evaluations_service=evaluations_service, ) - assert create_results.await_count == 1 - result_step_keys = [ - result.step_key for result in create_results.await_args.kwargs["results"] - ] - assert result_step_keys == ["query-live"] - - scenario_edit = edit_scenario.await_args.kwargs["scenario"] - assert isinstance(scenario_edit, EvaluationScenarioEdit) - assert scenario_edit.status == EvaluationStatus.PENDING - refresh_metrics.assert_not_awaited() + # live runs are never finalized on error — the scheduler keeps polling. + edit_run.assert_not_awaited() diff --git a/api/oss/tests/pytest/unit/evaluations/test_queue_dao_serialization.py b/api/oss/tests/pytest/unit/evaluations/test_queue_dao_serialization.py index cad44f6ee8..c7444a2b71 100644 --- a/api/oss/tests/pytest/unit/evaluations/test_queue_dao_serialization.py +++ b/api/oss/tests/pytest/unit/evaluations/test_queue_dao_serialization.py @@ -51,10 +51,14 @@ async def test_create_queue_serializes_queue_data_without_uuid_warnings(monkeypa "_get_run_flags", AsyncMock(return_value={}), ) + # Mock get_transactions_engine to return an engine with session method + mock_engine = type( + "MockEngine", (), {"session": lambda self: _DummySessionContext(session)} + )() monkeypatch.setattr( - dao_module.engine, - "core_session", - lambda: _DummySessionContext(session), + dao_module, + "get_transactions_engine", + lambda: mock_engine, ) def fake_create_dto_from_dbe(*, DTO, dbe): diff --git a/api/oss/tests/pytest/unit/evaluations/test_run_flag_matrix.py b/api/oss/tests/pytest/unit/evaluations/test_run_flag_matrix.py new file mode 100644 index 0000000000..84f7c8bfb7 --- /dev/null +++ b/api/oss/tests/pytest/unit/evaluations/test_run_flag_matrix.py @@ -0,0 +1,180 @@ +""" +Representative matrix for run-flag derivation (`create_run_flags`) and simple-queue +validation (`SimpleQueueData.validate_sources`). + +One row per equivalence class rather than the full cartesian product: each source +family once, each evaluator-origin class once, each independent flag toggled once. +Pure unit tests — no DB, no worker. + +Companion to `test_run_flags.py` (which covers cache/split preservation and the +direct-vs-backed source distinction); this file focuses on the broader has_*/origin +matrix and the queue "at least one human evaluator" rule. +""" + +from uuid import uuid4 + +import pytest +from pydantic import ValidationError + +from oss.src.core.evaluations.types import ( + EvaluationRun, + EvaluationRunData, + EvaluationRunDataStep, + EvaluationRunFlags, + SimpleQueueData, + SimpleQueueKind, +) +from oss.src.dbs.postgres.evaluations.utils import create_run_flags + + +# - helpers ------------------------------------------------------------------- + + +def _input_step(key, references=None, origin="custom"): + return EvaluationRunDataStep( + key=key, type="input", origin=origin, references=references or {} + ) + + +def _annotation_step(origin, key=None): + return EvaluationRunDataStep( + key=key or f"annotation-{origin}", + type="annotation", + origin=origin, + references={"evaluator_revision": {"id": str(uuid4())}}, + ) + + +def _run(steps, flags=None): + return EvaluationRun( + flags=flags or EvaluationRunFlags(), + data=EvaluationRunData(steps=steps), + ) + + +# - source family matrix ------------------------------------------------------ +# Each row: (label, input steps, expected source has_* flags) + +_QUERY_REF = {"query_revision": {"id": str(uuid4())}} +_TESTSET_REF = {"testset_revision": {"id": str(uuid4())}} + +SOURCE_ROWS = [ + # label steps expected (queries, testsets, traces, testcases) + ("queries", [_input_step("q", _QUERY_REF)], (True, False, False, False)), + ("testsets", [_input_step("t", _TESTSET_REF)], (False, True, False, False)), + ("traces-direct", [_input_step("traces", {})], (False, False, True, False)), + ("testcases-direct", [_input_step("testcases", {})], (False, False, False, True)), + ( + "multi-input-query+testset", + [_input_step("q", _QUERY_REF), _input_step("t", _TESTSET_REF)], + (True, True, False, False), + ), + ("none", [], (False, False, False, False)), +] + + +@pytest.mark.parametrize( + "label,steps,expected", + SOURCE_ROWS, + ids=[r[0] for r in SOURCE_ROWS], +) +def test_source_family_flag_derivation(label, steps, expected): + flags = create_run_flags(_run(steps)) + assert flags is not None + want_q, want_t, want_tr, want_tc = expected + assert flags.has_queries is want_q + assert flags.has_testsets is want_t + assert flags.has_traces is want_tr + assert flags.has_testcases is want_tc + + +# - evaluator origin matrix --------------------------------------------------- +# Each row: (label, annotation origins, expected (has_evaluators, has_human, has_auto, has_custom)) + +ORIGIN_ROWS = [ + ("none", [], (False, False, False, False)), + ("human", ["human"], (True, True, False, False)), + ("auto", ["auto"], (True, False, True, False)), + ("custom", ["custom"], (True, False, False, True)), + ("human+auto", ["human", "auto"], (True, True, True, False)), + ("human+custom", ["human", "custom"], (True, True, False, True)), + ("auto+custom", ["auto", "custom"], (True, False, True, True)), + ("all", ["human", "auto", "custom"], (True, True, True, True)), +] + + +@pytest.mark.parametrize( + "label,origins,expected", + ORIGIN_ROWS, + ids=[r[0] for r in ORIGIN_ROWS], +) +def test_evaluator_origin_flag_derivation(label, origins, expected): + steps = [_input_step("t", _TESTSET_REF)] + [ + _annotation_step(o, key=f"annotation-{i}") for i, o in enumerate(origins) + ] + flags = create_run_flags(_run(steps)) + assert flags is not None + want_eval, want_human, want_auto, want_custom = expected + assert flags.has_evaluators is want_eval + assert flags.has_human is want_human + assert flags.has_auto is want_auto + assert flags.has_custom is want_custom + + +# - independent flag preservation --------------------------------------------- +# is_live / is_cached / is_split are explicit flags, preserved across derivation. + + +@pytest.mark.parametrize("flag_name", ["is_live", "is_cached", "is_split"]) +def test_explicit_flags_preserved_through_derivation(flag_name): + flags_in = EvaluationRunFlags(**{flag_name: True}) + steps = [_input_step("t", _TESTSET_REF), _annotation_step("auto")] + flags = create_run_flags(_run(steps, flags=flags_in)) + assert flags is not None + assert getattr(flags, flag_name) is True + + +# - simple-queue evaluator validation ----------------------------------------- +# Rule: a simple queue must resolve to >=1 human evaluator. A bare list is +# origin-less (defaults to human) -> always valid. An explicit dict is valid only +# if at least one value is "human"; a dict of only non-human origins is rejected. + +_E1 = uuid4() +_E2 = uuid4() + + +def _queue_data(evaluators): + return SimpleQueueData(kind=SimpleQueueKind.TESTCASES, evaluators=evaluators) + + +def test_bare_evaluator_list_is_valid(): + data = _queue_data([_E1, _E2]) + assert data.evaluators == [_E1, _E2] + + +@pytest.mark.parametrize( + "evaluators,ok", + [ + ({_E1: "human"}, True), + ({_E1: "human", _E2: "auto"}, True), + ({_E1: "human", _E2: "custom"}, True), + ({_E1: "auto"}, False), + ({_E1: "custom"}, False), + ({_E1: "auto", _E2: "custom"}, False), + ], + ids=[ + "human", + "human+auto", + "human+custom", + "all-auto", + "all-custom", + "auto+custom", + ], +) +def test_simple_queue_human_evaluator_rule(evaluators, ok): + if ok: + data = _queue_data(evaluators) + assert data.evaluators == evaluators + else: + with pytest.raises(ValidationError): + _queue_data(evaluators) diff --git a/api/oss/tests/pytest/unit/evaluations/test_run_flags.py b/api/oss/tests/pytest/unit/evaluations/test_run_flags.py index 03bb40cb1a..1c16b4d1a1 100644 --- a/api/oss/tests/pytest/unit/evaluations/test_run_flags.py +++ b/api/oss/tests/pytest/unit/evaluations/test_run_flags.py @@ -56,3 +56,110 @@ def test_evaluation_run_query_flags_include_cache_and_split_when_explicit(): "is_cached": False, "is_split": False, } + + +def test_create_run_flags_keeps_direct_source_families_distinct_from_backed_sources(): + direct_run = EvaluationRun( + data=EvaluationRunData( + steps=[ + EvaluationRunDataStep( + key="traces", + type="input", + origin="custom", + references={}, + ), + EvaluationRunDataStep( + key="testcases", + type="input", + origin="custom", + references={}, + ), + ] + ) + ) + backed_run = EvaluationRun( + data=EvaluationRunData( + steps=[ + EvaluationRunDataStep( + key="query-main", + type="input", + origin="custom", + references={"query_revision": {"id": str(uuid4())}}, + ), + EvaluationRunDataStep( + key="testset-main", + type="input", + origin="custom", + references={"testset_revision": {"id": str(uuid4())}}, + ), + ] + ) + ) + + direct_flags = create_run_flags(direct_run) + backed_flags = create_run_flags(backed_run) + + assert direct_flags is not None + assert direct_flags.has_traces is True + assert direct_flags.has_testcases is True + assert direct_flags.has_queries is False + assert direct_flags.has_testsets is False + assert backed_flags is not None + assert backed_flags.has_queries is True + assert backed_flags.has_testsets is True + assert backed_flags.has_traces is False + assert backed_flags.has_testcases is False + + +def test_create_run_flags_uses_exact_reference_keys_not_substring(): + # Source-family detection keys on the exact reference key + # (`query_revision` / `testset_revision`). Keys that merely contain "query" + # or "testset" as a substring must NOT flip the family flags. + run = EvaluationRun( + data=EvaluationRunData( + steps=[ + EvaluationRunDataStep( + key="input-misc", + type="input", + origin="custom", + references={ + "query_anchor": {"id": str(uuid4())}, + "testset_metadata": {"id": str(uuid4())}, + "subquery_ref": {"id": str(uuid4())}, + }, + ), + ] + ) + ) + + flags = create_run_flags(run) + + assert flags is not None + assert flags.has_queries is False + assert flags.has_testsets is False + + +def test_create_run_flags_detects_exact_reference_keys_among_other_refs(): + # The exact key still triggers even when other (non-source) references are + # present on the same step. + run = EvaluationRun( + data=EvaluationRunData( + steps=[ + EvaluationRunDataStep( + key="input-query", + type="input", + origin="custom", + references={ + "query_revision": {"id": str(uuid4())}, + "query_anchor": {"id": str(uuid4())}, + }, + ), + ] + ) + ) + + flags = create_run_flags(run) + + assert flags is not None + assert flags.has_queries is True + assert flags.has_testsets is False diff --git a/api/oss/tests/pytest/unit/evaluations/test_runtime_topology_planner.py b/api/oss/tests/pytest/unit/evaluations/test_runtime_topology_planner.py new file mode 100644 index 0000000000..a3e0428dd4 --- /dev/null +++ b/api/oss/tests/pytest/unit/evaluations/test_runtime_topology_planner.py @@ -0,0 +1,2311 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, call +from uuid import uuid4 + +import pytest + +from oss.src.core.evaluations.runtime.adapters import ( + APICachedRunner, + APIEvaluatorRunner, + APIWorkflowRunner, + APIWorkflowServiceRunner, +) +from oss.src.core.evaluations.runtime.cache import RunnableCacheResolver +from oss.src.core.evaluations.runtime.models import ( + ProcessSummary, + ResolvedSourceItem, + TensorSlice, + TensorProbeSummary, +) +from oss.src.core.evaluations.runtime.planner import ( + EvaluationPlanner, + make_scenario_bindings, + plan_source_input_result_creates, + planned_cells_to_result_creates, +) +from oss.src.core.evaluations.runtime.sources import ( + SourceResolutionError, + resolve_direct_source_items, + resolve_live_query_traces, + resolve_queue_source_batches, + resolve_testset_input_specs, +) +from oss.src.core.evaluations.runtime.tensor import TensorSliceOperations +from oss.src.core.evaluations.runtime.runner import TaskiqEvaluationTaskRunner +from oss.src.core.evaluations.runtime.topology import classify_run_topology +from agenta.sdk.evaluations.runtime.models import ( + EvaluationStep as SdkEvaluationStep, + PlannedCell as SdkPlannedCell, + ResolvedSourceItem as SdkResolvedSourceItem, + WorkflowExecutionRequest, + WorkflowExecutionResult, +) +from agenta.sdk.evaluations.runtime.processor import ( + ProcessedScenario as SdkProcessedScenario, +) +from agenta.sdk.models.evaluations import EvaluationStatus as SdkEvaluationStatus +from oss.src.core.evaluations.types import ( + EvaluationResult, + EvaluationResultCreate, + EvaluationRun, + EvaluationRunData, + EvaluationRunDataStep, + EvaluationRunFlags, + EvaluationStatus, + SimpleEvaluation, + SimpleEvaluationData, + SimpleEvaluationFlags, +) +from oss.src.core.shared.dtos import Reference +from oss.src.core.tracing.dtos import Windowing +from oss.src.core.evaluations.service import SimpleEvaluationsService +from oss.src.core.evaluations.tasks import processor as source_slice_tasks +from oss.src.core.evaluations.tasks import run as run_tasks + + +def _run(*, steps, flags=None, repeats=1): + return EvaluationRun( + id=uuid4(), + flags=flags or EvaluationRunFlags(), + data=EvaluationRunData(steps=list(steps), repeats=repeats), + ) + + +def _step(key, type_, origin="custom", references=None): + return EvaluationRunDataStep( + key=key, + type=type_, + origin=origin, + references=references or {}, + ) + + +def test_topology_classifier_preserves_current_batch_dispatch_shapes(): + query_eval = _run( + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "evaluator-auto", + "annotation", + origin="auto", + references={"evaluator_revision": Reference(id=uuid4())}, + ), + ] + ) + testset_eval = _run( + steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step( + "application-main", + "invocation", + references={"application_revision": Reference(id=uuid4())}, + ), + _step( + "evaluator-auto", + "annotation", + origin="auto", + references={"evaluator_revision": Reference(id=uuid4())}, + ), + ] + ) + batch_inference = _run( + steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step( + "application-main", + "invocation", + references={"application_revision": Reference(id=uuid4())}, + ), + ] + ) + + assert classify_run_topology(query_eval).dispatch == "batch_query" + assert classify_run_topology(testset_eval).dispatch == "batch_testset" + assert classify_run_topology(batch_inference).dispatch == "batch_invocation" + + +def test_topology_classifier_names_deferred_shapes(): + query_to_app = _run( + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "application-main", + "invocation", + references={"application_revision": Reference(id=uuid4())}, + ), + ] + ) + testset_to_eval = _run( + steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step( + "evaluator-auto", + "annotation", + origin="auto", + references={"evaluator_revision": Reference(id=uuid4())}, + ), + ] + ) + multi_app = _run( + steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step("application-a", "invocation"), + _step("application-b", "invocation"), + ] + ) + + assert classify_run_topology(query_to_app).status == "potential" + assert classify_run_topology(testset_to_eval).status == "potential" + assert classify_run_topology(multi_app).status == "not_planned" + + +def test_topology_classifier_names_not_planned_source_shapes(): + mixed_sources = _run( + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step("evaluator-auto", "annotation", origin="auto"), + ] + ) + live_testset = _run( + flags=EvaluationRunFlags(is_live=True), + steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step("evaluator-auto", "annotation", origin="auto"), + ], + ) + + assert classify_run_topology(mixed_sources).status == "not_planned" + assert classify_run_topology(live_testset).status == "not_planned" + + +def test_planner_creates_repeat_aware_slots_and_keeps_manual_annotations_pending(): + run = _run( + repeats=3, + flags=EvaluationRunFlags(is_split=False), + steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step( + "application-main", + "invocation", + references={"application_revision": Reference(id=uuid4())}, + ), + _step( + "evaluator-auto", + "annotation", + origin="auto", + references={"evaluator_revision": Reference(id=uuid4())}, + ), + _step( + "evaluator-human", + "annotation", + origin="human", + references={"evaluator_revision": Reference(id=uuid4())}, + ), + ], + ) + scenario_id = uuid4() + bindings = make_scenario_bindings( + scenario_ids=[scenario_id], + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + ) + ], + ) + + plan = EvaluationPlanner().plan(run=run, bindings=bindings) + cells_by_step = {} + for cell in plan.cells: + cells_by_step.setdefault(cell.step_key, []).append(cell) + + assert [cell.repeat_idx for cell in cells_by_step["testset-main"]] == [0, 1, 2] + assert [cell.repeat_idx for cell in cells_by_step["application-main"]] == [0] + assert [cell.repeat_idx for cell in cells_by_step["evaluator-auto"]] == [0, 1, 2] + assert [cell.status for cell in cells_by_step["evaluator-human"]] == [ + EvaluationStatus.PENDING, + EvaluationStatus.PENDING, + EvaluationStatus.PENDING, + ] + assert {(cell.step_key, cell.repeat_idx) for cell in plan.executable_cells} == { + ("application-main", 0), + ("evaluator-auto", 0), + ("evaluator-auto", 1), + ("evaluator-auto", 2), + } + + +def test_planner_fans_out_application_for_batch_inference_without_evaluators(): + run = _run( + repeats=2, + steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step( + "application-main", + "invocation", + references={"application_revision": Reference(id=uuid4())}, + ), + ], + ) + bindings = make_scenario_bindings( + scenario_ids=[uuid4()], + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + ) + ], + ) + + plan = EvaluationPlanner().plan(run=run, bindings=bindings) + + assert [ + cell.repeat_idx for cell in plan.cells if cell.step_key == "application-main" + ] == [0, 1] + + +def test_planner_fans_out_application_when_split_is_enabled(): + run = _run( + repeats=3, + flags=EvaluationRunFlags(is_split=True), + steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step( + "application-main", + "invocation", + references={"application_revision": Reference(id=uuid4())}, + ), + _step("evaluator-auto", "annotation", origin="auto"), + ], + ) + plan = EvaluationPlanner().plan( + run=run, + bindings=make_scenario_bindings( + scenario_ids=[uuid4()], + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + ) + ], + ), + ) + + assert [ + cell.repeat_idx for cell in plan.cells if cell.step_key == "application-main" + ] == [0, 1, 2] + + +def test_planned_cells_convert_to_result_create_payloads(): + run = _run( + repeats=1, + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "evaluator-human", + "annotation", + origin="human", + references={"evaluator_revision": Reference(id=uuid4())}, + ), + ], + ) + trace_id = "trace-1" + scenario_id = uuid4() + plan = EvaluationPlanner().plan( + run=run, + bindings=make_scenario_bindings( + scenario_ids=[scenario_id], + source_items=[ + ResolvedSourceItem( + kind="trace", + step_key="query-main", + trace_id=trace_id, + ) + ], + ), + ) + + result_creates = planned_cells_to_result_creates(plan.cells) + + assert [(result.step_key, result.status) for result in result_creates] == [ + ("query-main", EvaluationStatus.SUCCESS), + ("evaluator-human", EvaluationStatus.PENDING), + ] + assert result_creates[0].trace_id == trace_id + assert result_creates[1].trace_id is None + + +def test_plan_source_input_result_creates_filters_to_source_step(): + run = _run( + repeats=2, + steps=[ + _step("query-other", "input"), + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step("evaluator-auto", "annotation", origin="auto"), + ], + ) + scenario_id = uuid4() + + result_creates = plan_source_input_result_creates( + run=run, + scenario_id=scenario_id, + source_item=ResolvedSourceItem( + kind="trace", + step_key="query-main", + trace_id="trace-main", + ), + ) + + assert [result.step_key for result in result_creates] == [ + "query-main", + "query-main", + ] + assert [result.repeat_idx for result in result_creates] == [0, 1] + assert [result.trace_id for result in result_creates] == [ + "trace-main", + "trace-main", + ] + + +@pytest.mark.asyncio +async def test_cache_resolver_skips_lookup_when_disabled_and_fetches_when_enabled(): + project_id = uuid4() + + class DummyTracingService: + async def query_traces(self, *, project_id, query): + return [SimpleNamespace(trace_id="trace-1"), SimpleNamespace(trace_id=None)] + + disabled = await RunnableCacheResolver().resolve( + tracing_service=DummyTracingService(), + project_id=project_id, + enabled=False, + references={"evaluator_revision": Reference(id=uuid4())}, + required_count=2, + ) + enabled = await RunnableCacheResolver().resolve( + tracing_service=DummyTracingService(), + project_id=project_id, + enabled=True, + references={"evaluator_revision": Reference(id=uuid4())}, + required_count=2, + ) + + assert disabled.reusable_traces == [] + assert disabled.missing_count == 2 + assert [trace.trace_id for trace in enabled.reusable_traces] == ["trace-1"] + assert enabled.missing_count == 1 + + +@pytest.mark.asyncio +async def test_cache_resolver_zero_required_count_does_not_query_traces(): + tracing_service = SimpleNamespace(query_traces=AsyncMock()) + + resolution = await RunnableCacheResolver().resolve( + tracing_service=tracing_service, + project_id=uuid4(), + enabled=True, + references={"evaluator_revision": Reference(id=uuid4())}, + required_count=0, + ) + + assert resolution.reusable_traces == [] + assert resolution.missing_count == 0 + tracing_service.query_traces.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_queue_source_resolver_resolves_query_and_testset_batches(): + project_id = uuid4() + query_revision_id = uuid4() + testset_revision_id = uuid4() + testcase_id_1 = uuid4() + testcase_id_2 = uuid4() + run = _run( + steps=[ + _step( + "query-source", + "input", + references={"query_revision": Reference(id=query_revision_id)}, + ), + _step( + "testset-source", + "input", + references={"testset_revision": Reference(id=testset_revision_id)}, + ), + _step("evaluator-human", "annotation", origin="human"), + ], + ) + queries_service = SimpleNamespace( + fetch_query_revision=AsyncMock( + return_value=SimpleNamespace( + data=SimpleNamespace(trace_ids=["trace-1", "trace-2"]) + ) + ) + ) + testsets_service = SimpleNamespace( + fetch_testset_revision=AsyncMock( + return_value=SimpleNamespace( + data=SimpleNamespace(testcase_ids=[testcase_id_1, testcase_id_2]) + ) + ) + ) + + batches = await resolve_queue_source_batches( + project_id=project_id, + run=run, + queries_service=queries_service, + testsets_service=testsets_service, + ) + + assert [batch.kind for batch in batches] == ["traces", "testcases"] + assert batches[0].step_key == "query-source" + assert batches[0].trace_ids == ["trace-1", "trace-2"] + assert batches[1].step_key == "testset-source" + assert batches[1].testcase_ids == [testcase_id_1, testcase_id_2] + queries_service.fetch_query_revision.assert_awaited_once() + testsets_service.fetch_testset_revision.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_queue_source_resolver_skips_empty_sources(): + run = _run( + steps=[ + _step( + "query-source", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "testset-source", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + ], + ) + + batches = await resolve_queue_source_batches( + project_id=uuid4(), + run=run, + queries_service=SimpleNamespace( + fetch_query_revision=AsyncMock( + return_value=SimpleNamespace(data=SimpleNamespace(trace_ids=[])) + ) + ), + testsets_service=SimpleNamespace( + fetch_testset_revision=AsyncMock( + return_value=SimpleNamespace(data=SimpleNamespace(testcase_ids=[])) + ) + ), + ) + + assert batches == [] + + +@pytest.mark.asyncio +async def test_queue_source_resolver_empty_query_does_not_fall_through_to_testset(): + # A query step whose query revision resolves to zero traces is a real empty + # batch — it must not fall through to the testset resolver, which should + # never be asked about a query-only step. + run = _run( + steps=[ + _step( + "query-source", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + ], + ) + fetch_testset_revision = AsyncMock( + return_value=SimpleNamespace(data=SimpleNamespace(testcase_ids=[uuid4()])) + ) + + batches = await resolve_queue_source_batches( + project_id=uuid4(), + run=run, + queries_service=SimpleNamespace( + fetch_query_revision=AsyncMock( + return_value=SimpleNamespace(data=SimpleNamespace(trace_ids=[])) + ) + ), + testsets_service=SimpleNamespace(fetch_testset_revision=fetch_testset_revision), + ) + + assert batches == [] + fetch_testset_revision.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_queue_source_resolver_rejects_step_with_multiple_source_refs(): + # An input step must carry exactly one recognized source reference. + run = _run( + steps=[ + _step( + "ambiguous-source", + "input", + references={ + "query_revision": Reference(id=uuid4()), + "testset_revision": Reference(id=uuid4()), + }, + ), + ], + ) + + with pytest.raises(SourceResolutionError): + await resolve_queue_source_batches( + project_id=uuid4(), + run=run, + queries_service=SimpleNamespace(fetch_query_revision=AsyncMock()), + testsets_service=SimpleNamespace(fetch_testset_revision=AsyncMock()), + ) + + +@pytest.mark.asyncio +async def test_testset_payload_source_resolver_preserves_testcase_payloads(): + project_id = uuid4() + testset_id = uuid4() + testset_variant_id = uuid4() + testset_revision_id = uuid4() + testcase_id = uuid4() + testcase = SimpleNamespace(id=testcase_id, data={"prompt": "hello"}) + testsets_service = SimpleNamespace( + fetch_testset_revision=AsyncMock( + return_value=SimpleNamespace( + id=testset_revision_id, + variant_id=testset_variant_id, + data=SimpleNamespace(testcases=[testcase]), + ) + ), + fetch_testset_variant=AsyncMock( + return_value=SimpleNamespace( + id=testset_variant_id, + testset_id=testset_id, + ) + ), + fetch_testset=AsyncMock( + return_value=SimpleNamespace( + id=testset_id, + slug="testset-main", + ) + ), + ) + + specs = await resolve_testset_input_specs( + project_id=project_id, + input_steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=testset_revision_id)}, + ) + ], + testsets_service=testsets_service, + ) + + assert len(specs) == 1 + assert specs[0].step_key == "testset-main" + assert specs[0].testcases == [testcase] + assert specs[0].testcases_data == [ + {"prompt": "hello", "testcase_id": str(testcase_id)} + ] + + +@pytest.mark.asyncio +async def test_direct_source_resolver_preserves_order_and_missing_testcases(): + project_id = uuid4() + testcase_id_1 = uuid4() + testcase_id_2 = uuid4() + testcase = SimpleNamespace(id=testcase_id_1, data={"input": "a"}) + testcases_service = SimpleNamespace( + fetch_testcases=AsyncMock(return_value=[testcase]) + ) + + source_items = await resolve_direct_source_items( + project_id=project_id, + testcase_ids=[testcase_id_1, testcase_id_2], + trace_ids=["trace-1"], + testcases_service=testcases_service, + ) + + assert [source_item.kind for source_item in source_items] == [ + "testcase", + "testcase", + "trace", + ] + assert source_items[0].testcase == testcase + assert source_items[1].testcase is None + assert source_items[2].trace_id == "trace-1" + + +@pytest.mark.asyncio +async def test_direct_source_resolver_loads_trace_context(): + project_id = uuid4() + trace_id = "trace-1" + span_id = "span-1" + trace_payload = { + "trace_id": trace_id, + "spans": { + span_id: { + "trace_id": trace_id, + "span_id": span_id, + "attributes": { + "ag": { + "data": { + "inputs": {"prompt": "hello"}, + "outputs": {"answer": "world"}, + } + } + }, + } + }, + } + trace = SimpleNamespace( + trace_id=trace_id, + spans={ + span_id: SimpleNamespace( + trace_id=trace_id, + span_id=span_id, + attributes=trace_payload["spans"][span_id]["attributes"], + ) + }, + model_dump=lambda **_: trace_payload, + ) + tracing_service = SimpleNamespace(fetch_trace=AsyncMock(return_value=trace)) + + source_items = await resolve_direct_source_items( + project_id=project_id, + trace_ids=[trace_id], + tracing_service=tracing_service, + ) + + assert len(source_items) == 1 + assert source_items[0].kind == "trace" + assert source_items[0].trace_id == trace_id + assert source_items[0].span_id == span_id + assert source_items[0].trace is not None + assert source_items[0].inputs == {"prompt": "hello"} + assert source_items[0].outputs == {"answer": "world"} + + +@pytest.mark.asyncio +async def test_live_query_trace_resolver_applies_default_windowing(): + project_id = uuid4() + traces = [SimpleNamespace(trace_id="trace-1")] + + class DummyTracingService: + def __init__(self): + self.query = None + + async def query_traces(self, *, project_id, query): + self.query = query + return traces + + tracing_service = DummyTracingService() + + resolved = await resolve_live_query_traces( + project_id=project_id, + query_revisions={ + "query-main": SimpleNamespace(data=SimpleNamespace()), + }, + tracing_service=tracing_service, + ) + + assert resolved == {"query-main": traces} + assert tracing_service.query.windowing.order == "ascending" + assert tracing_service.query.windowing.limit is None + + +@pytest.mark.asyncio +async def test_live_query_trace_resolver_uses_revision_windowing_when_requested(): + class DummyTracingService: + def __init__(self): + self.query = None + + async def query_traces(self, *, project_id, query): + self.query = query + return [] + + tracing_service = DummyTracingService() + revision_windowing = Windowing( + oldest=None, + newest=None, + limit=25, + order="descending", + rate=0.5, + ) + + await resolve_live_query_traces( + project_id=uuid4(), + query_revisions={ + "query-main": SimpleNamespace( + data=SimpleNamespace(filtering=None, windowing=revision_windowing) + ), + }, + tracing_service=tracing_service, + use_windowing=True, + ) + + assert tracing_service.query.windowing.limit == 25 + assert tracing_service.query.windowing.order == "descending" + assert tracing_service.query.windowing.rate == 0.5 + + +@pytest.mark.asyncio +async def test_tensor_slice_operations_probe_populate_prune_and_process(): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + scenario_id = uuid4() + result_id = uuid4() + result = EvaluationResult( + id=result_id, + run_id=run_id, + scenario_id=scenario_id, + step_key="evaluator-auto", + repeat_idx=0, + status=EvaluationStatus.SUCCESS, + ) + evaluations_service = SimpleNamespace( + query_results=AsyncMock(return_value=[result]), + # prune deletes by id, so it uses the ID-only query (UEL-029). + query_result_ids=AsyncMock(return_value=[result_id]), + set_results=AsyncMock(return_value=[result]), + delete_results=AsyncMock(return_value=[result_id]), + refresh_metrics=AsyncMock(return_value=[]), + # refresh() reads run kind + scenarios for the aggregate boundary. + fetch_run=AsyncMock( + return_value=SimpleNamespace(flags=SimpleNamespace(is_live=False)) + ), + query_scenarios=AsyncMock(return_value=[]), + ) + operations = TensorSliceOperations(evaluations_service=evaluations_service) + tensor_slice = TensorSlice( + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + repeat_idxs=[0], + ) + + probed = await operations.probe( + project_id=project_id, + tensor_slice=tensor_slice, + ) + populated = await operations.populate( + project_id=project_id, + user_id=user_id, + results=[ + EvaluationResultCreate( + run_id=run_id, + scenario_id=scenario_id, + step_key="evaluator-auto", + repeat_idx=0, + status=EvaluationStatus.SUCCESS, + ) + ], + ) + pruned = await operations.prune( + project_id=project_id, + user_id=user_id, + tensor_slice=tensor_slice, + ) + + assert probed == [result] + assert populated == [result] + assert pruned == [result_id] + # probe() is the only full-DTO query here (prune now uses query_result_ids). + assert evaluations_service.query_results.await_count == 1 + assert evaluations_service.query_result_ids.await_count == 1 + assert evaluations_service.set_results.await_count == 1 + assert evaluations_service.delete_results.await_count == 1 + # prune is a tensor-write op: after removing result cells it re-triggers a + # metrics refresh over the affected scope (like populate/process), so + # aggregates recompute over the now-smaller cell set. refresh() does both the + # variational and the aggregate (global, here) pass — so prune drives >= 1 + # refresh_metrics call. probe/populate do not refresh on their own here. + after_prune = evaluations_service.refresh_metrics.await_count + assert after_prune >= 1 + + # an explicit refresh() recomputes the slice's scope again (another batch of + # variational + aggregate refresh_metrics calls). + await operations.refresh( + project_id=project_id, + user_id=user_id, + tensor_slice=tensor_slice, + ) + assert evaluations_service.refresh_metrics.await_count > after_prune + + # process() with no wired slice_processor must fail loudly rather than + # masquerade as execution by silently refreshing metrics (UEL-015). + with pytest.raises(NotImplementedError): + await operations.process( + project_id=project_id, + user_id=user_id, + tensor_slice=tensor_slice, + ) + + +@pytest.mark.asyncio +async def test_tensor_slice_process_delegates_to_injected_processor(): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + expected = ProcessSummary(created=2, pending=1) + slice_processor = SimpleNamespace(process=AsyncMock(return_value=expected)) + operations = TensorSliceOperations( + evaluations_service=SimpleNamespace(), + slice_processor=slice_processor, + ) + tensor_slice = TensorSlice( + run_id=run_id, + scenario_ids=[uuid4()], + step_keys=["evaluator-auto"], + repeat_idxs=[0], + ) + + summary = await operations.process( + project_id=project_id, + user_id=user_id, + tensor_slice=tensor_slice, + ) + + assert summary == expected + slice_processor.process.assert_awaited_once_with( + project_id=project_id, + user_id=user_id, + tensor_slice=tensor_slice, + ) + + +@pytest.mark.asyncio +async def test_tensor_slice_probe_summary_counts_statuses_and_missing_cells(): + project_id = uuid4() + run_id = uuid4() + scenario_id = uuid4() + evaluations_service = SimpleNamespace( + query_results=AsyncMock( + return_value=[ + EvaluationResult( + id=uuid4(), + run_id=run_id, + scenario_id=scenario_id, + step_key="step-success", + repeat_idx=0, + status=EvaluationStatus.SUCCESS, + ), + EvaluationResult( + id=uuid4(), + run_id=run_id, + scenario_id=scenario_id, + step_key="step-failure", + repeat_idx=0, + status=EvaluationStatus.FAILURE, + ), + EvaluationResult( + id=uuid4(), + run_id=run_id, + scenario_id=scenario_id, + step_key="step-pending", + repeat_idx=0, + status=EvaluationStatus.PENDING, + ), + ] + ) + ) + + summary = await TensorSliceOperations( + evaluations_service=evaluations_service + ).probe_summary( + project_id=project_id, + tensor_slice=TensorSlice(run_id=run_id), + expected_count=5, + ) + + assert summary == TensorProbeSummary( + existing_count=3, + missing_count=2, + success_count=1, + failure_count=1, + pending_count=1, + any_count=3, + ) + + +@pytest.mark.asyncio +async def test_tensor_slice_empty_dimension_short_circuits_probe_and_process(): + project_id = uuid4() + user_id = uuid4() + operations = TensorSliceOperations( + evaluations_service=SimpleNamespace( + query_results=AsyncMock(), + refresh_metrics=AsyncMock(), + ) + ) + tensor_slice = TensorSlice(run_id=uuid4(), scenario_ids=[]) + + assert ( + await operations.probe( + project_id=project_id, + tensor_slice=tensor_slice, + ) + == [] + ) + assert ( + await operations.process( + project_id=project_id, + user_id=user_id, + tensor_slice=tensor_slice, + ) + == ProcessSummary() + ) + operations.evaluations_service.query_results.assert_not_awaited() + operations.evaluations_service.refresh_metrics.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_backend_workflow_service_runner_adapts_sdk_runtime_request(): + workflows_service = SimpleNamespace( + invoke_workflow=AsyncMock( + return_value=SimpleNamespace( + status=SimpleNamespace(code=200), + trace_id="trace-success", + span_id="span-success", + outputs={"score": 1}, + ) + ) + ) + runner = APIWorkflowServiceRunner( + workflows_service=workflows_service, + request_builder=lambda request: { + "project_id": "project", + "step_key": request.step.key, + }, + ) + request = WorkflowExecutionRequest( + step=SdkEvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + cell=SdkPlannedCell( + run_id=uuid4(), + scenario_id=uuid4(), + step_key="evaluator-auto", + step_type="annotation", + origin="auto", + repeat_idx=0, + status=SdkEvaluationStatus.QUEUED, + ), + source=SdkResolvedSourceItem(kind="trace", step_key="query-main"), + revision={"slug": "evaluator-auto"}, + ) + + result = await runner.execute(request) + + assert result.status == SdkEvaluationStatus.SUCCESS + assert result.trace_id == "trace-success" + assert result.span_id == "span-success" + workflows_service.invoke_workflow.assert_awaited_once_with( + project_id="project", + step_key="evaluator-auto", + ) + + +@pytest.mark.asyncio +async def test_taskiq_evaluation_task_runner_omits_empty_optional_kwargs(): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + worker = SimpleNamespace( + process_run_from_source=SimpleNamespace(kiq=AsyncMock(return_value="run-task")), + process_run_from_batch=SimpleNamespace( + kiq=AsyncMock(return_value="slice-task") + ), + ) + runner = TaskiqEvaluationTaskRunner(worker=worker) + + assert ( + await runner.process_run_from_source( + project_id=project_id, + user_id=user_id, + run_id=run_id, + ) + == "run-task" + ) + assert ( + await runner.process_run_from_batch( + project_id=project_id, + user_id=user_id, + run_id=run_id, + source_kind="traces", + trace_ids=["trace-1"], + ) + == "slice-task" + ) + + worker.process_run_from_source.kiq.assert_awaited_once_with( + project_id=project_id, + user_id=user_id, + run_id=run_id, + ) + worker.process_run_from_batch.kiq.assert_awaited_once_with( + project_id=project_id, + user_id=user_id, + run_id=run_id, + source_kind="traces", + trace_ids=["trace-1"], + ) + + +@pytest.mark.asyncio +async def test_backend_workflow_runner_invokes_application_through_workflow_service(): + project_id = uuid4() + user_id = uuid4() + application_revision_id = uuid4() + workflows_service = SimpleNamespace( + invoke_workflow=AsyncMock( + return_value=SimpleNamespace( + status=SimpleNamespace(code=200), + trace_id="app-trace", + span_id="app-span", + outputs={"answer": "world"}, + ) + ) + ) + runner = APIWorkflowRunner( + project_id=project_id, + user_id=user_id, + workflows_service=workflows_service, + ) + revision = { + "id": str(application_revision_id), + "data": { + "uri": "http://application", + "schemas": { + "inputs": { + "type": "object", + "properties": {"input": {"type": "string"}}, + } + }, + "parameters": {"temperature": 0.1}, + }, + "flags": {"is_chat": True}, + } + request = WorkflowExecutionRequest( + step=SdkEvaluationStep(key="application-main", type="invocation"), + cell=SdkPlannedCell( + run_id=uuid4(), + scenario_id=uuid4(), + step_key="application-main", + step_type="invocation", + origin="custom", + repeat_idx=0, + status=SdkEvaluationStatus.QUEUED, + ), + source=SdkResolvedSourceItem( + kind="testcase", + step_key="testset-main", + inputs={ + "input": "hello", + "correct_answer": "world", + "testcase_id": "testcase-id", + "testcase_dedup_id": "dedup-id", + }, + ), + revision=revision, + references={"application_revision": {"id": str(application_revision_id)}}, + ) + + result = await runner.execute(request) + + assert result.status == SdkEvaluationStatus.SUCCESS + assert result.trace_id == "app-trace" + workflows_service.invoke_workflow.assert_awaited_once() + kwargs = workflows_service.invoke_workflow.await_args.kwargs + assert kwargs["project_id"] == project_id + assert kwargs["user_id"] == user_id + assert "annotate" not in kwargs + workflow_request = kwargs["request"] + assert workflow_request.flags == {"is_chat": True} + assert workflow_request.data.revision == revision + assert workflow_request.data.revision["data"]["uri"] == "http://application" + assert workflow_request.data.revision["data"]["schemas"] == { + "inputs": { + "type": "object", + "properties": {"input": {"type": "string"}}, + } + } + assert workflow_request.data.parameters == {"temperature": 0.1} + assert workflow_request.data.inputs == {"input": "hello"} + assert ( + workflow_request.references["application_revision"].id + == application_revision_id + ) + + +@pytest.mark.asyncio +async def test_backend_evaluator_runner_sends_normalized_workflow_request(): + project_id = uuid4() + user_id = uuid4() + workflow_revision_id = uuid4() + workflows_service = SimpleNamespace( + invoke_workflow=AsyncMock( + return_value=SimpleNamespace( + status=SimpleNamespace(code=200), + trace_id="eval-trace", + span_id="eval-span", + outputs={"score": 1}, + ) + ) + ) + runner = APIEvaluatorRunner( + project_id=project_id, + user_id=user_id, + workflows_service=workflows_service, + ) + revision = SimpleNamespace( + id=workflow_revision_id, + data=SimpleNamespace( + uri="http://evaluator", + url=None, + headers={"authorization": "secret"}, + schemas={"outputs": {"type": "object"}}, + script="return score", + parameters={"threshold": 0.5}, + ), + flags=SimpleNamespace(model_dump=lambda **kwargs: {"is_custom": True}), + model_dump=lambda **kwargs: {"id": str(workflow_revision_id)}, + ) + request = WorkflowExecutionRequest( + step=SdkEvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + cell=SdkPlannedCell( + run_id=uuid4(), + scenario_id=uuid4(), + step_key="evaluator-auto", + step_type="annotation", + origin="auto", + repeat_idx=0, + status=SdkEvaluationStatus.QUEUED, + ), + source=SdkResolvedSourceItem( + kind="testcase", + step_key="testset-main", + inputs={"input": "hello"}, + outputs={"answer": "world"}, + ), + revision=revision, + references={"evaluator_revision": {"id": str(workflow_revision_id)}}, + links={"invocation": {"trace_id": "app-trace", "span_id": "app-span"}}, + upstream_trace={"trace_id": "app-trace"}, + upstream_outputs={"answer": "world"}, + ) + + result = await runner.execute(request) + + assert result.status == SdkEvaluationStatus.SUCCESS + assert result.trace_id == "eval-trace" + workflows_service.invoke_workflow.assert_awaited_once() + kwargs = workflows_service.invoke_workflow.await_args.kwargs + assert kwargs["project_id"] == project_id + assert kwargs["user_id"] == user_id + assert "annotate" not in kwargs + workflow_request = kwargs["request"] + assert workflow_request.flags == {"is_custom": True} + assert workflow_request.data.revision == {"id": str(workflow_revision_id)} + assert workflow_request.data.parameters == {"threshold": 0.5} + assert workflow_request.data.inputs == {"input": "hello"} + assert workflow_request.data.outputs == {"answer": "world"} + assert workflow_request.links["invocation"].trace_id == "app-trace" + assert workflow_request.links["invocation"].span_id == "app-span" + + +@pytest.mark.asyncio +async def test_backend_evaluator_runner_preserves_dict_revision_data(): + project_id = uuid4() + user_id = uuid4() + workflow_revision_id = uuid4() + workflows_service = SimpleNamespace( + invoke_workflow=AsyncMock( + return_value=SimpleNamespace( + status=SimpleNamespace(code=200), + trace_id="eval-trace", + span_id="eval-span", + outputs={"score": 1}, + ) + ) + ) + runner = APIEvaluatorRunner( + project_id=project_id, + user_id=user_id, + workflows_service=workflows_service, + ) + revision = { + "id": str(workflow_revision_id), + "data": { + "uri": "http://evaluator", + "url": None, + "headers": {"authorization": "secret"}, + "schemas": {"outputs": {"type": "object"}}, + "script": "return score", + "parameters": {"threshold": 0.5}, + }, + "flags": {"is_custom": True}, + } + request = WorkflowExecutionRequest( + step=SdkEvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + cell=SdkPlannedCell( + run_id=uuid4(), + scenario_id=uuid4(), + step_key="evaluator-auto", + step_type="annotation", + origin="auto", + repeat_idx=0, + status=SdkEvaluationStatus.QUEUED, + ), + source=SdkResolvedSourceItem( + kind="testcase", + step_key="testset-main", + inputs={"input": "hello"}, + ), + revision=revision, + ) + + result = await runner.execute(request) + + assert result.status == SdkEvaluationStatus.SUCCESS + workflows_service.invoke_workflow.assert_awaited_once() + workflow_request = workflows_service.invoke_workflow.await_args.kwargs["request"] + assert workflow_request.flags == {"is_custom": True} + assert workflow_request.data.revision["data"]["uri"] == "http://evaluator" + assert workflow_request.data.revision["data"]["headers"] == { + "authorization": "secret" + } + assert workflow_request.data.revision["data"]["schemas"] == { + "outputs": {"type": "object"} + } + assert workflow_request.data.revision["data"]["script"] == "return score" + assert workflow_request.data.revision["data"]["parameters"] == {"threshold": 0.5} + assert workflow_request.data.parameters == {"threshold": 0.5} + + +@pytest.mark.asyncio +async def test_backend_cached_runner_preserves_partial_hit_order(): + project_id = uuid4() + cached_trace = SimpleNamespace(trace_id="cached-trace") + tracing_service = SimpleNamespace( + query_traces=AsyncMock(side_effect=[[cached_trace], []]) + ) + + class BatchRunner: + def __init__(self): + self.requests = [] + + async def execute_batch(self, requests, semaphore=None): + self.requests.append(requests) + return [ + WorkflowExecutionResult( + status=SdkEvaluationStatus.SUCCESS, + trace_id="fresh-trace", + ) + ] + + batch_runner = BatchRunner() + runner = APICachedRunner( + runner=batch_runner, + tracing_service=tracing_service, + project_id=project_id, + enabled=True, + ) + requests = [ + WorkflowExecutionRequest( + step=SdkEvaluationStep(key="evaluator-auto", type="annotation"), + cell=SdkPlannedCell( + run_id=uuid4(), + scenario_id=uuid4(), + step_key="evaluator-auto", + step_type="annotation", + origin="auto", + repeat_idx=idx, + status=SdkEvaluationStatus.QUEUED, + ), + source=SdkResolvedSourceItem(kind="trace", step_key="query-main"), + revision={"id": "evaluator-revision"}, + references={"evaluator_revision": {"id": f"revision-{idx}"}}, + ) + for idx in range(2) + ] + + results = await runner.execute_batch(requests) + + assert [result.trace_id for result in results] == ["cached-trace", "fresh-trace"] + assert len(batch_runner.requests) == 1 + assert [request.cell.repeat_idx for request in batch_runner.requests[0]] == [1] + + +@pytest.mark.asyncio +async def test_simple_evaluation_start_dispatches_batch_invocation_by_topology(): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + run = _run( + steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step( + "application-main", + "invocation", + references={"application_revision": Reference(id=uuid4())}, + ), + ], + ) + run.id = run_id + worker = SimpleNamespace( + process_run_from_source=SimpleNamespace(kiq=AsyncMock()), + ) + service = SimpleEvaluationsService( + testsets_service=None, # type: ignore[arg-type] + queries_service=None, # type: ignore[arg-type] + applications_service=None, # type: ignore[arg-type] + evaluators_service=None, # type: ignore[arg-type] + evaluations_service=None, # type: ignore[arg-type] + evaluations_worker=worker, + ) + service.fetch = AsyncMock( + return_value=SimpleEvaluation( + id=run_id, + flags=SimpleEvaluationFlags(is_live=False), + data=SimpleEvaluationData( + status=None, + testset_steps={uuid4(): "custom"}, + application_steps={uuid4(): "custom"}, + ), + ) + ) + service._activate_evaluation_run = AsyncMock(return_value=run) + + await service.start( + project_id=project_id, + user_id=user_id, + evaluation_id=run_id, + ) + + worker.process_run_from_source.kiq.assert_awaited_once_with( + project_id=project_id, + user_id=user_id, + run_id=run_id, + ) + + +@pytest.mark.asyncio +async def test_simple_evaluation_start_does_not_dispatch_potential_topology(): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + run = _run( + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "application-main", + "invocation", + references={"application_revision": Reference(id=uuid4())}, + ), + ], + ) + run.id = run_id + worker = SimpleNamespace( + process_run_from_source=SimpleNamespace(kiq=AsyncMock()), + ) + service = SimpleEvaluationsService( + testsets_service=None, # type: ignore[arg-type] + queries_service=None, # type: ignore[arg-type] + applications_service=None, # type: ignore[arg-type] + evaluators_service=None, # type: ignore[arg-type] + evaluations_service=None, # type: ignore[arg-type] + evaluations_worker=worker, + ) + service.fetch = AsyncMock( + return_value=SimpleEvaluation( + id=run_id, + flags=SimpleEvaluationFlags(is_live=False), + data=SimpleEvaluationData( + status=None, + query_steps={uuid4(): "custom"}, + application_steps={uuid4(): "custom"}, + ), + ) + ) + service._activate_evaluation_run = AsyncMock(return_value=run) + + await service.start( + project_id=project_id, + user_id=user_id, + evaluation_id=run_id, + ) + + worker.process_run_from_source.kiq.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_simple_evaluation_queue_batches_dispatch_through_slice_processor(): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + testcase_id = uuid4() + run = _run( + flags=EvaluationRunFlags( + is_queue=True, + has_queries=True, + has_testsets=True, + ), + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step("evaluator-human", "annotation", origin="human"), + ], + ) + run.id = run_id + worker = SimpleNamespace( + process_run_from_batch=SimpleNamespace(kiq=AsyncMock()), + ) + evaluations_service = SimpleNamespace(fetch_run=AsyncMock(return_value=run)) + service = SimpleEvaluationsService( + testsets_service=None, # type: ignore[arg-type] + queries_service=None, # type: ignore[arg-type] + applications_service=None, # type: ignore[arg-type] + evaluators_service=None, # type: ignore[arg-type] + evaluations_service=evaluations_service, # type: ignore[arg-type] + evaluations_worker=worker, + ) + service._ensure_human_annotation_queue = AsyncMock() + + traces_ok = await service.dispatch_trace_slice( + project_id=project_id, + user_id=user_id, + run_id=run_id, + trace_ids=["trace-1"], + input_step_key="query-main", + ) + testcases_ok = await service.dispatch_testcase_slice( + project_id=project_id, + user_id=user_id, + run_id=run_id, + testcase_ids=[testcase_id], + input_step_key="testset-main", + ) + + assert traces_ok is True + assert testcases_ok is True + assert worker.process_run_from_batch.kiq.await_args_list == [ + call( + project_id=project_id, + user_id=user_id, + run_id=run_id, + source_kind="traces", + trace_ids=["trace-1"], + input_step_key="query-main", + ), + call( + project_id=project_id, + user_id=user_id, + run_id=run_id, + source_kind="testcases", + testcase_ids=[testcase_id], + input_step_key="testset-main", + ), + ] + + +@pytest.mark.asyncio +async def test_direct_id_ingest_adds_scenarios_populates_then_processes(monkeypatch): + # Direct-id ingest is add_scenarios -> populate -> process: it creates one + # skeleton scenario per id, writes each scenario's input cell carrying the + # id, then re-executes (force) over the new scenarios. `process` itself never + # creates scenarios. + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + testcase_id = uuid4() + scenario_a = uuid4() + scenario_b = uuid4() + + run = _run( + flags=EvaluationRunFlags(has_traces=True), + steps=[_step("query-main", "input")], + ) + run.id = run_id + + created_scenarios = [SimpleNamespace(id=scenario_a), SimpleNamespace(id=scenario_b)] + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + create_scenarios=AsyncMock(return_value=created_scenarios), + set_results=AsyncMock(return_value=[]), + ) + + # The direct ids are hydrated once by the resolver; the unified flow seeds + # those items into the executor (no downstream re-fetch). + monkeypatch.setattr( + run_tasks, + "resolve_direct_source_items", + AsyncMock( + side_effect=lambda **kwargs: ( + [ + run_tasks.ResolvedSourceItem( + kind="trace", step_key="", trace_id=tid + ) + for tid in (kwargs.get("trace_ids") or []) + ] + + [ + run_tasks.ResolvedSourceItem( + kind="testcase", step_key="", testcase_id=tcid + ) + for tcid in (kwargs.get("testcase_ids") or []) + ] + ) + ), + ) + + process_calls = [] + seed_calls = [] + + class _FakeSliceProcessor: + def __init__(self, **kwargs): + pass + + async def process( + self, + *, + project_id, + user_id, + tensor_slice, + seed_bindings=None, + refresh_metrics_without_auto_results=True, + finalize_run_status=True, + ): + process_calls.append(tensor_slice) + seed_calls.append(seed_bindings) + return ProcessSummary(created=len(tensor_slice.scenario_ids or [])) + + monkeypatch.setattr(run_tasks, "APISliceProcessor", _FakeSliceProcessor) + + ok = await run_tasks.run_from_batch( + project_id=project_id, + user_id=user_id, + run_id=run_id, + source_kind="traces", + trace_ids=["trace-1", "trace-2"], + input_step_key="query-main", + tracing_service=object(), # type: ignore[arg-type] + testcases_service=object(), # type: ignore[arg-type] + workflows_service=object(), # type: ignore[arg-type] + applications_service=object(), # type: ignore[arg-type] + evaluations_service=evaluations_service, # type: ignore[arg-type] + ) + + assert ok is True + # add_scenarios: one skeleton scenario per id + evaluations_service.create_scenarios.assert_awaited_once() + _, create_kwargs = evaluations_service.create_scenarios.await_args + assert len(create_kwargs["scenarios"]) == 2 + assert all(s.run_id == run_id for s in create_kwargs["scenarios"]) + + # bind: no input cell is pre-written (the SDK slice loop logs it on execute, + # avoiding a redundant per-scenario write). The hydrated source rides on the + # binding instead. + evaluations_service.set_results.assert_not_awaited() + + # process: re-execute (force) over exactly the new scenarios + assert len(process_calls) == 1 + tensor_slice = process_calls[0] + assert tensor_slice.run_id == run_id + assert set(tensor_slice.scenario_ids) == {scenario_a, scenario_b} + assert tensor_slice.process_mode == "force" + + # seed bindings carry the hydrated source for each minted scenario (Option A: + # the executor reuses them instead of re-reading the input cell). Each binds + # its trace id at the input step key, with no testcase id. + bindings = seed_calls[0] + assert set(bindings.keys()) == {scenario_a, scenario_b} + assert {b.source.trace_id for b in bindings.values()} == {"trace-1", "trace-2"} + assert all(b.source.step_key == "query-main" for b in bindings.values()) + assert all(b.source.testcase_id is None for b in bindings.values()) + + # testcase ingest binds testcase_id (not trace_id) on the seeded source + evaluations_service.create_scenarios.reset_mock() + evaluations_service.create_scenarios.return_value = [SimpleNamespace(id=scenario_a)] + process_calls.clear() + seed_calls.clear() + + ok = await run_tasks.run_from_batch( + project_id=project_id, + user_id=user_id, + run_id=run_id, + source_kind="testcases", + testcase_ids=[testcase_id], + input_step_key="query-main", + tracing_service=object(), # type: ignore[arg-type] + testcases_service=object(), # type: ignore[arg-type] + workflows_service=object(), # type: ignore[arg-type] + applications_service=object(), # type: ignore[arg-type] + evaluations_service=evaluations_service, # type: ignore[arg-type] + ) + assert ok is True + binding = seed_calls[0][scenario_a] + assert binding.source.testcase_id == testcase_id + assert binding.source.trace_id is None + + +@pytest.mark.asyncio +async def test_run_processor_routes_batch_inference_through_testset_application_loop( + monkeypatch, +): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + run = _run( + steps=[ + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + _step( + "application-main", + "invocation", + references={"application_revision": Reference(id=uuid4())}, + ), + ], + ) + run.id = run_id + run_testset_source = AsyncMock() + monkeypatch.setattr( + run_tasks, + "_run_testset_source", + run_testset_source, + ) + + processed = await run_tasks.run_from_source( + project_id=project_id, + user_id=user_id, + run_id=run_id, + tracing_service=object(), # type: ignore[arg-type] + testsets_service=object(), # type: ignore[arg-type] + queries_service=object(), # type: ignore[arg-type] + workflows_service=object(), # type: ignore[arg-type] + applications_service=object(), # type: ignore[arg-type] + evaluations_service=SimpleNamespace(fetch_run=AsyncMock(return_value=run)), + simple_evaluators_service=object(), # type: ignore[arg-type] + ) + + assert processed is True + run_testset_source.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_run_processor_routes_query_topologies_with_windowing(monkeypatch): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + query_revision_id = uuid4() + evaluator_revision_id = uuid4() + newest = object() + oldest = object() + run_query_source = AsyncMock() + monkeypatch.setattr( + run_tasks, + "_run_query_source", + run_query_source, + ) + live_run = _run( + flags=EvaluationRunFlags(is_live=True), + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=query_revision_id)}, + ), + _step( + "evaluator-auto", + "annotation", + origin="auto", + references={"evaluator_revision": Reference(id=evaluator_revision_id)}, + ), + ], + ) + live_run.id = run_id + batch_run = live_run.model_copy(update={"flags": EvaluationRunFlags(is_live=False)}) + + for run, expected_use_windowing in [(live_run, False), (batch_run, True)]: + run_query_source.reset_mock() + + processed = await run_tasks.run_from_source( + project_id=project_id, + user_id=user_id, + run_id=run_id, + newest=newest, # type: ignore[arg-type] + oldest=oldest, # type: ignore[arg-type] + tracing_service=object(), # type: ignore[arg-type] + testsets_service=object(), # type: ignore[arg-type] + queries_service=object(), # type: ignore[arg-type] + workflows_service=object(), # type: ignore[arg-type] + applications_service=object(), # type: ignore[arg-type] + evaluations_service=SimpleNamespace(fetch_run=AsyncMock(return_value=run)), + simple_evaluators_service=object(), # type: ignore[arg-type] + ) + + assert processed is True + kwargs = run_query_source.await_args.kwargs + assert kwargs["use_windowing"] is expected_use_windowing + # Batch query runs drop the scheduler tick's range; live runs keep it. + if expected_use_windowing: + assert kwargs["newest"] is None + assert kwargs["oldest"] is None + else: + assert kwargs["newest"] is newest + assert kwargs["oldest"] is oldest + + +@pytest.mark.asyncio +async def test_run_processor_returns_false_for_missing_or_unsupported_run(): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + unsupported_run = _run( + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "testset-main", + "input", + references={"testset_revision": Reference(id=uuid4())}, + ), + ], + ) + unsupported_run.id = run_id + + common_kwargs = dict( + project_id=project_id, + user_id=user_id, + run_id=run_id, + tracing_service=object(), + testsets_service=object(), + queries_service=object(), + workflows_service=object(), + applications_service=object(), + simple_evaluators_service=object(), + ) + + assert ( + await run_tasks.run_from_source( + **common_kwargs, # type: ignore[arg-type] + evaluations_service=SimpleNamespace(fetch_run=AsyncMock(return_value=None)), + ) + is False + ) + assert ( + await run_tasks.run_from_source( + **common_kwargs, # type: ignore[arg-type] + evaluations_service=SimpleNamespace( + fetch_run=AsyncMock(return_value=unsupported_run) + ), + ) + is False + ) + + +@pytest.mark.asyncio +async def test_backend_slice_processor_reexecutes_existing_scenario(monkeypatch): + # A TensorSlice addresses an EXISTING scenario. The backend slice processor + # must rebuild that scenario's source binding from its stored input cell, + # re-hydrate trace context, and re-run the SDK loop AGAINST THE EXISTING + # scenario (not a freshly created one), returning a populated ProcessSummary. + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + scenario_id = uuid4() + evaluator_revision_id = uuid4() + trace_id = "trace-existing" + + run = _run( + flags=EvaluationRunFlags(has_traces=True), + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "evaluator-auto", + "annotation", + origin="auto", + references={"evaluator_revision": Reference(id=evaluator_revision_id)}, + ), + ], + ) + run.id = run_id + + # The slice probe returns the evaluator cell; the per-scenario probe returns + # the input cell carrying the bound trace_id. + input_cell = EvaluationResult( + id=uuid4(), + run_id=run_id, + scenario_id=scenario_id, + step_key="query-main", + repeat_idx=0, + status=EvaluationStatus.SUCCESS, + trace_id=trace_id, + ) + evaluator_cell = EvaluationResult( + id=uuid4(), + run_id=run_id, + scenario_id=scenario_id, + step_key="evaluator-auto", + repeat_idx=0, + status=EvaluationStatus.FAILURE, + ) + + async def _query_results(*, project_id, result=None, windowing=None): + if result is not None and result.scenario_id == scenario_id: + return [input_cell, evaluator_cell] + return [evaluator_cell] + + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + query_results=AsyncMock(side_effect=_query_results), + # process now owns "done": per-scenario status writes + run finalize + + # incremental metrics refresh. + edit_scenario=AsyncMock(), + edit_run=AsyncMock(), + refresh_metrics=AsyncMock(return_value=[]), + ) + workflows_service = SimpleNamespace( + fetch_workflow_revision=AsyncMock( + return_value=SimpleNamespace(id=evaluator_revision_id) + ) + ) + + monkeypatch.setattr( + source_slice_tasks, + "resolve_direct_source_items", + AsyncMock( + return_value=[ + ResolvedSourceItem( + kind="trace", + step_key="query-main", + trace_id=trace_id, + inputs={"prompt": "hi"}, + ) + ] + ), + ) + sdk_loop = AsyncMock( + return_value=[ + SdkProcessedScenario( + scenario=SimpleNamespace(id=scenario_id), + results={"evaluator-auto": object()}, + auto_results_created=True, + ) + ] + ) + monkeypatch.setattr( + source_slice_tasks, + "sdk_process_evaluation_source_slice", + sdk_loop, + ) + + processor = source_slice_tasks.APISliceProcessor( + evaluations_service=evaluations_service, + tracing_service=None, + testcases_service=None, + workflows_service=workflows_service, + applications_service=None, + ) + summary = await processor.process( + project_id=project_id, + user_id=user_id, + tensor_slice=TensorSlice( + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + repeat_idxs=[0], + process_mode="force", + ), + ) + + # The auto evaluator re-ran for the existing scenario. + assert summary.created == 1 + sdk_loop.assert_awaited_once() + kwargs = sdk_loop.await_args.kwargs + # Source rebuilt from the input cell's trace_id. + assert kwargs["source_items"][0].trace_id == trace_id + # The loop reuses the EXISTING scenario rather than creating a new one. + created_scenario = await kwargs["create_scenario"](run_id) + assert created_scenario.id == scenario_id + # The auto evaluator runner was wired from the run's current revision. + assert "evaluator-auto" in kwargs["runners"] + assert "evaluator-auto" in kwargs["revisions"] + + +@pytest.mark.asyncio +async def test_backend_slice_processor_uses_requested_scenarios_for_missing_cells( + monkeypatch, +): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + scenario_id = uuid4() + evaluator_revision_id = uuid4() + trace_id = "trace-existing" + + run = _run( + flags=EvaluationRunFlags(has_traces=True), + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "app-main", + "invocation", + references={"application_revision": Reference(id=uuid4())}, + ), + _step( + "evaluator-auto", + "annotation", + origin="auto", + references={"evaluator_revision": Reference(id=evaluator_revision_id)}, + ), + ], + repeats=2, + ) + run.id = run_id + + input_cell = EvaluationResult( + id=uuid4(), + run_id=run_id, + scenario_id=scenario_id, + step_key="query-main", + repeat_idx=1, + status=EvaluationStatus.SUCCESS, + trace_id=trace_id, + ) + invocation_cell = EvaluationResult( + id=uuid4(), + run_id=run_id, + scenario_id=scenario_id, + step_key="app-main", + repeat_idx=1, + status=EvaluationStatus.SUCCESS, + trace_id=trace_id, + ) + + async def _query_results(*, project_id, result=None, windowing=None): + if result is not None and result.scenario_id == scenario_id: + return [input_cell, invocation_cell] + return [] + + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + query_results=AsyncMock(side_effect=_query_results), + edit_scenario=AsyncMock(), + edit_run=AsyncMock(), + refresh_metrics=AsyncMock(return_value=[]), + ) + workflows_service = SimpleNamespace( + fetch_workflow_revision=AsyncMock( + return_value=SimpleNamespace(id=evaluator_revision_id) + ), + fetch_application_revision=AsyncMock(return_value=SimpleNamespace(id=uuid4())), + ) + applications_service = SimpleNamespace( + fetch_application_revision=AsyncMock(return_value=SimpleNamespace(id=uuid4())) + ) + + monkeypatch.setattr( + source_slice_tasks, + "resolve_direct_source_items", + AsyncMock( + side_effect=[ + [ + ResolvedSourceItem( + kind="trace", + step_key="query-main", + trace_id=trace_id, + inputs={"prompt": "hi"}, + ) + ], + [ + ResolvedSourceItem( + kind="trace", + step_key="", + trace_id=trace_id, + span_id="span-1", + outputs={"answer": "ok"}, + ) + ], + ] + ), + ) + sdk_loop = AsyncMock( + return_value=[ + SdkProcessedScenario( + scenario=SimpleNamespace(id=scenario_id), + results={"evaluator-auto": object()}, + auto_results_created=True, + ) + ] + ) + monkeypatch.setattr( + source_slice_tasks, "sdk_process_evaluation_source_slice", sdk_loop + ) + + processor = source_slice_tasks.APISliceProcessor( + evaluations_service=evaluations_service, + tracing_service=None, + testcases_service=None, + workflows_service=workflows_service, + applications_service=applications_service, + ) + summary = await processor.process( + project_id=project_id, + user_id=user_id, + tensor_slice=TensorSlice( + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + repeat_idxs=[1], + ), + ) + + assert summary.created == 1 + sdk_loop.assert_awaited_once() + kwargs = sdk_loop.await_args.kwargs + # initial_context_by_repeat is now a per-scenario async callable (batched + # slice): resolve it for this scenario to get its {repeat: ctx} dict. + scenario_context = await kwargs["initial_context_by_repeat"](scenario_id) + assert scenario_context[1]["trace_id"] == trace_id + assert scenario_context[1]["outputs"] == {"answer": "ok"} + assert kwargs["plan_cell_filter"]( + SdkPlannedCell( + run_id=run_id, + scenario_id=scenario_id, + step_key="evaluator-auto", + step_type="annotation", + origin="auto", + repeat_idx=1, + status=EvaluationStatus.QUEUED, + should_execute=True, + ) + ) + assert not kwargs["plan_cell_filter"]( + SdkPlannedCell( + run_id=run_id, + scenario_id=scenario_id, + step_key="evaluator-auto", + step_type="annotation", + origin="auto", + repeat_idx=0, + status=EvaluationStatus.QUEUED, + should_execute=True, + ) + ) + + +@pytest.mark.asyncio +async def test_backend_slice_processor_distinguishes_fill_missing_and_force( + monkeypatch, +): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + scenario_id = uuid4() + evaluator_revision_id = uuid4() + trace_id = "trace-existing" + + run = _run( + flags=EvaluationRunFlags(has_traces=True), + steps=[ + _step( + "query-main", + "input", + references={"query_revision": Reference(id=uuid4())}, + ), + _step( + "evaluator-auto", + "annotation", + origin="auto", + references={"evaluator_revision": Reference(id=evaluator_revision_id)}, + ), + ], + ) + run.id = run_id + + input_cell = EvaluationResult( + id=uuid4(), + run_id=run_id, + scenario_id=scenario_id, + step_key="query-main", + repeat_idx=0, + status=EvaluationStatus.SUCCESS, + trace_id=trace_id, + ) + evaluator_cell = EvaluationResult( + id=uuid4(), + run_id=run_id, + scenario_id=scenario_id, + step_key="evaluator-auto", + repeat_idx=0, + status=EvaluationStatus.SUCCESS, + ) + + async def _query_results(*, project_id, result=None, windowing=None): + if result is not None and result.scenario_id == scenario_id: + return [input_cell, evaluator_cell] + return [evaluator_cell] + + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + query_results=AsyncMock(side_effect=_query_results), + edit_scenario=AsyncMock(), + edit_run=AsyncMock(), + refresh_metrics=AsyncMock(return_value=[]), + ) + workflows_service = SimpleNamespace( + fetch_workflow_revision=AsyncMock( + return_value=SimpleNamespace(id=evaluator_revision_id) + ) + ) + monkeypatch.setattr( + source_slice_tasks, + "resolve_direct_source_items", + AsyncMock( + return_value=[ + ResolvedSourceItem( + kind="trace", + step_key="query-main", + trace_id=trace_id, + inputs={"prompt": "hi"}, + ) + ] + ), + ) + sdk_loop = AsyncMock( + return_value=[ + SdkProcessedScenario( + scenario=SimpleNamespace(id=scenario_id), + results={"evaluator-auto": object()}, + auto_results_created=True, + ) + ] + ) + monkeypatch.setattr( + source_slice_tasks, "sdk_process_evaluation_source_slice", sdk_loop + ) + + processor = source_slice_tasks.APISliceProcessor( + evaluations_service=evaluations_service, + tracing_service=None, + testcases_service=None, + workflows_service=workflows_service, + applications_service=None, + ) + + fill_missing = await processor.process( + project_id=project_id, + user_id=user_id, + tensor_slice=TensorSlice( + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + repeat_idxs=[0], + ), + ) + assert fill_missing.created == 0 + assert fill_missing.reused == 1 + sdk_loop.assert_not_awaited() + + force = await processor.process( + project_id=project_id, + user_id=user_id, + tensor_slice=TensorSlice( + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + repeat_idxs=[0], + process_mode="force", + ), + ) + assert force.created == 1 + sdk_loop.assert_awaited_once() diff --git a/api/oss/tests/pytest/unit/evaluations/test_tensor_slice_ops.py b/api/oss/tests/pytest/unit/evaluations/test_tensor_slice_ops.py new file mode 100644 index 0000000000..1dfddc07d4 --- /dev/null +++ b/api/oss/tests/pytest/unit/evaluations/test_tensor_slice_ops.py @@ -0,0 +1,825 @@ +"""Unit tests for the tensor slice operations surface (PR: unify eval loops). + +Covers the coordinate-addressed ops over EXISTING scenarios — distinct from the +source-keyed dispatch_*_slice path (which ingests NEW source items): + + - TaskiqEvaluationTaskRunner.process_rerun (dispatch, omits empty kwargs) + - SimpleEvaluationsService.dispatch_tensor_slice / probe_slice / populate_slice + - EvaluationsService self-builds TensorSliceOperations from its sub-services + - rerun entry fn runs process() then refresh() +""" + +from datetime import datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock +from uuid import uuid4 + +import pytest + +from oss.src.core.evaluations.service import ( + EvaluationsService, + SimpleEvaluationsService, +) +from oss.src.core.evaluations.runtime.runner import TaskiqEvaluationTaskRunner +from oss.src.core.evaluations.runtime.tensor import TensorSliceOperations +from oss.src.core.evaluations.runtime.models import TensorSlice +from oss.src.core.evaluations.tasks import run as run_module +from oss.src.core.evaluations.tasks.run import ( + run_from_source, + rerun, +) +from oss.src.core.evaluations.runtime.models import TopologyDecision +from oss.src.core.evaluations.types import ( + EvaluationResult, + EvaluationResultCreate, + EvaluationRun, + EvaluationRunData, + EvaluationRunDataStep, + EvaluationStatus, +) + + +# --- runner dispatch ---------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_runner_process_rerun_dispatches_and_omits_empty_kwargs(): + project_id = uuid4() + user_id = uuid4() + run_id = uuid4() + scenario_id = uuid4() + worker = SimpleNamespace( + process_rerun=SimpleNamespace(kiq=AsyncMock(return_value="tensor-task")), + ) + runner = TaskiqEvaluationTaskRunner(worker=worker) + + result = await runner.process_rerun( + project_id=project_id, + user_id=user_id, + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + # repeat_idxs / process_mode left None -> must be omitted from the call + ) + + assert result == "tensor-task" + worker.process_rerun.kiq.assert_awaited_once_with( + project_id=project_id, + user_id=user_id, + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + ) + + +@pytest.mark.asyncio +async def test_runner_process_rerun_forwards_all_kwargs_when_present(): + worker = SimpleNamespace( + process_rerun=SimpleNamespace(kiq=AsyncMock()), + ) + runner = TaskiqEvaluationTaskRunner(worker=worker) + project_id, user_id, run_id, scenario_id = uuid4(), uuid4(), uuid4(), uuid4() + + await runner.process_rerun( + project_id=project_id, + user_id=user_id, + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + repeat_idxs=[0, 1], + process_mode="force", + ) + + worker.process_rerun.kiq.assert_awaited_once_with( + project_id=project_id, + user_id=user_id, + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + repeat_idxs=[0, 1], + process_mode="force", + ) + + +# --- SimpleEvaluationsService.dispatch_tensor_slice --------------------------- + + +def _simple_service(*, worker=None, evaluations_service=None): + return SimpleEvaluationsService( + testsets_service=None, # type: ignore[arg-type] + queries_service=None, # type: ignore[arg-type] + applications_service=None, # type: ignore[arg-type] + evaluators_service=None, # type: ignore[arg-type] + evaluations_service=evaluations_service, # type: ignore[arg-type] + evaluations_worker=worker, + ) + + +@pytest.mark.asyncio +async def test_dispatch_tensor_slice_dispatches_to_runner(): + project_id, user_id, run_id, scenario_id = uuid4(), uuid4(), uuid4(), uuid4() + run = SimpleNamespace(id=run_id, flags=SimpleNamespace()) + worker = SimpleNamespace( + process_rerun=SimpleNamespace(kiq=AsyncMock()), + ) + evaluations_service = SimpleNamespace(fetch_run=AsyncMock(return_value=run)) + service = _simple_service(worker=worker, evaluations_service=evaluations_service) + + ok = await service.dispatch_tensor_slice( + project_id=project_id, + user_id=user_id, + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + process_mode="force", + ) + + assert ok is True + worker.process_rerun.kiq.assert_awaited_once_with( + project_id=project_id, + user_id=user_id, + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + process_mode="force", + ) + + +@pytest.mark.asyncio +async def test_dispatch_tensor_slice_returns_false_without_runner(): + run_id = uuid4() + service = _simple_service(worker=None, evaluations_service=SimpleNamespace()) + + ok = await service.dispatch_tensor_slice( + project_id=uuid4(), + user_id=uuid4(), + run_id=run_id, + ) + + assert ok is False + + +@pytest.mark.asyncio +async def test_dispatch_tensor_slice_returns_false_when_run_missing(): + worker = SimpleNamespace( + process_rerun=SimpleNamespace(kiq=AsyncMock()), + ) + evaluations_service = SimpleNamespace(fetch_run=AsyncMock(return_value=None)) + service = _simple_service(worker=worker, evaluations_service=evaluations_service) + + ok = await service.dispatch_tensor_slice( + project_id=uuid4(), + user_id=uuid4(), + run_id=uuid4(), + ) + + assert ok is False + worker.process_rerun.kiq.assert_not_awaited() + + +# --- SimpleEvaluationsService.probe_slice / populate_slice -------------------- + + +@pytest.mark.asyncio +async def test_probe_slice_delegates_to_tensor_ops_with_built_slice(): + project_id, run_id, scenario_id = uuid4(), uuid4(), uuid4() + result = MagicMock(spec=EvaluationResult) + tensor_ops = SimpleNamespace(probe=AsyncMock(return_value=[result])) + evaluations_service = SimpleNamespace(tensor_slice_operations=tensor_ops) + service = _simple_service(evaluations_service=evaluations_service) + + results = await service.probe_slice( + project_id=project_id, + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + repeat_idxs=[0], + ) + + assert results == [result] + tensor_ops.probe.assert_awaited_once() + _, kwargs = tensor_ops.probe.await_args + assert kwargs["project_id"] == project_id + built: TensorSlice = kwargs["tensor_slice"] + assert built.run_id == run_id + assert built.scenario_ids == [scenario_id] + assert built.step_keys == ["evaluator-auto"] + assert built.repeat_idxs == [0] + + +@pytest.mark.asyncio +async def test_probe_slice_returns_empty_when_tensor_ops_unwired(): + evaluations_service = SimpleNamespace(tensor_slice_operations=None) + service = _simple_service(evaluations_service=evaluations_service) + + results = await service.probe_slice(project_id=uuid4(), run_id=uuid4()) + + assert results == [] + + +@pytest.mark.asyncio +async def test_populate_slice_delegates_results_to_tensor_ops(): + project_id, user_id = uuid4(), uuid4() + result = MagicMock(spec=EvaluationResult) + tensor_ops = SimpleNamespace(populate=AsyncMock(return_value=[result])) + evaluations_service = SimpleNamespace(tensor_slice_operations=tensor_ops) + service = _simple_service(evaluations_service=evaluations_service) + + create = EvaluationResultCreate( + run_id=uuid4(), + scenario_id=uuid4(), + step_key="evaluator-auto", + status=EvaluationStatus.SUCCESS, + ) + results = await service.populate_slice( + project_id=project_id, + user_id=user_id, + results=[create], + ) + + assert results == [result] + tensor_ops.populate.assert_awaited_once_with( + project_id=project_id, + user_id=user_id, + results=[create], + ) + + +@pytest.mark.asyncio +async def test_populate_slice_returns_empty_when_tensor_ops_unwired(): + evaluations_service = SimpleNamespace(tensor_slice_operations=None) + service = _simple_service(evaluations_service=evaluations_service) + + results = await service.populate_slice( + project_id=uuid4(), + user_id=uuid4(), + results=[], + ) + + assert results == [] + + +# --- EvaluationsService self-builds TensorSliceOperations --------------------- + + +def _evaluations_service(*, with_sub_services: bool) -> EvaluationsService: + extra = {} + if with_sub_services: + extra = dict( + testcases_service=MagicMock(), + workflows_service=MagicMock(), + applications_service=MagicMock(), + ) + return EvaluationsService( + evaluations_dao=MagicMock(), + tracing_service=MagicMock(), + queries_service=MagicMock(), + testsets_service=MagicMock(), + evaluators_service=MagicMock(), + **extra, + ) + + +def test_service_builds_tensor_ops_when_sub_services_present(): + service = _evaluations_service(with_sub_services=True) + + ops = service.tensor_slice_operations + assert isinstance(ops, TensorSliceOperations) + assert ops.slice_processor is not None + # the ops reference the owning service (probe/populate go through it) + assert ops.evaluations_service is service + + +def test_service_leaves_tensor_ops_none_without_sub_services(): + service = _evaluations_service(with_sub_services=False) + + assert service.tensor_slice_operations is None + + +# --- rerun entry fn -------------------------------- + + +@pytest.mark.asyncio +async def test_rerun_runs_process_then_refresh(monkeypatch): + """rerun orchestrates process(slice) then refresh(slice) over the same scope. + + The refresh DETAIL (variational + global/temporal aggregate) lives in + TensorSliceOperations.refresh and is covered by + test_tensor_operations_refresh_* — here we only assert rerun delegates both + steps, in order, against the same coordinate slice. + """ + project_id, user_id, run_id, scenario_id = uuid4(), uuid4(), uuid4(), uuid4() + + calls = [] + captured = {} + + class _FakeTensorOps: + def __init__(self, *, evaluations_service, slice_processor): + captured["slice_processor"] = slice_processor + + async def process(self, *, project_id, user_id, tensor_slice): + calls.append("process") + captured["process_slice"] = tensor_slice + + async def refresh(self, *, project_id, user_id, tensor_slice): + calls.append("refresh") + captured["refresh_slice"] = tensor_slice + + monkeypatch.setattr( + "oss.src.core.evaluations.tasks.run.TensorSliceOperations", + _FakeTensorOps, + ) + + ok = await rerun( + project_id=project_id, + user_id=user_id, + run_id=run_id, + scenario_ids=[scenario_id], + step_keys=["evaluator-auto"], + process_mode="force", + tracing_service=MagicMock(), + testcases_service=MagicMock(), + workflows_service=MagicMock(), + applications_service=MagicMock(), + evaluations_service=SimpleNamespace(), + ) + + assert ok is True + # process THEN refresh, both over the same coordinate slice. + assert calls == ["process", "refresh"] + assert captured["process_slice"].run_id == run_id + assert captured["process_slice"].scenario_ids == [scenario_id] + assert captured["process_slice"].step_keys == ["evaluator-auto"] + assert captured["process_slice"].process_mode == "force" + assert captured["refresh_slice"].scenario_ids == [scenario_id] + + +@pytest.mark.asyncio +async def test_tensor_operations_refresh_global_for_non_live(): + """Non-live run -> refresh does variational + the GLOBAL aggregate row.""" + project_id, user_id, run_id, scenario_id = uuid4(), uuid4(), uuid4(), uuid4() + + refresh_mock = AsyncMock(return_value=[]) + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock( + return_value=SimpleNamespace(flags=SimpleNamespace(is_live=False)) + ), + query_scenarios=AsyncMock(return_value=[]), + refresh_metrics=refresh_mock, + ) + operations = TensorSliceOperations(evaluations_service=evaluations_service) + + await operations.refresh( + project_id=project_id, + user_id=user_id, + tensor_slice=TensorSlice(run_id=run_id, scenario_ids=[scenario_id]), + ) + + kinds = [c.kwargs["metrics"] for c in refresh_mock.await_args_list] + # variational: scenario_ids set, no timestamp. + assert any(m.scenario_ids == [scenario_id] and not m.timestamps for m in kinds) + # global: run_id only, no scenario, no timestamp. + assert any( + m.run_id == run_id + and m.scenario_ids is None + and m.scenario_id is None + and not m.timestamps + for m in kinds + ) + + +@pytest.mark.asyncio +async def test_tensor_operations_refresh_temporal_for_live(): + """Live run -> refresh does variational + a TEMPORAL aggregate per interval.""" + project_id, user_id, run_id = uuid4(), uuid4(), uuid4() + s1, s2 = uuid4(), uuid4() + ts = datetime(2026, 6, 2, 14, 0, 0) + + refresh_mock = AsyncMock(return_value=[]) + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock( + return_value=SimpleNamespace(flags=SimpleNamespace(is_live=True)) + ), + query_scenarios=AsyncMock( + return_value=[ + SimpleNamespace(id=s1, timestamp=ts, interval=1), + SimpleNamespace(id=s2, timestamp=ts, interval=1), + ] + ), + refresh_metrics=refresh_mock, + ) + operations = TensorSliceOperations(evaluations_service=evaluations_service) + + await operations.refresh( + project_id=project_id, + user_id=user_id, + tensor_slice=TensorSlice(run_id=run_id, scenario_ids=[s1, s2]), + ) + + kinds = [c.kwargs["metrics"] for c in refresh_mock.await_args_list] + # one temporal refresh for the single affected interval, carrying its bucket. + assert any( + m.run_id == run_id and m.interval == 1 and m.timestamps == [ts] for m in kinds + ) + + +# --- graph-shape ops: add/remove_scenarios (height), add/remove_steps (width), +# set_repeats (depth) --------------------------------------------------- + + +@pytest.mark.asyncio +async def test_add_scenarios_creates_n_skeleton_rows(): + run_id = uuid4() + created = [SimpleNamespace(id=uuid4()), SimpleNamespace(id=uuid4())] + evaluations_service = SimpleNamespace( + create_scenarios=AsyncMock(return_value=created) + ) + service = _simple_service(evaluations_service=evaluations_service) + + result = await service.add_scenarios( + project_id=uuid4(), + user_id=uuid4(), + run_id=run_id, + count=2, + ) + + assert result == created + evaluations_service.create_scenarios.assert_awaited_once() + _, kwargs = evaluations_service.create_scenarios.await_args + scenarios = kwargs["scenarios"] + assert len(scenarios) == 2 + # skeleton only: run-scoped, no input cells / results, no temporal bucket + assert all(s.run_id == run_id for s in scenarios) + assert all(s.timestamp is None and s.interval is None for s in scenarios) + + +@pytest.mark.asyncio +async def test_add_scenarios_floors_timestamp_and_sets_interval(): + run_id = uuid4() + evaluations_service = SimpleNamespace( + create_scenarios=AsyncMock(return_value=[SimpleNamespace(id=uuid4())]) + ) + service = _simple_service(evaluations_service=evaluations_service) + + await service.add_scenarios( + project_id=uuid4(), + user_id=uuid4(), + run_id=run_id, + count=1, + timestamp=datetime(2026, 6, 2, 14, 23, 47, 500000), + ) + + _, kwargs = evaluations_service.create_scenarios.await_args + scenario = kwargs["scenarios"][0] + # floored to the minute; interval fixed at 1 (DEFAULT_REFRESH_INTERVAL) + assert scenario.timestamp == datetime(2026, 6, 2, 14, 23, 0, 0) + assert scenario.interval == 1 + + +@pytest.mark.asyncio +async def test_add_scenarios_zero_count_is_noop(): + evaluations_service = SimpleNamespace(create_scenarios=AsyncMock()) + service = _simple_service(evaluations_service=evaluations_service) + + result = await service.add_scenarios( + project_id=uuid4(), + user_id=uuid4(), + run_id=uuid4(), + count=0, + ) + + assert result == [] + evaluations_service.create_scenarios.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_remove_scenarios_deletes_rows(): + scenario_ids = [uuid4(), uuid4()] + evaluations_service = SimpleNamespace( + delete_scenarios=AsyncMock(return_value=scenario_ids) + ) + service = _simple_service(evaluations_service=evaluations_service) + + result = await service.remove_scenarios( + project_id=uuid4(), + scenario_ids=scenario_ids, + ) + + assert result == scenario_ids + evaluations_service.delete_scenarios.assert_awaited_once() + _, kwargs = evaluations_service.delete_scenarios.await_args + assert kwargs["scenario_ids"] == scenario_ids + + +@pytest.mark.asyncio +async def test_remove_scenarios_empty_is_noop(): + evaluations_service = SimpleNamespace(delete_scenarios=AsyncMock()) + service = _simple_service(evaluations_service=evaluations_service) + + result = await service.remove_scenarios(project_id=uuid4(), scenario_ids=[]) + + assert result == [] + evaluations_service.delete_scenarios.assert_not_awaited() + + +def _step(key: str) -> EvaluationRunDataStep: + return EvaluationRunDataStep( + key=key, + type="input", + origin="custom", + references={}, + ) + + +@pytest.mark.asyncio +async def test_add_steps_appends_new_columns(): + run_id = uuid4() + run = EvaluationRun( + id=run_id, + status=EvaluationStatus.RUNNING, + data=EvaluationRunData(steps=[_step("input")], repeats=1), + ) + edited = SimpleNamespace(id=run_id) + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + edit_run=AsyncMock(return_value=edited), + ) + service = _simple_service(evaluations_service=evaluations_service) + + result = await service.add_steps( + project_id=uuid4(), + user_id=uuid4(), + run_id=run_id, + steps=[_step("evaluator")], + ) + + assert result == edited + _, kwargs = evaluations_service.edit_run.await_args + assert [s.key for s in kwargs["run"].data.steps] == ["input", "evaluator"] + + +@pytest.mark.asyncio +async def test_add_steps_skips_existing_key(): + run_id = uuid4() + run = EvaluationRun( + id=run_id, + status=EvaluationStatus.RUNNING, + data=EvaluationRunData(steps=[_step("input")], repeats=1), + ) + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + edit_run=AsyncMock(), + ) + service = _simple_service(evaluations_service=evaluations_service) + + result = await service.add_steps( + project_id=uuid4(), + user_id=uuid4(), + run_id=run_id, + steps=[_step("input")], + ) + + # all keys already present -> no edit, run returned unchanged + assert result is run + evaluations_service.edit_run.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_remove_steps_drops_named_columns(): + run_id = uuid4() + run = EvaluationRun( + id=run_id, + status=EvaluationStatus.RUNNING, + data=EvaluationRunData(steps=[_step("input"), _step("evaluator")], repeats=1), + ) + edited = SimpleNamespace(id=run_id) + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + edit_run=AsyncMock(return_value=edited), + ) + service = _simple_service(evaluations_service=evaluations_service) + + result = await service.remove_steps( + project_id=uuid4(), + user_id=uuid4(), + run_id=run_id, + step_keys=["evaluator"], + ) + + assert result == edited + _, kwargs = evaluations_service.edit_run.await_args + assert [s.key for s in kwargs["run"].data.steps] == ["input"] + + +@pytest.mark.asyncio +async def test_remove_steps_unknown_key_is_noop(): + run_id = uuid4() + run = EvaluationRun( + id=run_id, + status=EvaluationStatus.RUNNING, + data=EvaluationRunData(steps=[_step("input")], repeats=1), + ) + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + edit_run=AsyncMock(), + ) + service = _simple_service(evaluations_service=evaluations_service) + + result = await service.remove_steps( + project_id=uuid4(), + user_id=uuid4(), + run_id=run_id, + step_keys=["missing"], + ) + + assert result is run + evaluations_service.edit_run.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_set_repeats_sets_run_data_repeats(): + run_id = uuid4() + # set_repeats builds a real EvaluationRunEdit, so the run/data must be the + # real DTOs (model_copy + validation run for real). + run = EvaluationRun( + id=run_id, + status=EvaluationStatus.RUNNING, + data=EvaluationRunData(repeats=1), + ) + edited = SimpleNamespace(id=run_id) + evaluations_service = SimpleNamespace( + fetch_run=AsyncMock(return_value=run), + edit_run=AsyncMock(return_value=edited), + ) + service = _simple_service(evaluations_service=evaluations_service) + + result = await service.set_repeats( + project_id=uuid4(), + user_id=uuid4(), + run_id=run_id, + repeats=3, + ) + + assert result == edited + evaluations_service.edit_run.assert_awaited_once() + _, kwargs = evaluations_service.edit_run.await_args + assert kwargs["run"].data.repeats == 3 + + +@pytest.mark.asyncio +async def test_set_repeats_returns_none_when_run_missing(): + evaluations_service = SimpleNamespace(fetch_run=AsyncMock(return_value=None)) + service = _simple_service(evaluations_service=evaluations_service) + + result = await service.set_repeats( + project_id=uuid4(), + user_id=uuid4(), + run_id=uuid4(), + repeats=3, + ) + + assert result is None + + +# --- prune uses the ID-only query (UEL-029) ---------------------------------- + + +@pytest.mark.asyncio +async def test_prune_uses_query_result_ids_not_full_probe(): + """prune deletes by id, so it must use the ID-only query (no full DTO hydrate).""" + project_id, user_id, run_id = uuid4(), uuid4(), uuid4() + result_ids = [uuid4(), uuid4()] + + evaluations_service = SimpleNamespace( + query_result_ids=AsyncMock(return_value=result_ids), + # query_results is the full-DTO path; prune must NOT call it. + query_results=AsyncMock(return_value=["should-not-be-used"]), + delete_results=AsyncMock(return_value=result_ids), + refresh_metrics=AsyncMock(return_value=[]), + # prune -> refresh() reads run kind + scenarios for the aggregate pass. + fetch_run=AsyncMock( + return_value=SimpleNamespace(flags=SimpleNamespace(is_live=False)) + ), + query_scenarios=AsyncMock(return_value=[]), + ) + tensor_ops = TensorSliceOperations(evaluations_service=evaluations_service) + + deleted = await tensor_ops.prune( + project_id=project_id, + user_id=user_id, + tensor_slice=TensorSlice(run_id=run_id, scenario_ids=[uuid4()]), + ) + + assert deleted == result_ids + evaluations_service.query_result_ids.assert_awaited_once() + evaluations_service.query_results.assert_not_awaited() + evaluations_service.delete_results.assert_awaited_once_with( + project_id=project_id, + result_ids=result_ids, + ) + + +@pytest.mark.asyncio +async def test_prune_empty_slice_is_noop(): + """An empty slice dimension ([]) addresses nothing — no query, no delete.""" + evaluations_service = SimpleNamespace( + query_result_ids=AsyncMock(return_value=[]), + delete_results=AsyncMock(), + refresh_metrics=AsyncMock(), + ) + tensor_ops = TensorSliceOperations(evaluations_service=evaluations_service) + + deleted = await tensor_ops.prune( + project_id=uuid4(), + user_id=uuid4(), + tensor_slice=TensorSlice(run_id=uuid4(), scenario_ids=[]), + ) + + assert deleted == [] + evaluations_service.query_result_ids.assert_not_awaited() + evaluations_service.delete_results.assert_not_awaited() + + +# --- run dispatch routes queue_* topologies (UEL-019) ------------------------ + + +@pytest.mark.parametrize("dispatch", ["queue_traces", "queue_testcases"]) +@pytest.mark.asyncio +async def test_run_from_source_routes_queue_topologies(monkeypatch, dispatch): + """queue_traces/queue_testcases must be handled (returns True), not dropped. + + Before UEL-019 these truthy-dispatch topologies fell through to the + "unsupported topology" branch and returned False at run-start, silently + doing nothing. They are now routed to a clean "open queue awaiting batches" + finalize. + """ + run_id = uuid4() + run = EvaluationRun( + id=run_id, + status=EvaluationStatus.RUNNING, + data=EvaluationRunData(steps=[_step("input")], repeats=1), + ) + monkeypatch.setattr( + run_module, + "classify_run_topology", + lambda _run: TopologyDecision( + status="supported", + label="direct -> evaluator", + reason="worker-dispatched", + dispatch=dispatch, + ), + ) + evaluations_service = SimpleNamespace(fetch_run=AsyncMock(return_value=run)) + + ok = await run_from_source( + project_id=uuid4(), + user_id=uuid4(), + run_id=run_id, + tracing_service=MagicMock(), + testsets_service=MagicMock(), + queries_service=MagicMock(), + workflows_service=MagicMock(), + applications_service=MagicMock(), + evaluations_service=evaluations_service, + simple_evaluators_service=MagicMock(), + ) + + # Handled (not the unsupported-topology False). + assert ok is True + + +@pytest.mark.asyncio +async def test_run_from_source_unsupported_topology_returns_false(monkeypatch): + """A genuinely unsupported topology (no dispatch) still returns False.""" + run_id = uuid4() + run = EvaluationRun( + id=run_id, + status=EvaluationStatus.RUNNING, + data=EvaluationRunData(steps=[_step("input")], repeats=1), + ) + monkeypatch.setattr( + run_module, + "classify_run_topology", + lambda _run: TopologyDecision( + status="unsupported", + label="unsupported", + reason="no path", + dispatch=None, + ), + ) + evaluations_service = SimpleNamespace(fetch_run=AsyncMock(return_value=run)) + + ok = await run_from_source( + project_id=uuid4(), + user_id=uuid4(), + run_id=run_id, + tracing_service=MagicMock(), + testsets_service=MagicMock(), + queries_service=MagicMock(), + workflows_service=MagicMock(), + applications_service=MagicMock(), + evaluations_service=evaluations_service, + simple_evaluators_service=MagicMock(), + ) + + assert ok is False diff --git a/api/oss/tests/pytest/unit/events/test_events_utils.py b/api/oss/tests/pytest/unit/events/test_events_utils.py index 2f5acf0ca8..476fd30fdc 100644 --- a/api/oss/tests/pytest/unit/events/test_events_utils.py +++ b/api/oss/tests/pytest/unit/events/test_events_utils.py @@ -772,9 +772,15 @@ async def _fake_check(**kwargs): ), patch("oss.src.core.events.utils.is_ee", return_value=True), patch( - "ee.src.utils.entitlements.check_entitlements", new=_fake_check, create=True + "ee.src.core.access.entitlements.service.check_entitlements", + new=_fake_check, + create=True, + ), + patch( + "ee.src.core.access.entitlements.service.scope_from", + new=lambda **kw: kw, + create=True, ), - patch("ee.src.utils.entitlements.scope_from", new=lambda **kw: kw, create=True), ): await publish_trace_fetched( request=_make_request(), @@ -799,9 +805,15 @@ async def _fake_check(**kwargs): ), patch("oss.src.core.events.utils.is_ee", return_value=True), patch( - "ee.src.utils.entitlements.check_entitlements", new=_fake_check, create=True + "ee.src.core.access.entitlements.service.check_entitlements", + new=_fake_check, + create=True, + ), + patch( + "ee.src.core.access.entitlements.service.scope_from", + new=lambda **kw: kw, + create=True, ), - patch("ee.src.utils.entitlements.scope_from", new=lambda **kw: kw, create=True), ): await publish_trace_fetched( request=_make_request(), @@ -826,9 +838,15 @@ async def _fake_check(**kwargs): ), patch("oss.src.core.events.utils.is_ee", return_value=True), patch( - "ee.src.utils.entitlements.check_entitlements", new=_fake_check, create=True + "ee.src.core.access.entitlements.service.check_entitlements", + new=_fake_check, + create=True, + ), + patch( + "ee.src.core.access.entitlements.service.scope_from", + new=lambda **kw: kw, + create=True, ), - patch("ee.src.utils.entitlements.scope_from", new=lambda **kw: kw, create=True), ): await publish_trace_fetched( request=_make_request(), @@ -856,9 +874,15 @@ async def _fake_check(**kwargs): ), patch("oss.src.core.events.utils.is_ee", return_value=True), patch( - "ee.src.utils.entitlements.check_entitlements", new=_fake_check, create=True + "ee.src.core.access.entitlements.service.check_entitlements", + new=_fake_check, + create=True, + ), + patch( + "ee.src.core.access.entitlements.service.scope_from", + new=lambda **kw: kw, + create=True, ), - patch("ee.src.utils.entitlements.scope_from", new=lambda **kw: kw, create=True), ): await publish_trace_fetched( request=request, @@ -886,9 +910,15 @@ async def _fake_check(**kwargs): ), patch("oss.src.core.events.utils.is_ee", return_value=False), patch( - "ee.src.utils.entitlements.check_entitlements", new=_fake_check, create=True + "ee.src.core.access.entitlements.service.check_entitlements", + new=_fake_check, + create=True, + ), + patch( + "ee.src.core.access.entitlements.service.scope_from", + new=lambda **kw: kw, + create=True, ), - patch("ee.src.utils.entitlements.scope_from", new=lambda **kw: kw, create=True), ): await publish_trace_fetched( request=_make_request(), diff --git a/api/oss/tests/pytest/unit/events/test_events_worker_l2.py b/api/oss/tests/pytest/unit/events/test_events_worker_l2.py index c93f6af0b4..6bd6a525a5 100644 --- a/api/oss/tests/pytest/unit/events/test_events_worker_l2.py +++ b/api/oss/tests/pytest/unit/events/test_events_worker_l2.py @@ -7,9 +7,10 @@ called directly with synthetic Redis-stream payloads, and the entitlements helpers are patched. -Note: feature-gating (audit-log access) is intentionally NOT enforced -at ingest; only the counter quota is. The audit flag lives at the -query side (`POST /events/query`). +Note: persisting events (the audit log) is EE-only — OSS runs this worker +for webhook dispatch but does not write to the `events` table. Under EE, +ingest is gated by the `Counter.EVENTS_INGESTED` quota here; the audit +*query* flag (`Flag.AUDIT`) is enforced separately at `POST /events/query`. """ import zlib @@ -236,8 +237,13 @@ async def _fake_check(**kwargs): @pytest.mark.asyncio -async def test_l2_skipped_on_oss(): - """OSS (is_ee=False) → no Counter check, ingest proceeds.""" +async def test_l2_skipped_on_oss_and_not_persisted(): + """OSS (is_ee=False) → no Counter check AND no ingest. + + Persisting the `events` table is the (EE-gated) audit log, so OSS must not + write events. The batch is still returned in `allowed` so webhooks dispatch + normally — OSS gets webhooks without an audit trail. + """ org_id = uuid4() proj_id = uuid4() payload = _make_event_message(organization_id=org_id, project_id=proj_id) @@ -248,9 +254,13 @@ async def test_l2_skipped_on_oss(): with patch("oss.src.tasks.asyncio.events.worker.is_ee", return_value=False): total, processed_ids, allowed = await worker.process_batch(batch) - assert total == 1 + # Nothing persisted in OSS... + assert total == 0 + worker.service.ingest.assert_not_called() + # ...but the batch is still allowed through for webhook dispatch, and the + # message is ACKed so it doesn't linger in the PEL. assert len(allowed) == 1 - worker.service.ingest.assert_awaited_once() + assert len(processed_ids) == 1 @pytest.mark.asyncio diff --git a/api/oss/tests/pytest/unit/test_evaluation_runtime_locks.py b/api/oss/tests/pytest/unit/test_evaluation_runtime_locks.py index 8e739cc170..4cbc3b79f9 100644 --- a/api/oss/tests/pytest/unit/test_evaluation_runtime_locks.py +++ b/api/oss/tests/pytest/unit/test_evaluation_runtime_locks.py @@ -37,6 +37,7 @@ async def fake_redis(): fakeredis = pytest.importorskip("fakeredis") aioredis = pytest.importorskip("fakeredis.aioredis") from oss.src.utils import caching + from oss.src.utils import locking server = fakeredis.FakeServer() client = aioredis.FakeRedis(server=server, decode_responses=False) @@ -47,7 +48,7 @@ async def _renew_lock_for_tests( key=None, project_id=None, user_id=None, - ttl: int = caching.AGENTA_LOCK_TTL, + ttl: int = locking.AGENTA_LOCK_TTL, owner=None, ) -> bool: lock_key = caching.pack( @@ -85,14 +86,19 @@ async def _release_lock_for_tests( return False return bool(await client.delete(lock_key)) + engine = pytest.importorskip("oss.src.dbs.redis.shared.engine") + lock_engine = engine.get_lock_engine() + with ( - patch("oss.src.utils.caching.r_lock", client), + # The lock engine delegates redis ops to `_client()`; point it at the + # fakeredis instance so locking goes through the in-memory server. + patch.object(lock_engine, "_client", return_value=client), patch( - "oss.src.utils.caching.renew_lock", + "oss.src.utils.locking.renew_lock", _renew_lock_for_tests, ), patch( - "oss.src.utils.caching.release_lock", + "oss.src.utils.locking.release_lock", _release_lock_for_tests, ), ): @@ -116,22 +122,16 @@ def _job_id() -> str: def _genson_patch(): module = types.ModuleType("genson") - live_module = types.ModuleType("oss.src.core.evaluations.tasks.live") class SchemaBuilder: ... - async def evaluate_live_query(*args, **kwargs): - return None - module.SchemaBuilder = SchemaBuilder - live_module.evaluate_live_query = evaluate_live_query stack = ExitStack() stack.enter_context( patch.dict( sys.modules, { "genson": module, - "oss.src.core.evaluations.tasks.live": live_module, }, ) ) @@ -473,7 +473,7 @@ async def _failing_coro(): @pytest.mark.asyncio async def test_refresh_worker_heartbeat_preserves_created_at_without_fakeredis(): - from oss.src.core.evaluations.runtime import locks + import oss.src.core.evaluations.runtime.locks as locks class DummyRedis: def __init__(self): @@ -487,9 +487,11 @@ async def set(self, key, value, ex=None): return True dummy = DummyRedis() + engine = pytest.importorskip("oss.src.dbs.redis.shared.engine") + lock_engine = engine.get_lock_engine() with ( - patch("oss.src.utils.caching.r_lock", dummy), + patch.object(lock_engine, "_client", return_value=dummy), patch( "oss.src.core.evaluations.runtime.locks._now_iso", side_effect=["2026-03-25T10:00:00Z", "2026-03-25T10:01:00Z"], @@ -506,7 +508,7 @@ async def set(self, key, value, ex=None): @pytest.mark.asyncio async def test_run_job_heartbeat_fails_after_missing_renew_deadline(): - from oss.src.core.evaluations.runtime import locks + import oss.src.core.evaluations.runtime.locks as locks clock = {"now": 0.0} diff --git a/api/oss/tests/pytest/unit/test_llm_apps_service.py b/api/oss/tests/pytest/unit/test_llm_apps_service.py deleted file mode 100644 index ee1079ba7f..0000000000 --- a/api/oss/tests/pytest/unit/test_llm_apps_service.py +++ /dev/null @@ -1,314 +0,0 @@ -from oss.src.services.llm_apps_service import ( - _build_inspect_url, - _extract_batch_invoke_metadata, - parse_legacy_inputs, - build_invoke_request, - get_parameters_from_inspect, - get_parameters_from_schemas, -) -import pytest - - -def test_get_parameters_from_schemas_prefers_revision_schemas_for_completion(): - parameters, is_chat = get_parameters_from_schemas( - schemas={ - "parameters": { - "type": "object", - "properties": { - "prompt": { - "type": "object", - } - }, - }, - "inputs": { - "type": "object", - "properties": { - "country": { - "type": "string", - } - }, - }, - } - ) - - assert is_chat is False - assert parameters == [ - { - "name": "ag_config", - "type": "dict", - "default": ["prompt"], - }, - { - "name": "inputs", - "type": "dict", - "default": ["country"], - }, - ] - - -def test_get_parameters_from_schemas_detects_chat_messages(): - parameters, is_chat = get_parameters_from_schemas( - schemas={ - "parameters": { - "type": "object", - "properties": { - "prompt": { - "type": "object", - } - }, - }, - "inputs": { - "type": "object", - "properties": { - "messages": { - "type": "array", - "x-ag-type-ref": "messages", - }, - "context": { - "type": "string", - }, - }, - }, - } - ) - - assert is_chat is True - assert parameters == [ - { - "name": "ag_config", - "type": "dict", - "default": ["prompt"], - }, - { - "name": "messages", - "type": "messages", - "default": [], - }, - { - "name": "inputs", - "type": "dict", - "default": ["context"], - }, - ] - - -def test_build_invoke_request_wraps_inputs_for_invoke_endpoint(): - request = build_invoke_request( - inputs={ - "country": "France", - "messages": [ - { - "role": "user", - "content": "What is the capital?", - } - ], - }, - parameters={ - "prompt": { - "messages": [ - { - "role": "system", - "content": "You are an expert in geography.", - } - ] - } - }, - references={ - "application": {"id": "app-id"}, - "application_variant": {"id": "variant-id"}, - "application_revision": {"id": "revision-id"}, - }, - ) - - assert request == { - "references": { - "application": {"id": "app-id"}, - "application_variant": {"id": "variant-id"}, - "application_revision": {"id": "revision-id"}, - }, - "data": { - "parameters": { - "prompt": { - "messages": [ - { - "role": "system", - "content": "You are an expert in geography.", - } - ] - } - }, - "inputs": { - "country": "France", - "messages": [ - { - "role": "user", - "content": "What is the capital?", - } - ], - }, - }, - } - - -def test_build_inspect_url_for_root_route(): - assert ( - _build_inspect_url( - runtime_prefix="http://localhost:8080", - route_path="", - ) - == "http://localhost:8080/inspect" - ) - - -def test_build_inspect_url_for_nested_route(): - assert ( - _build_inspect_url( - runtime_prefix="http://localhost:8080/service/", - route_path="/summarize", - ) - == "http://localhost:8080/service/summarize/inspect" - ) - - -def test_extract_batch_invoke_metadata_prefers_revision_values(): - parameters, schemas, is_chat = _extract_batch_invoke_metadata( - revision={ - "data": { - "parameters": {"prompt": {"messages": [{"role": "system"}]}}, - "schemas": { - "inputs": { - "type": "object", - "properties": {"messages": {"x-ag-type-ref": "messages"}}, - } - }, - }, - "flags": {"is_chat": True}, - }, - parameters=None, - schemas=None, - is_chat=None, - ) - - assert parameters == {"prompt": {"messages": [{"role": "system"}]}} - assert schemas == { - "inputs": { - "type": "object", - "properties": {"messages": {"x-ag-type-ref": "messages"}}, - } - } - assert is_chat is True - - -def test_extract_batch_invoke_metadata_prefers_explicit_overrides(): - parameters, schemas, is_chat = _extract_batch_invoke_metadata( - revision={ - "data": { - "parameters": {"prompt": {"temperature": 0.1}}, - "schemas": {"inputs": {"type": "object", "properties": {}}}, - }, - "flags": {"is_chat": False}, - }, - parameters={"prompt": {"temperature": 0.7}}, - schemas={"inputs": {"type": "object", "properties": {"country": {}}}}, - is_chat=True, - ) - - assert parameters == {"prompt": {"temperature": 0.7}} - assert schemas == {"inputs": {"type": "object", "properties": {"country": {}}}} - assert is_chat is True - - -def test_parse_legacy_inputs_prefers_input_keys_over_full_datapoint(): - inputs = parse_legacy_inputs( - datapoint={ - "country": "Spain", - "correct_answer": "Madrid", - "testcase_id": "tc-1", - "testcase_dedup_id": "dedup-1", - }, - parameters={ - "prompt": { - "input_keys": ["country"], - } - }, - ) - - assert inputs == {"country": "Spain"} - - -def test_parse_legacy_inputs_falls_back_to_schema_inputs_when_input_keys_missing(): - inputs = parse_legacy_inputs( - datapoint={ - "country": "Germany", - "correct_answer": "Berlin", - "testcase_id": "tc-2", - "testcase_dedup_id": "dedup-2", - }, - parameters={}, - schemas={ - "inputs": { - "type": "object", - "properties": { - "country": {"type": "string"}, - }, - } - }, - ) - - assert inputs == {"country": "Germany"} - - -@pytest.mark.asyncio -async def test_get_parameters_from_inspect_prefers_revision_schemas(monkeypatch): - async def fake_post_json_to_uri(uri, headers, body): - assert uri == "http://localhost:8080/inspect" - assert headers == {"Authorization": "Secret test"} - assert body == {} - return { - "flags": {"is_chat": False}, - "data": { - "revision": { - "data": { - "schemas": { - "parameters": { - "type": "object", - "properties": { - "prompt": {"type": "object"}, - }, - }, - "inputs": { - "type": "object", - "properties": { - "country": {"type": "string"}, - }, - }, - } - } - } - }, - } - - monkeypatch.setattr( - "oss.src.services.llm_apps_service._post_json_to_uri", - fake_post_json_to_uri, - ) - - parameters, is_chat = await get_parameters_from_inspect( - runtime_prefix="http://localhost:8080", - route_path="", - headers={"Authorization": "Secret test"}, - ) - - assert is_chat is False - assert parameters == [ - { - "name": "ag_config", - "type": "dict", - "default": ["prompt"], - }, - { - "name": "inputs", - "type": "dict", - "default": ["country"], - }, - ] diff --git a/api/oss/tests/pytest/unit/tracing/utils/test_parsing.py b/api/oss/tests/pytest/unit/tracing/utils/test_parsing.py index 24aec405ac..61b2d1797e 100644 --- a/api/oss/tests/pytest/unit/tracing/utils/test_parsing.py +++ b/api/oss/tests/pytest/unit/tracing/utils/test_parsing.py @@ -1,6 +1,5 @@ from copy import deepcopy from datetime import datetime -from uuid import UUID import pytest @@ -116,12 +115,10 @@ def test_parse_id_helpers_raise_on_invalid_values(fn, value): def test_parse_uuid_to_wire_ids(): - trace_uuid = UUID(TRACE_UUID) - span_uuid = UUID(SPAN_UUID) - - assert parse_trace_id_from_uuid(trace_uuid) == TRACE_HEX + # The DTO/domain layer holds trace/span ids as canonical hex strings, so + # these parsers take a str. UUID-style (dashed) input is accepted too, + # since str(UUID(...)) normalizes it. assert parse_trace_id_from_uuid(TRACE_UUID) == TRACE_HEX - assert parse_span_id_from_uuid(span_uuid) == SPAN_HEX assert parse_span_id_from_uuid(SPAN_UUID) == SPAN_HEX diff --git a/api/pyproject.toml b/api/pyproject.toml index 87c13c060b..b8061f33f0 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -52,6 +52,10 @@ dev = [ "pytest-mock>=3,<4", "pytest-html>=4,<5", "fakeredis>=2.34.1,<3", + # Starlette 1.2's TestClient prefers httpx2 and warns when only httpx is + # present. Runtime code stays on httpx (0.28); this test-only dep silences + # the deprecation without touching the runtime HTTP stack. + "httpx2>=2.3,<3", ] [tool.uv] diff --git a/api/uv.lock b/api/uv.lock index 737df876bb..bbf258a44b 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -33,7 +33,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "agenta-client", editable = "../clients/python" }, - { name = "daytona", specifier = ">=0.183,<0.184" }, + { name = "daytona", specifier = ">=0.184,<0.185" }, { name = "fastapi", specifier = ">=0.136,<0.137" }, { name = "httpx", specifier = ">=0.28,<0.29" }, { name = "jinja2", specifier = ">=3,<4" }, @@ -300,6 +300,7 @@ dependencies = [ dev = [ { name = "click" }, { name = "fakeredis" }, + { name = "httpx2" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-html" }, @@ -348,6 +349,7 @@ requires-dist = [ dev = [ { name = "click", specifier = ">=8,<9" }, { name = "fakeredis", specifier = ">=2.34.1,<3" }, + { name = "httpx2", specifier = ">=2.3,<3" }, { name = "pytest", specifier = ">=9,<10" }, { name = "pytest-asyncio", specifier = ">=1,<2" }, { name = "pytest-html", specifier = ">=4,<5" }, @@ -615,7 +617,7 @@ wheels = [ [[package]] name = "daytona" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, @@ -639,14 +641,14 @@ dependencies = [ { name = "urllib3" }, { name = "wsproto" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c8/dd/3914a2ae3dc4ecf3fd861eff221f4222e48ba4e5530ff7e585d4c4aa7965/daytona-0.183.0.tar.gz", hash = "sha256:2f0020537712ae67693fd2341d3008d464ada06dcd8e89baa6183846271e2f76", size = 141636, upload-time = "2026-05-29T10:59:50.565Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/6d/482ed38e868e5d6904961bf8c0317a94981bdd2b165f2611acb4f7094357/daytona-0.184.0.tar.gz", hash = "sha256:38a8de4a2daf34cb5940590db4d0f816ee8354d9919ca8ab1051df4a38f2107c", size = 141946, upload-time = "2026-06-03T14:46:56.231Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/ae/b23f7e5381ca5d66f2be0f802ebef0110b5e21022946fa7b9d6e873ec6ad/daytona-0.183.0-py3-none-any.whl", hash = "sha256:89af1bcfc9b37c580329fcad01167bcfab786330e2d0ae74ad3fae8cb40397d2", size = 172192, upload-time = "2026-05-29T10:59:48.838Z" }, + { url = "https://files.pythonhosted.org/packages/10/da/5ff417bbf0ca7d480ee88c310372fec3ab039a66878c27ecab97203d00c0/daytona-0.184.0-py3-none-any.whl", hash = "sha256:630df33c62aebae2ae34679eff79198806014567850c98dcd8d45accdcc5437a", size = 172586, upload-time = "2026-06-03T14:46:54.467Z" }, ] [[package]] name = "daytona-api-client" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -654,14 +656,14 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/40/a5090268e734f804489b8ca3dcdbd4de7c392b1b4d4b02a60c5d85c7198f/daytona_api_client-0.183.0.tar.gz", hash = "sha256:659fff84a6461d1f51986fb814bf6e3fbc51ac203e6502eb6ae6f26029d69814", size = 147714, upload-time = "2026-05-29T10:59:01.709Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/63/cdb3ea6dd35fd2091a7e1f988ce1d3194d9bec1accfa6dbbf63ad424e671/daytona_api_client-0.184.0.tar.gz", hash = "sha256:3cdb111cf2a21be13cc648e444162c094d45f80d1eeb638b671fc854be307c23", size = 148346, upload-time = "2026-06-03T14:46:09.888Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/70/6a/8890e773ecf1cd9c1b70ddb0fda57e4941f5a8a44ced1c91574253797f63/daytona_api_client-0.183.0-py3-none-any.whl", hash = "sha256:c319ae3b4666ec15402cacb8ec3aaffbf55802c0dbc33f6826ef07afceae5085", size = 406274, upload-time = "2026-05-29T10:59:00.427Z" }, + { url = "https://files.pythonhosted.org/packages/88/65/44993d1bb3cb1f5f294ac97bb3246346c14bf7ba3e79cdd4e92c7df0c6df/daytona_api_client-0.184.0-py3-none-any.whl", hash = "sha256:8fec5062d757fb533e82e9bf295fc1625ae294b4a154c24081bbd2bf30bcb5dd", size = 407600, upload-time = "2026-06-03T14:46:08.086Z" }, ] [[package]] name = "daytona-api-client-async" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -670,14 +672,14 @@ dependencies = [ { name = "python-dateutil" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/90/aa/c9de46d48ee45228566a53bb699bc4bdab693dc8ce300516ce334bfec82f/daytona_api_client_async-0.183.0.tar.gz", hash = "sha256:4f5b397e375635fe3614c342cf34faa236d5a0918c53588d7059b5061d881407", size = 148190, upload-time = "2026-05-29T10:59:14.315Z" } +sdist = { url = "https://files.pythonhosted.org/packages/df/2f/caba484cbac693d269b417df73237f117882e2e28d1af4c3878e568bf148/daytona_api_client_async-0.184.0.tar.gz", hash = "sha256:a3cb0ebe5eec2c824b7848d4189a035a13cd84de1e0083c3f0c033f5445f3b71", size = 149131, upload-time = "2026-06-03T14:46:19.64Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/71/db/dde2e2478faf010b048af24227e5ff8b4c635d8104ace26c66ed1f29dd49/daytona_api_client_async-0.183.0-py3-none-any.whl", hash = "sha256:8be2b0bac344893294a8ffd8df7a4703427ec48376f6948975aca85f09dba541", size = 409552, upload-time = "2026-05-29T10:59:12.964Z" }, + { url = "https://files.pythonhosted.org/packages/8e/fc/c6e304db0bb382a0842cda913474ba42060acbf4f6cc91d8708f7997e337/daytona_api_client_async-0.184.0-py3-none-any.whl", hash = "sha256:19f268e6fb4d4190e7a4ea2ad3f4600bf14c25ab8f4997390d8d07d4a281de61", size = 410889, upload-time = "2026-06-03T14:46:18.051Z" }, ] [[package]] name = "daytona-toolbox-api-client" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -685,14 +687,14 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ac/ee/1c8b9785c4028a0932f655e131e44417380a1e3d6ff4eb58b4e10b9aac18/daytona_toolbox_api_client-0.183.0.tar.gz", hash = "sha256:7cf4eb8ab36ec488f436695822b0e4c06e82d12d8fb2fe28e3239151dd43c4b7", size = 77732, upload-time = "2026-05-29T10:58:40.866Z" } +sdist = { url = "https://files.pythonhosted.org/packages/26/8f/78b0c06e91065e573ce1f2f4c6b7941066be939de46128a7e97a30b2e4a7/daytona_toolbox_api_client-0.184.0.tar.gz", hash = "sha256:c325a050ca45e7832c7d05127260634ed0742d8bc75b2d9de65847153fba70e6", size = 77893, upload-time = "2026-06-03T14:45:52.776Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/3a/904bbf95eac89036775051410dbaf55414a73a1e6d074e1486ca29ee3dbb/daytona_toolbox_api_client-0.183.0-py3-none-any.whl", hash = "sha256:bc0c2fb986f50cad23791afa181c04f27d975b59301d7ce59ce519c1f518483c", size = 205626, upload-time = "2026-05-29T10:58:39.381Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f9/311c8a6d8fdbb48793c1d5504fa212b0325047887e1c8c8dc2560687cff6/daytona_toolbox_api_client-0.184.0-py3-none-any.whl", hash = "sha256:05b6b179018247bf3e7de667b049b0d7bc0e5e83f58b498e0371988e61b3041e", size = 205757, upload-time = "2026-06-03T14:45:51.343Z" }, ] [[package]] name = "daytona-toolbox-api-client-async" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -701,9 +703,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/0f/85e2ad6511ae228e1c423944d6d90ba3ace2153685ba8aa3b4035c5ccc6f/daytona_toolbox_api_client_async-0.183.0.tar.gz", hash = "sha256:31984e9fc5c440efbb5d23426f187046fa419c075c42d8691b127304692c4e6b", size = 71347, upload-time = "2026-05-29T10:59:03.772Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/cc/d10617b9bf1eae7f61870d0f98d5f22e6a56b0777ee9643427be5d8ec488/daytona_toolbox_api_client_async-0.184.0.tar.gz", hash = "sha256:495526bbd8de1d1ce2d86ecdb561efd5c4e7e86007abf4bc550ecc0c5efaf2e9", size = 71490, upload-time = "2026-06-03T14:46:20.57Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/23/66/af01390a47ba626454fd2ba44766282ffb7174ecc60c1742ab4859bd074f/daytona_toolbox_api_client_async-0.183.0-py3-none-any.whl", hash = "sha256:54a25297cfaf68568d85cecfd5c07da2e7fd862c8ab445df97b76a360951f757", size = 203902, upload-time = "2026-05-29T10:59:02.046Z" }, + { url = "https://files.pythonhosted.org/packages/db/71/d6e98539e8a855bc63a55ce60f554f0b0063e9275424fea0476eb46f3a2a/daytona_toolbox_api_client_async-0.184.0-py3-none-any.whl", hash = "sha256:b971bc3475c0f658c38a86de40a5d4d31394bf60ad4b4c7f8b7693a5a56647b1", size = 204036, upload-time = "2026-06-03T14:46:18.771Z" }, ] [[package]] @@ -1030,6 +1032,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/34/18f1c596e677962f040284246f393b10a1f8ce440b3a7e69c637d0f1c7ad/httpcore2-2.3.0.tar.gz", hash = "sha256:07327e251560960eea8e969d92d4c6a325feb13cca39e25340731336c3baf924", size = 64300, upload-time = "2026-06-01T13:15:02.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/dd/3357218c69360d1cecc196c230c9a1d5c9afd5dba362056e23e60a5e64e5/httpcore2-2.3.0-py3-none-any.whl", hash = "sha256:477e9e334f74e5240dcac002e890580f36a57d40ff0fb14cc9655731d23b8415", size = 80024, upload-time = "2026-06-01T13:15:00.001Z" }, +] + [[package]] name = "httptools" version = "0.8.0" @@ -1089,6 +1104,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/f8/a6bc80313a9e93c888fa10534dfce2ad76ff86911b6f485777ce6de6a073/httpx_ws-0.9.0-py3-none-any.whl", hash = "sha256:71640d2fb1bf9a225775015b33cd755cfd4c5f7e21c885192fe3adc4c387b248", size = 15759, upload-time = "2026-03-28T14:11:11.887Z" }, ] +[[package]] +name = "httpx2" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpcore2" }, + { name = "idna" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/9a/cca0b9145f13d8ae34b885ae28d403a1469a433abc78e0f94f4ce94e650b/httpx2-2.3.0.tar.gz", hash = "sha256:227e7c41d95a76d4077a52640564132777215fc3394e07b66a3116c33d668fa9", size = 81115, upload-time = "2026-06-01T13:15:04.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/ce/ae2911859847f9ba1d6b23027e53481cbeb50b93234f355a968d300ca2cb/httpx2-2.3.0-py3-none-any.whl", hash = "sha256:6f393663bdf6dbe7fe90118e3eb5b2bd024a675cae0390ac08cec9198812d8b7", size = 74538, upload-time = "2026-06-01T13:15:01.566Z" }, +] + [[package]] name = "huggingface-hub" version = "1.17.0" @@ -1112,11 +1142,11 @@ wheels = [ [[package]] name = "idna" -version = "3.17" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/28/99c51f664567218d824af024c0251650fb27e4ca066df188dab0769c5b91/idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f", size = 196048, upload-time = "2026-05-28T14:32:38.55Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/a7/f76514cc40ad6234098ecdebda08732d75964776c51a42845b7da10649e2/idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c", size = 65316, upload-time = "2026-05-28T14:32:37.035Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -1762,7 +1792,7 @@ wheels = [ [[package]] name = "posthog" -version = "7.16.3" +version = "7.16.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -1770,9 +1800,9 @@ dependencies = [ { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/20/d4/6e1c24823c515683cb6f74bd1dcc3fbe55f60a27743c1694bdc61c5db5d0/posthog-7.16.3.tar.gz", hash = "sha256:ff8972813c836ae4fcb634b499cf06d643bc3c4f49931218fe142e7aa8a39810", size = 226535, upload-time = "2026-06-01T13:24:07.781Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/af/10ba67866a1246e608fccf0d186c63b55054feb66c0fc9d77be458aac9c0/posthog-7.16.4.tar.gz", hash = "sha256:a2baa6595c7089865350826932867ed31355e27307ae4774bc88d42536678848", size = 228616, upload-time = "2026-06-03T13:12:16.006Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/0b/562afe6f037ab3dde06d2bbe33f1de9f8516ac22f882ad263e4e0bfb665b/posthog-7.16.3-py3-none-any.whl", hash = "sha256:f0cbaf25ac06211b87c0a43500673fa2d8de86eb1edb75b8bf688f1b884878ce", size = 264300, upload-time = "2026-06-01T13:24:06.074Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b2/4c9659e06074b473216dc69360fcf7ddba9134e622d16ef10bb45da5cc91/posthog-7.16.4-py3-none-any.whl", hash = "sha256:92ede0237aafb7b90cdb730dacf828cf915676366a69522a76d300fa580f0da0", size = 267073, upload-time = "2026-06-03T13:12:14.415Z" }, ] [[package]] @@ -2721,6 +2751,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "twilio" version = "9.10.9" diff --git a/clients/python/agenta_client/__init__.py b/clients/python/agenta_client/__init__.py index 546e96673c..9500a326f9 100644 --- a/clients/python/agenta_client/__init__.py +++ b/clients/python/agenta_client/__init__.py @@ -5,7 +5,7 @@ import typing from importlib import import_module if typing.TYPE_CHECKING: - from .types import AdminAccountCreateOptions, AdminAccountRead, AdminAccountsCreate, AdminAccountsDelete, AdminAccountsDeleteTarget, AdminAccountsResponse, AdminApiKeyCreate, AdminApiKeyResponse, AdminDeleteResponse, AdminDeletedEntities, AdminDeletedEntity, AdminOrganizationCreate, AdminOrganizationMembershipCreate, AdminOrganizationMembershipRead, AdminOrganizationRead, AdminProjectCreate, AdminProjectMembershipCreate, AdminProjectMembershipRead, AdminProjectRead, AdminSimpleAccountCreate, AdminSimpleAccountDeleteEntry, AdminSimpleAccountRead, AdminSimpleAccountsApiKeysCreate, AdminSimpleAccountsCreate, AdminSimpleAccountsDelete, AdminSimpleAccountsOrganizationsCreate, AdminSimpleAccountsOrganizationsMembershipsCreate, AdminSimpleAccountsOrganizationsTransferOwnership, AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjects, AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjectsZero, AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspaces, AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspacesZero, AdminSimpleAccountsOrganizationsTransferOwnershipResponse, AdminSimpleAccountsProjectsCreate, AdminSimpleAccountsProjectsMembershipsCreate, AdminSimpleAccountsResponse, AdminSimpleAccountsUsersCreate, AdminSimpleAccountsUsersIdentitiesCreate, AdminSimpleAccountsUsersResetPassword, AdminSimpleAccountsWorkspacesCreate, AdminSimpleAccountsWorkspacesMembershipsCreate, AdminStructuredError, AdminSubscriptionCreate, AdminSubscriptionRead, AdminUserCreate, AdminUserIdentityCreate, AdminUserIdentityRead, AdminUserIdentityReadStatus, AdminUserRead, AdminWorkspaceCreate, AdminWorkspaceMembershipCreate, AdminWorkspaceMembershipRead, AdminWorkspaceRead, Analytics, AnalyticsResponse, Annotation, AnnotationCreate, AnnotationCreateLinks, AnnotationEdit, AnnotationEditLinks, AnnotationLinkResponse, AnnotationLinks, AnnotationQuery, AnnotationQueryLinks, AnnotationResponse, AnnotationsResponse, Application, ApplicationArtifactFlags, ApplicationArtifactQueryFlags, ApplicationCatalogPreset, ApplicationCatalogPresetResponse, ApplicationCatalogPresetsResponse, ApplicationCatalogTemplate, ApplicationCatalogTemplateResponse, ApplicationCatalogTemplatesResponse, ApplicationCatalogType, ApplicationCatalogTypesResponse, ApplicationCreate, ApplicationEdit, ApplicationFlags, ApplicationFork, ApplicationQuery, ApplicationResponse, ApplicationRevision, ApplicationRevisionCommit, ApplicationRevisionCreate, ApplicationRevisionDataInput, ApplicationRevisionDataInputHeadersValue, ApplicationRevisionDataInputRuntime, ApplicationRevisionDataOutput, ApplicationRevisionDataOutputHeadersValue, ApplicationRevisionDataOutputRuntime, ApplicationRevisionEdit, ApplicationRevisionFlags, ApplicationRevisionFork, ApplicationRevisionQuery, ApplicationRevisionQueryFlags, ApplicationRevisionResolveResponse, ApplicationRevisionResponse, ApplicationRevisionsLog, ApplicationRevisionsResponse, ApplicationVariant, ApplicationVariantCreate, ApplicationVariantEdit, ApplicationVariantFlags, ApplicationVariantFork, ApplicationVariantResponse, ApplicationVariantsResponse, ApplicationsResponse, BodyConfigsFetchVariantsConfigsFetchPost, Bucket, CollectStatusResponse, ComparisonOperator, Condition, ConditionOperator, ConditionOptions, ConditionValue, ConfigResponseModel, CustomModelSettingsDto, CustomProviderDto, CustomProviderKind, CustomProviderSettingsDto, DictOperator, DiscoverResponse, DiscoverResponseMethodsValue, EeSrcModelsApiOrganizationModelsOrganization, EntityRef, Environment, EnvironmentCreate, EnvironmentEdit, EnvironmentFlags, EnvironmentQueryFlags, EnvironmentResponse, EnvironmentRevision, EnvironmentRevisionCommit, EnvironmentRevisionCreate, EnvironmentRevisionData, EnvironmentRevisionDelta, EnvironmentRevisionEdit, EnvironmentRevisionResolveResponse, EnvironmentRevisionResponse, EnvironmentRevisionsLog, EnvironmentRevisionsResponse, EnvironmentVariant, EnvironmentVariantCreate, EnvironmentVariantEdit, EnvironmentVariantResponse, EnvironmentVariantsResponse, EnvironmentsResponse, ErrorPolicy, EvaluationMetrics, EvaluationMetricsCreate, EvaluationMetricsEdit, EvaluationMetricsIdsResponse, EvaluationMetricsQuery, EvaluationMetricsQueryScenarioIds, EvaluationMetricsQueryTimestamps, EvaluationMetricsRefresh, EvaluationMetricsResponse, EvaluationQueue, EvaluationQueueCreate, EvaluationQueueData, EvaluationQueueEdit, EvaluationQueueFlags, EvaluationQueueIdResponse, EvaluationQueueIdsResponse, EvaluationQueueQuery, EvaluationQueueQueryFlags, EvaluationQueueResponse, EvaluationQueueScenariosQuery, EvaluationQueuesResponse, EvaluationResult, EvaluationResultCreate, EvaluationResultEdit, EvaluationResultIdResponse, EvaluationResultIdsResponse, EvaluationResultQuery, EvaluationResultResponse, EvaluationResultsResponse, EvaluationRun, EvaluationRunCreate, EvaluationRunData, EvaluationRunDataMapping, EvaluationRunDataMappingColumn, EvaluationRunDataMappingStep, EvaluationRunDataStep, EvaluationRunDataStepInput, EvaluationRunDataStepOrigin, EvaluationRunDataStepType, EvaluationRunEdit, EvaluationRunFlags, EvaluationRunIdResponse, EvaluationRunIdsRequest, EvaluationRunIdsResponse, EvaluationRunQuery, EvaluationRunQueryFlags, EvaluationRunResponse, EvaluationRunsResponse, EvaluationScenario, EvaluationScenarioCreate, EvaluationScenarioEdit, EvaluationScenarioIdResponse, EvaluationScenarioIdsResponse, EvaluationScenarioQuery, EvaluationScenarioResponse, EvaluationScenariosResponse, EvaluationStatus, Evaluator, EvaluatorArtifactFlags, EvaluatorArtifactQueryFlags, EvaluatorCatalogPreset, EvaluatorCatalogPresetResponse, EvaluatorCatalogPresetsResponse, EvaluatorCatalogTemplate, EvaluatorCatalogTemplateResponse, EvaluatorCatalogTemplatesResponse, EvaluatorCatalogType, EvaluatorCatalogTypesResponse, EvaluatorCreate, EvaluatorEdit, EvaluatorFlags, EvaluatorFork, EvaluatorQuery, EvaluatorResponse, EvaluatorRevision, EvaluatorRevisionCommit, EvaluatorRevisionCreate, EvaluatorRevisionDataInput, EvaluatorRevisionDataInputHeadersValue, EvaluatorRevisionDataInputRuntime, EvaluatorRevisionDataOutput, EvaluatorRevisionDataOutputHeadersValue, EvaluatorRevisionDataOutputRuntime, EvaluatorRevisionEdit, EvaluatorRevisionFlags, EvaluatorRevisionFork, EvaluatorRevisionQuery, EvaluatorRevisionQueryFlags, EvaluatorRevisionResolveResponse, EvaluatorRevisionResponse, EvaluatorRevisionsLog, EvaluatorRevisionsResponse, EvaluatorTemplate, EvaluatorTemplatesResponse, EvaluatorVariant, EvaluatorVariantCreate, EvaluatorVariantEdit, EvaluatorVariantFlags, EvaluatorVariantFork, EvaluatorVariantResponse, EvaluatorVariantsResponse, EvaluatorsResponse, Event, EventQuery, EventType, EventsQueryResponse, ExistenceOperator, FilteringInput, FilteringInputConditionsItem, FilteringOutput, FilteringOutputConditionsItem, Focus, Folder, FolderCreate, FolderEdit, FolderIdResponse, FolderKind, FolderQuery, FolderQueryKinds, FolderResponse, FoldersResponse, Format, Formatting, FullJsonInput, FullJsonOutput, Header, HttpValidationError, InviteRequest, Invocation, InvocationCreate, InvocationCreateLinks, InvocationEdit, InvocationEditLinks, InvocationLinkResponse, InvocationLinks, InvocationQuery, InvocationQueryLinks, InvocationResponse, InvocationsResponse, JsonSchemasInput, JsonSchemasOutput, LabelJsonInput, LabelJsonOutput, LegacyLifecycleDto, ListApiKeysResponse, ListOperator, ListOptions, LogicalOperator, MetricSpec, MetricType, MetricsBucket, NumericOperator, OTelEventInput, OTelEventInputTimestamp, OTelEventOutput, OTelEventOutputTimestamp, OTelHashInput, OTelHashOutput, OTelLinkInput, OTelLinkOutput, OTelLinksResponse, OTelReferenceInput, OTelReferenceOutput, OTelSpanKind, OTelStatusCode, OTelTracingRequest, OTelTracingResponse, OldAnalyticsResponse, OrganizationDetails, OrganizationDomainResponse, OrganizationProviderResponse, OrganizationUpdate, OssSrcModelsApiOrganizationModelsOrganization, Permission, ProjectsResponse, QueriesResponse, Query, QueryCreate, QueryEdit, QueryFlags, QueryQueryFlags, QueryResponse, QueryRevision, QueryRevisionCommit, QueryRevisionCreate, QueryRevisionDataInput, QueryRevisionDataOutput, QueryRevisionEdit, QueryRevisionQuery, QueryRevisionResponse, QueryRevisionsLog, QueryRevisionsResponse, QueryVariant, QueryVariantCreate, QueryVariantEdit, QueryVariantQuery, QueryVariantResponse, QueryVariantsResponse, Reference, ReferenceRequestModelInput, ReferenceRequestModelOutput, RequestType, ResolutionInfo, RetrievalInfo, RevisionFork, SecretDto, SecretDtoData, SecretKind, SecretResponseDto, SecretResponseDtoData, SessionIdsResponse, SimpleApplication, SimpleApplicationCreate, SimpleApplicationDataInput, SimpleApplicationDataInputHeadersValue, SimpleApplicationDataInputRuntime, SimpleApplicationDataOutput, SimpleApplicationDataOutputHeadersValue, SimpleApplicationDataOutputRuntime, SimpleApplicationEdit, SimpleApplicationFlags, SimpleApplicationQuery, SimpleApplicationQueryFlags, SimpleApplicationResponse, SimpleApplicationsResponse, SimpleEnvironment, SimpleEnvironmentCreate, SimpleEnvironmentEdit, SimpleEnvironmentQuery, SimpleEnvironmentResponse, SimpleEnvironmentsResponse, SimpleEvaluation, SimpleEvaluationCreate, SimpleEvaluationData, SimpleEvaluationDataApplicationSteps, SimpleEvaluationDataApplicationStepsOneValue, SimpleEvaluationDataEvaluatorSteps, SimpleEvaluationDataEvaluatorStepsOneValue, SimpleEvaluationDataQuerySteps, SimpleEvaluationDataQueryStepsOneValue, SimpleEvaluationDataTestsetSteps, SimpleEvaluationDataTestsetStepsOneValue, SimpleEvaluationEdit, SimpleEvaluationIdResponse, SimpleEvaluationQuery, SimpleEvaluationResponse, SimpleEvaluationsResponse, SimpleEvaluator, SimpleEvaluatorCreate, SimpleEvaluatorDataInput, SimpleEvaluatorDataInputHeadersValue, SimpleEvaluatorDataInputRuntime, SimpleEvaluatorDataOutput, SimpleEvaluatorDataOutputHeadersValue, SimpleEvaluatorDataOutputRuntime, SimpleEvaluatorEdit, SimpleEvaluatorFlags, SimpleEvaluatorQuery, SimpleEvaluatorQueryFlags, SimpleEvaluatorResponse, SimpleEvaluatorsResponse, SimpleQueriesResponse, SimpleQuery, SimpleQueryCreate, SimpleQueryEdit, SimpleQueryQuery, SimpleQueryResponse, SimpleQueue, SimpleQueueCreate, SimpleQueueData, SimpleQueueDataEvaluators, SimpleQueueDataEvaluatorsOneValue, SimpleQueueIdResponse, SimpleQueueKind, SimpleQueueQuery, SimpleQueueResponse, SimpleQueueScenariosQuery, SimpleQueueScenariosResponse, SimpleQueueSettings, SimpleQueuesResponse, SimpleTestset, SimpleTestsetCreate, SimpleTestsetEdit, SimpleTestsetQuery, SimpleTestsetResponse, SimpleTestsetsResponse, SimpleTrace, SimpleTraceChannel, SimpleTraceCreate, SimpleTraceCreateLinks, SimpleTraceEdit, SimpleTraceEditLinks, SimpleTraceKind, SimpleTraceLinkResponse, SimpleTraceLinks, SimpleTraceOrigin, SimpleTraceQuery, SimpleTraceQueryLinks, SimpleTraceReferences, SimpleTraceResponse, SimpleTracesResponse, SimpleWorkflow, SimpleWorkflowCreate, SimpleWorkflowDataInput, SimpleWorkflowDataInputHeadersValue, SimpleWorkflowDataInputRuntime, SimpleWorkflowDataOutput, SimpleWorkflowDataOutputHeadersValue, SimpleWorkflowDataOutputRuntime, SimpleWorkflowEdit, SimpleWorkflowFlags, SimpleWorkflowQuery, SimpleWorkflowQueryFlags, SimpleWorkflowResponse, SimpleWorkflowsResponse, SpanInput, SpanInputEndTime, SpanInputStartTime, SpanOutput, SpanOutputEndTime, SpanOutputStartTime, SpanResponse, SpanType, SpansNodeInput, SpansNodeInputEndTime, SpansNodeInputSpansValue, SpansNodeInputStartTime, SpansNodeOutput, SpansNodeOutputEndTime, SpansNodeOutputSpansValue, SpansNodeOutputStartTime, SpansResponse, SpansTreeInput, SpansTreeInputSpansValue, SpansTreeOutput, SpansTreeOutputSpansValue, SsoProviderDto, SsoProviderInfo, SsoProviderSettingsDto, SsoProviders, StandardProviderDto, StandardProviderKind, StandardProviderSettingsDto, Status, StringOperator, TestcaseInput, TestcaseOutput, TestcaseResponse, TestcasesResponse, Testset, TestsetCreate, TestsetEdit, TestsetFlags, TestsetQuery, TestsetResponse, TestsetRevision, TestsetRevisionCommit, TestsetRevisionCreate, TestsetRevisionDataInput, TestsetRevisionDataOutput, TestsetRevisionDelta, TestsetRevisionDeltaColumns, TestsetRevisionDeltaRows, TestsetRevisionEdit, TestsetRevisionQuery, TestsetRevisionResponse, TestsetRevisionsLog, TestsetRevisionsResponse, TestsetVariant, TestsetVariantCreate, TestsetVariantEdit, TestsetVariantQuery, TestsetVariantResponse, TestsetVariantsResponse, TestsetsResponse, TextOptions, ToolAuthScheme, ToolCallData, ToolCallFunction, ToolCallResponse, ToolCatalogAction, ToolCatalogActionDetails, ToolCatalogActionResponse, ToolCatalogActionResponseAction, ToolCatalogActionsResponse, ToolCatalogActionsResponseActionsItem, ToolCatalogIntegration, ToolCatalogIntegrationDetails, ToolCatalogIntegrationResponse, ToolCatalogIntegrationResponseIntegration, ToolCatalogIntegrationsResponse, ToolCatalogIntegrationsResponseIntegrationsItem, ToolCatalogProvider, ToolCatalogProviderDetails, ToolCatalogProviderResponse, ToolCatalogProviderResponseProvider, ToolCatalogProvidersResponse, ToolCatalogProvidersResponseProvidersItem, ToolConnection, ToolConnectionCreate, ToolConnectionCreateData, ToolConnectionResponse, ToolConnectionStatus, ToolConnectionsResponse, ToolProviderKind, ToolResult, ToolResultData, TraceIdResponse, TraceIdsResponse, TraceInput, TraceInputSpansValue, TraceOutput, TraceOutputSpansValue, TraceRequest, TraceResponse, TraceType, TracesRequest, TracesResponse, TracingQuery, UserIdsResponse, ValidationError, ValidationErrorLocItem, VariantFork, WebhookDeliveriesResponse, WebhookDelivery, WebhookDeliveryCreate, WebhookDeliveryData, WebhookDeliveryQuery, WebhookDeliveryResponse, WebhookDeliveryResponseInfo, WebhookEventType, WebhookProviderDto, WebhookProviderSettingsDto, WebhookSubscription, WebhookSubscriptionCreate, WebhookSubscriptionData, WebhookSubscriptionDataAuthMode, WebhookSubscriptionEdit, WebhookSubscriptionQuery, WebhookSubscriptionResponse, WebhookSubscriptionsResponse, Windowing, WindowingOrder, Workflow, WorkflowArtifactFlags, WorkflowCatalogFlags, WorkflowCatalogPreset, WorkflowCatalogPresetResponse, WorkflowCatalogPresetsResponse, WorkflowCatalogTemplate, WorkflowCatalogTemplateResponse, WorkflowCatalogTemplatesResponse, WorkflowCatalogType, WorkflowCatalogTypeResponse, WorkflowCatalogTypesResponse, WorkflowCreate, WorkflowEdit, WorkflowFlags, WorkflowFork, WorkflowResponse, WorkflowRevisionCommit, WorkflowRevisionCreate, WorkflowRevisionDataInput, WorkflowRevisionDataInputHeadersValue, WorkflowRevisionDataInputRuntime, WorkflowRevisionDataOutput, WorkflowRevisionDataOutputHeadersValue, WorkflowRevisionDataOutputRuntime, WorkflowRevisionEdit, WorkflowRevisionFlags, WorkflowRevisionFork, WorkflowRevisionInput, WorkflowRevisionOutput, WorkflowRevisionResolveResponse, WorkflowRevisionResponse, WorkflowRevisionsLog, WorkflowRevisionsResponse, WorkflowVariant, WorkflowVariantCreate, WorkflowVariantEdit, WorkflowVariantFlags, WorkflowVariantFork, WorkflowVariantResponse, WorkflowVariantsResponse, WorkflowsResponse, Workspace, WorkspaceMemberResponse, WorkspacePermission, WorkspaceResponse + from .types import AdminAccountCreateOptions, AdminAccountRead, AdminAccountsCreate, AdminAccountsDelete, AdminAccountsDeleteTarget, AdminAccountsResponse, AdminApiKeyCreate, AdminApiKeyResponse, AdminDeleteResponse, AdminDeletedEntities, AdminDeletedEntity, AdminOrganizationCreate, AdminOrganizationMembershipCreate, AdminOrganizationMembershipRead, AdminOrganizationRead, AdminProjectCreate, AdminProjectMembershipCreate, AdminProjectMembershipRead, AdminProjectRead, AdminSimpleAccountCreate, AdminSimpleAccountDeleteEntry, AdminSimpleAccountRead, AdminSimpleAccountsApiKeysCreate, AdminSimpleAccountsCreate, AdminSimpleAccountsDelete, AdminSimpleAccountsOrganizationsCreate, AdminSimpleAccountsOrganizationsMembershipsCreate, AdminSimpleAccountsOrganizationsTransferOwnership, AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjects, AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjectsZero, AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspaces, AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspacesZero, AdminSimpleAccountsOrganizationsTransferOwnershipResponse, AdminSimpleAccountsProjectsCreate, AdminSimpleAccountsProjectsMembershipsCreate, AdminSimpleAccountsResponse, AdminSimpleAccountsUsersCreate, AdminSimpleAccountsUsersIdentitiesCreate, AdminSimpleAccountsUsersResetPassword, AdminSimpleAccountsWorkspacesCreate, AdminSimpleAccountsWorkspacesMembershipsCreate, AdminStructuredError, AdminSubscriptionCreate, AdminSubscriptionRead, AdminUserCreate, AdminUserIdentityCreate, AdminUserIdentityRead, AdminUserIdentityReadStatus, AdminUserRead, AdminWorkspaceCreate, AdminWorkspaceMembershipCreate, AdminWorkspaceMembershipRead, AdminWorkspaceRead, Analytics, AnalyticsResponse, Annotation, AnnotationCreate, AnnotationCreateLinks, AnnotationEdit, AnnotationEditLinks, AnnotationLinkResponse, AnnotationLinks, AnnotationQuery, AnnotationQueryLinks, AnnotationResponse, AnnotationsResponse, Application, ApplicationArtifactFlags, ApplicationArtifactQueryFlags, ApplicationCatalogPreset, ApplicationCatalogPresetResponse, ApplicationCatalogPresetsResponse, ApplicationCatalogTemplate, ApplicationCatalogTemplateResponse, ApplicationCatalogTemplatesResponse, ApplicationCatalogType, ApplicationCatalogTypesResponse, ApplicationCreate, ApplicationEdit, ApplicationFlags, ApplicationFork, ApplicationQuery, ApplicationResponse, ApplicationRevision, ApplicationRevisionCommit, ApplicationRevisionCreate, ApplicationRevisionDataInput, ApplicationRevisionDataInputHeadersValue, ApplicationRevisionDataInputRuntime, ApplicationRevisionDataOutput, ApplicationRevisionDataOutputHeadersValue, ApplicationRevisionDataOutputRuntime, ApplicationRevisionEdit, ApplicationRevisionFlags, ApplicationRevisionFork, ApplicationRevisionQuery, ApplicationRevisionQueryFlags, ApplicationRevisionResolveResponse, ApplicationRevisionResponse, ApplicationRevisionsLog, ApplicationRevisionsResponse, ApplicationVariant, ApplicationVariantCreate, ApplicationVariantEdit, ApplicationVariantFlags, ApplicationVariantFork, ApplicationVariantResponse, ApplicationVariantsResponse, ApplicationsResponse, BodyConfigsFetchVariantsConfigsFetchPost, Bucket, CollectStatusResponse, ComparisonOperator, Condition, ConditionOperator, ConditionOptions, ConditionValue, ConfigResponseModel, CustomModelSettingsDto, CustomProviderDto, CustomProviderKind, CustomProviderSettingsDto, DictOperator, DiscoverResponse, DiscoverResponseMethodsValue, EeSrcModelsApiOrganizationModelsOrganization, EntityRef, Environment, EnvironmentCreate, EnvironmentEdit, EnvironmentFlags, EnvironmentQueryFlags, EnvironmentResponse, EnvironmentRevision, EnvironmentRevisionCommit, EnvironmentRevisionCreate, EnvironmentRevisionData, EnvironmentRevisionDelta, EnvironmentRevisionEdit, EnvironmentRevisionResolveResponse, EnvironmentRevisionResponse, EnvironmentRevisionsLog, EnvironmentRevisionsResponse, EnvironmentVariant, EnvironmentVariantCreate, EnvironmentVariantEdit, EnvironmentVariantResponse, EnvironmentVariantsResponse, EnvironmentsResponse, ErrorPolicy, EvaluationMetrics, EvaluationMetricsCreate, EvaluationMetricsIdsResponse, EvaluationMetricsQuery, EvaluationMetricsQueryScenarioIds, EvaluationMetricsQueryTimestamps, EvaluationMetricsRefresh, EvaluationMetricsResponse, EvaluationQueue, EvaluationQueueCreate, EvaluationQueueData, EvaluationQueueEdit, EvaluationQueueFlags, EvaluationQueueIdResponse, EvaluationQueueIdsResponse, EvaluationQueueQuery, EvaluationQueueQueryFlags, EvaluationQueueResponse, EvaluationQueueScenariosQuery, EvaluationQueuesResponse, EvaluationResult, EvaluationResultCreate, EvaluationResultIdResponse, EvaluationResultIdsResponse, EvaluationResultQuery, EvaluationResultResponse, EvaluationResultsResponse, EvaluationRun, EvaluationRunCreate, EvaluationRunData, EvaluationRunDataConcurrency, EvaluationRunDataMapping, EvaluationRunDataMappingColumn, EvaluationRunDataMappingStep, EvaluationRunDataStep, EvaluationRunDataStepInput, EvaluationRunDataStepOrigin, EvaluationRunDataStepType, EvaluationRunEdit, EvaluationRunFlags, EvaluationRunIdResponse, EvaluationRunIdsRequest, EvaluationRunIdsResponse, EvaluationRunQuery, EvaluationRunQueryFlags, EvaluationRunResponse, EvaluationRunsResponse, EvaluationScenario, EvaluationScenarioCreate, EvaluationScenarioEdit, EvaluationScenarioIdResponse, EvaluationScenarioIdsResponse, EvaluationScenarioQuery, EvaluationScenarioResponse, EvaluationScenariosResponse, EvaluationStatus, Evaluator, EvaluatorArtifactFlags, EvaluatorArtifactQueryFlags, EvaluatorCatalogPreset, EvaluatorCatalogPresetResponse, EvaluatorCatalogPresetsResponse, EvaluatorCatalogTemplate, EvaluatorCatalogTemplateResponse, EvaluatorCatalogTemplatesResponse, EvaluatorCatalogType, EvaluatorCatalogTypesResponse, EvaluatorCreate, EvaluatorEdit, EvaluatorFlags, EvaluatorFork, EvaluatorQuery, EvaluatorResponse, EvaluatorRevision, EvaluatorRevisionCommit, EvaluatorRevisionCreate, EvaluatorRevisionDataInput, EvaluatorRevisionDataInputHeadersValue, EvaluatorRevisionDataInputRuntime, EvaluatorRevisionDataOutput, EvaluatorRevisionDataOutputHeadersValue, EvaluatorRevisionDataOutputRuntime, EvaluatorRevisionEdit, EvaluatorRevisionFlags, EvaluatorRevisionFork, EvaluatorRevisionQuery, EvaluatorRevisionQueryFlags, EvaluatorRevisionResolveResponse, EvaluatorRevisionResponse, EvaluatorRevisionsLog, EvaluatorRevisionsResponse, EvaluatorTemplate, EvaluatorTemplatesResponse, EvaluatorVariant, EvaluatorVariantCreate, EvaluatorVariantEdit, EvaluatorVariantFlags, EvaluatorVariantFork, EvaluatorVariantResponse, EvaluatorVariantsResponse, EvaluatorsResponse, Event, EventQuery, EventType, EventsQueryResponse, ExistenceOperator, FilteringInput, FilteringInputConditionsItem, FilteringOutput, FilteringOutputConditionsItem, Focus, Folder, FolderCreate, FolderEdit, FolderIdResponse, FolderKind, FolderQuery, FolderQueryKinds, FolderResponse, FoldersResponse, Format, Formatting, FullJsonInput, FullJsonOutput, Header, HttpValidationError, InviteRequest, Invocation, InvocationCreate, InvocationCreateLinks, InvocationEdit, InvocationEditLinks, InvocationLinkResponse, InvocationLinks, InvocationQuery, InvocationQueryLinks, InvocationResponse, InvocationsResponse, JsonSchemasInput, JsonSchemasOutput, LabelJsonInput, LabelJsonOutput, LegacyLifecycleDto, ListApiKeysResponse, ListOperator, ListOptions, LogicalOperator, MetricSpec, MetricType, MetricsBucket, NumericOperator, OTelEventInput, OTelEventInputTimestamp, OTelEventOutput, OTelEventOutputTimestamp, OTelHashInput, OTelHashOutput, OTelLinkInput, OTelLinkOutput, OTelLinksResponse, OTelReferenceInput, OTelReferenceOutput, OTelSpanKind, OTelStatusCode, OTelTracingRequest, OTelTracingResponse, OldAnalyticsResponse, OrganizationDetails, OrganizationDomainResponse, OrganizationProviderResponse, OrganizationUpdate, OssSrcModelsApiOrganizationModelsOrganization, Permission, ProjectsResponse, QueriesResponse, Query, QueryCreate, QueryEdit, QueryFlags, QueryQueryFlags, QueryResponse, QueryRevision, QueryRevisionCommit, QueryRevisionCreate, QueryRevisionDataInput, QueryRevisionDataOutput, QueryRevisionEdit, QueryRevisionQuery, QueryRevisionResponse, QueryRevisionsLog, QueryRevisionsResponse, QueryVariant, QueryVariantCreate, QueryVariantEdit, QueryVariantQuery, QueryVariantResponse, QueryVariantsResponse, Reference, ReferenceRequestModelInput, ReferenceRequestModelOutput, RequestType, ResolutionInfo, RetrievalInfo, RevisionFork, SecretDto, SecretDtoData, SecretKind, SecretResponseDto, SecretResponseDtoData, SessionIdsResponse, SimpleApplication, SimpleApplicationCreate, SimpleApplicationDataInput, SimpleApplicationDataInputHeadersValue, SimpleApplicationDataInputRuntime, SimpleApplicationDataOutput, SimpleApplicationDataOutputHeadersValue, SimpleApplicationDataOutputRuntime, SimpleApplicationEdit, SimpleApplicationFlags, SimpleApplicationQuery, SimpleApplicationQueryFlags, SimpleApplicationResponse, SimpleApplicationsResponse, SimpleEnvironment, SimpleEnvironmentCreate, SimpleEnvironmentEdit, SimpleEnvironmentQuery, SimpleEnvironmentResponse, SimpleEnvironmentsResponse, SimpleEvaluation, SimpleEvaluationCreate, SimpleEvaluationData, SimpleEvaluationDataApplicationSteps, SimpleEvaluationDataApplicationStepsOneValue, SimpleEvaluationDataEvaluatorSteps, SimpleEvaluationDataEvaluatorStepsOneValue, SimpleEvaluationDataQuerySteps, SimpleEvaluationDataQueryStepsOneValue, SimpleEvaluationDataTestsetSteps, SimpleEvaluationDataTestsetStepsOneValue, SimpleEvaluationEdit, SimpleEvaluationIdResponse, SimpleEvaluationQuery, SimpleEvaluationResponse, SimpleEvaluationsResponse, SimpleEvaluator, SimpleEvaluatorCreate, SimpleEvaluatorDataInput, SimpleEvaluatorDataInputHeadersValue, SimpleEvaluatorDataInputRuntime, SimpleEvaluatorDataOutput, SimpleEvaluatorDataOutputHeadersValue, SimpleEvaluatorDataOutputRuntime, SimpleEvaluatorEdit, SimpleEvaluatorFlags, SimpleEvaluatorQuery, SimpleEvaluatorQueryFlags, SimpleEvaluatorResponse, SimpleEvaluatorsResponse, SimpleQueriesResponse, SimpleQuery, SimpleQueryCreate, SimpleQueryEdit, SimpleQueryQuery, SimpleQueryResponse, SimpleQueue, SimpleQueueCreate, SimpleQueueData, SimpleQueueDataEvaluators, SimpleQueueDataEvaluatorsOneValue, SimpleQueueIdResponse, SimpleQueueKind, SimpleQueueQuery, SimpleQueueResponse, SimpleQueueScenariosQuery, SimpleQueueScenariosResponse, SimpleQueueSettings, SimpleQueuesResponse, SimpleTestset, SimpleTestsetCreate, SimpleTestsetEdit, SimpleTestsetQuery, SimpleTestsetResponse, SimpleTestsetsResponse, SimpleTrace, SimpleTraceChannel, SimpleTraceCreate, SimpleTraceCreateLinks, SimpleTraceEdit, SimpleTraceEditLinks, SimpleTraceKind, SimpleTraceLinkResponse, SimpleTraceLinks, SimpleTraceOrigin, SimpleTraceQuery, SimpleTraceQueryLinks, SimpleTraceReferences, SimpleTraceResponse, SimpleTracesResponse, SimpleWorkflow, SimpleWorkflowCreate, SimpleWorkflowDataInput, SimpleWorkflowDataInputHeadersValue, SimpleWorkflowDataInputRuntime, SimpleWorkflowDataOutput, SimpleWorkflowDataOutputHeadersValue, SimpleWorkflowDataOutputRuntime, SimpleWorkflowEdit, SimpleWorkflowFlags, SimpleWorkflowQuery, SimpleWorkflowQueryFlags, SimpleWorkflowResponse, SimpleWorkflowsResponse, SpanInput, SpanInputEndTime, SpanInputStartTime, SpanOutput, SpanOutputEndTime, SpanOutputStartTime, SpanResponse, SpanType, SpansNodeInput, SpansNodeInputEndTime, SpansNodeInputSpansValue, SpansNodeInputStartTime, SpansNodeOutput, SpansNodeOutputEndTime, SpansNodeOutputSpansValue, SpansNodeOutputStartTime, SpansResponse, SpansTreeInput, SpansTreeInputSpansValue, SpansTreeOutput, SpansTreeOutputSpansValue, SsoProviderDto, SsoProviderInfo, SsoProviderSettingsDto, SsoProviders, StandardProviderDto, StandardProviderKind, StandardProviderSettingsDto, Status, StringOperator, TestcaseInput, TestcaseOutput, TestcaseResponse, TestcasesResponse, Testset, TestsetCreate, TestsetEdit, TestsetFlags, TestsetQuery, TestsetResponse, TestsetRevision, TestsetRevisionCommit, TestsetRevisionCreate, TestsetRevisionDataInput, TestsetRevisionDataOutput, TestsetRevisionDelta, TestsetRevisionDeltaColumns, TestsetRevisionDeltaRows, TestsetRevisionEdit, TestsetRevisionQuery, TestsetRevisionResponse, TestsetRevisionsLog, TestsetRevisionsResponse, TestsetVariant, TestsetVariantCreate, TestsetVariantEdit, TestsetVariantQuery, TestsetVariantResponse, TestsetVariantsResponse, TestsetsResponse, TextOptions, ToolAuthScheme, ToolCallData, ToolCallFunction, ToolCallResponse, ToolCatalogAction, ToolCatalogActionDetails, ToolCatalogActionResponse, ToolCatalogActionResponseAction, ToolCatalogActionsResponse, ToolCatalogActionsResponseActionsItem, ToolCatalogIntegration, ToolCatalogIntegrationDetails, ToolCatalogIntegrationResponse, ToolCatalogIntegrationResponseIntegration, ToolCatalogIntegrationsResponse, ToolCatalogIntegrationsResponseIntegrationsItem, ToolCatalogProvider, ToolCatalogProviderDetails, ToolCatalogProviderResponse, ToolCatalogProviderResponseProvider, ToolCatalogProvidersResponse, ToolCatalogProvidersResponseProvidersItem, ToolConnection, ToolConnectionCreate, ToolConnectionCreateData, ToolConnectionResponse, ToolConnectionStatus, ToolConnectionsResponse, ToolProviderKind, ToolResult, ToolResultData, TraceIdResponse, TraceIdsResponse, TraceInput, TraceInputSpansValue, TraceOutput, TraceOutputSpansValue, TraceRequest, TraceResponse, TraceType, TracesRequest, TracesResponse, TracingQuery, UserIdsResponse, ValidationError, ValidationErrorLocItem, VariantFork, WebhookDeliveriesResponse, WebhookDelivery, WebhookDeliveryCreate, WebhookDeliveryData, WebhookDeliveryQuery, WebhookDeliveryResponse, WebhookDeliveryResponseInfo, WebhookEventType, WebhookProviderDto, WebhookProviderSettingsDto, WebhookSubscription, WebhookSubscriptionCreate, WebhookSubscriptionData, WebhookSubscriptionDataAuthMode, WebhookSubscriptionEdit, WebhookSubscriptionQuery, WebhookSubscriptionResponse, WebhookSubscriptionsResponse, Windowing, WindowingOrder, Workflow, WorkflowArtifactFlags, WorkflowCatalogFlags, WorkflowCatalogPreset, WorkflowCatalogPresetResponse, WorkflowCatalogPresetsResponse, WorkflowCatalogTemplate, WorkflowCatalogTemplateResponse, WorkflowCatalogTemplatesResponse, WorkflowCatalogType, WorkflowCatalogTypeResponse, WorkflowCatalogTypesResponse, WorkflowCreate, WorkflowEdit, WorkflowFlags, WorkflowFork, WorkflowResponse, WorkflowRevisionCommit, WorkflowRevisionCreate, WorkflowRevisionDataInput, WorkflowRevisionDataInputHeadersValue, WorkflowRevisionDataInputRuntime, WorkflowRevisionDataOutput, WorkflowRevisionDataOutputHeadersValue, WorkflowRevisionDataOutputRuntime, WorkflowRevisionEdit, WorkflowRevisionFlags, WorkflowRevisionFork, WorkflowRevisionInput, WorkflowRevisionOutput, WorkflowRevisionResolveResponse, WorkflowRevisionResponse, WorkflowRevisionsLog, WorkflowRevisionsResponse, WorkflowVariant, WorkflowVariantCreate, WorkflowVariantEdit, WorkflowVariantFlags, WorkflowVariantFork, WorkflowVariantResponse, WorkflowVariantsResponse, WorkflowsResponse, Workspace, WorkspaceMemberResponse, WorkspacePermission, WorkspaceResponse from .errors import UnprocessableEntityError from . import access, annotations, applications, billing, environments, evaluations, evaluators, events, folders, invocations, keys, legacy, organizations, projects, queries, secrets, status, testcases, testsets, tools, traces, users, webhooks, workflows, workspaces from .applications import QueryApplicationVariantsRequestOrder @@ -19,7 +19,7 @@ from .traces import QuerySpansAnalyticsRequestNewest, QuerySpansAnalyticsRequestOldest from .webhooks import WebhookSubscriptionTestRequestSubscription from .workflows import QueryWorkflowRevisionsRequestOrder, QueryWorkflowVariantsRequestOrder, QueryWorkflowsRequestOrder -_dynamic_imports: typing.Dict[str, str] = {"AdminAccountCreateOptions": ".types", "AdminAccountRead": ".types", "AdminAccountsCreate": ".types", "AdminAccountsDelete": ".types", "AdminAccountsDeleteTarget": ".types", "AdminAccountsResponse": ".types", "AdminApiKeyCreate": ".types", "AdminApiKeyResponse": ".types", "AdminDeleteResponse": ".types", "AdminDeletedEntities": ".types", "AdminDeletedEntity": ".types", "AdminOrganizationCreate": ".types", "AdminOrganizationMembershipCreate": ".types", "AdminOrganizationMembershipRead": ".types", "AdminOrganizationRead": ".types", "AdminProjectCreate": ".types", "AdminProjectMembershipCreate": ".types", "AdminProjectMembershipRead": ".types", "AdminProjectRead": ".types", "AdminSimpleAccountCreate": ".types", "AdminSimpleAccountDeleteEntry": ".types", "AdminSimpleAccountRead": ".types", "AdminSimpleAccountsApiKeysCreate": ".types", "AdminSimpleAccountsCreate": ".types", "AdminSimpleAccountsDelete": ".types", "AdminSimpleAccountsOrganizationsCreate": ".types", "AdminSimpleAccountsOrganizationsMembershipsCreate": ".types", "AdminSimpleAccountsOrganizationsTransferOwnership": ".types", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjects": ".types", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjectsZero": ".types", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspaces": ".types", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspacesZero": ".types", "AdminSimpleAccountsOrganizationsTransferOwnershipResponse": ".types", "AdminSimpleAccountsProjectsCreate": ".types", "AdminSimpleAccountsProjectsMembershipsCreate": ".types", "AdminSimpleAccountsResponse": ".types", "AdminSimpleAccountsUsersCreate": ".types", "AdminSimpleAccountsUsersIdentitiesCreate": ".types", "AdminSimpleAccountsUsersResetPassword": ".types", "AdminSimpleAccountsWorkspacesCreate": ".types", "AdminSimpleAccountsWorkspacesMembershipsCreate": ".types", "AdminStructuredError": ".types", "AdminSubscriptionCreate": ".types", "AdminSubscriptionRead": ".types", "AdminUserCreate": ".types", "AdminUserIdentityCreate": ".types", "AdminUserIdentityRead": ".types", "AdminUserIdentityReadStatus": ".types", "AdminUserRead": ".types", "AdminWorkspaceCreate": ".types", "AdminWorkspaceMembershipCreate": ".types", "AdminWorkspaceMembershipRead": ".types", "AdminWorkspaceRead": ".types", "AgentaApi": ".client", "AgentaApiEnvironment": ".environment", "Analytics": ".types", "AnalyticsResponse": ".types", "Annotation": ".types", "AnnotationCreate": ".types", "AnnotationCreateLinks": ".types", "AnnotationEdit": ".types", "AnnotationEditLinks": ".types", "AnnotationLinkResponse": ".types", "AnnotationLinks": ".types", "AnnotationQuery": ".types", "AnnotationQueryLinks": ".types", "AnnotationResponse": ".types", "AnnotationsResponse": ".types", "Application": ".types", "ApplicationArtifactFlags": ".types", "ApplicationArtifactQueryFlags": ".types", "ApplicationCatalogPreset": ".types", "ApplicationCatalogPresetResponse": ".types", "ApplicationCatalogPresetsResponse": ".types", "ApplicationCatalogTemplate": ".types", "ApplicationCatalogTemplateResponse": ".types", "ApplicationCatalogTemplatesResponse": ".types", "ApplicationCatalogType": ".types", "ApplicationCatalogTypesResponse": ".types", "ApplicationCreate": ".types", "ApplicationEdit": ".types", "ApplicationFlags": ".types", "ApplicationFork": ".types", "ApplicationQuery": ".types", "ApplicationResponse": ".types", "ApplicationRevision": ".types", "ApplicationRevisionCommit": ".types", "ApplicationRevisionCreate": ".types", "ApplicationRevisionDataInput": ".types", "ApplicationRevisionDataInputHeadersValue": ".types", "ApplicationRevisionDataInputRuntime": ".types", "ApplicationRevisionDataOutput": ".types", "ApplicationRevisionDataOutputHeadersValue": ".types", "ApplicationRevisionDataOutputRuntime": ".types", "ApplicationRevisionEdit": ".types", "ApplicationRevisionFlags": ".types", "ApplicationRevisionFork": ".types", "ApplicationRevisionQuery": ".types", "ApplicationRevisionQueryFlags": ".types", "ApplicationRevisionResolveResponse": ".types", "ApplicationRevisionResponse": ".types", "ApplicationRevisionsLog": ".types", "ApplicationRevisionsResponse": ".types", "ApplicationVariant": ".types", "ApplicationVariantCreate": ".types", "ApplicationVariantEdit": ".types", "ApplicationVariantFlags": ".types", "ApplicationVariantFork": ".types", "ApplicationVariantResponse": ".types", "ApplicationVariantsResponse": ".types", "ApplicationsResponse": ".types", "AsyncAgentaApi": ".client", "BodyConfigsFetchVariantsConfigsFetchPost": ".types", "Bucket": ".types", "CollectStatusResponse": ".types", "ComparisonOperator": ".types", "Condition": ".types", "ConditionOperator": ".types", "ConditionOptions": ".types", "ConditionValue": ".types", "ConfigResponseModel": ".types", "CreateSimpleTestsetFromFileRequestFileType": ".testsets", "CreateTestsetRevisionFromFileRequestFileType": ".testsets", "CustomModelSettingsDto": ".types", "CustomProviderDto": ".types", "CustomProviderKind": ".types", "CustomProviderSettingsDto": ".types", "DictOperator": ".types", "DiscoverResponse": ".types", "DiscoverResponseMethodsValue": ".types", "EditSimpleTestsetFromFileRequestFileType": ".testsets", "EeSrcModelsApiOrganizationModelsOrganization": ".types", "EntityRef": ".types", "Environment": ".types", "EnvironmentCreate": ".types", "EnvironmentEdit": ".types", "EnvironmentFlags": ".types", "EnvironmentQueryFlags": ".types", "EnvironmentResponse": ".types", "EnvironmentRevision": ".types", "EnvironmentRevisionCommit": ".types", "EnvironmentRevisionCreate": ".types", "EnvironmentRevisionData": ".types", "EnvironmentRevisionDelta": ".types", "EnvironmentRevisionEdit": ".types", "EnvironmentRevisionResolveResponse": ".types", "EnvironmentRevisionResponse": ".types", "EnvironmentRevisionsLog": ".types", "EnvironmentRevisionsResponse": ".types", "EnvironmentVariant": ".types", "EnvironmentVariantCreate": ".types", "EnvironmentVariantEdit": ".types", "EnvironmentVariantResponse": ".types", "EnvironmentVariantsResponse": ".types", "EnvironmentsResponse": ".types", "ErrorPolicy": ".types", "EvaluationMetrics": ".types", "EvaluationMetricsCreate": ".types", "EvaluationMetricsEdit": ".types", "EvaluationMetricsIdsResponse": ".types", "EvaluationMetricsQuery": ".types", "EvaluationMetricsQueryScenarioIds": ".types", "EvaluationMetricsQueryTimestamps": ".types", "EvaluationMetricsRefresh": ".types", "EvaluationMetricsResponse": ".types", "EvaluationQueue": ".types", "EvaluationQueueCreate": ".types", "EvaluationQueueData": ".types", "EvaluationQueueEdit": ".types", "EvaluationQueueFlags": ".types", "EvaluationQueueIdResponse": ".types", "EvaluationQueueIdsResponse": ".types", "EvaluationQueueQuery": ".types", "EvaluationQueueQueryFlags": ".types", "EvaluationQueueResponse": ".types", "EvaluationQueueScenariosQuery": ".types", "EvaluationQueuesResponse": ".types", "EvaluationResult": ".types", "EvaluationResultCreate": ".types", "EvaluationResultEdit": ".types", "EvaluationResultIdResponse": ".types", "EvaluationResultIdsResponse": ".types", "EvaluationResultQuery": ".types", "EvaluationResultResponse": ".types", "EvaluationResultsResponse": ".types", "EvaluationRun": ".types", "EvaluationRunCreate": ".types", "EvaluationRunData": ".types", "EvaluationRunDataMapping": ".types", "EvaluationRunDataMappingColumn": ".types", "EvaluationRunDataMappingStep": ".types", "EvaluationRunDataStep": ".types", "EvaluationRunDataStepInput": ".types", "EvaluationRunDataStepOrigin": ".types", "EvaluationRunDataStepType": ".types", "EvaluationRunEdit": ".types", "EvaluationRunFlags": ".types", "EvaluationRunIdResponse": ".types", "EvaluationRunIdsRequest": ".types", "EvaluationRunIdsResponse": ".types", "EvaluationRunQuery": ".types", "EvaluationRunQueryFlags": ".types", "EvaluationRunResponse": ".types", "EvaluationRunsResponse": ".types", "EvaluationScenario": ".types", "EvaluationScenarioCreate": ".types", "EvaluationScenarioEdit": ".types", "EvaluationScenarioIdResponse": ".types", "EvaluationScenarioIdsResponse": ".types", "EvaluationScenarioQuery": ".types", "EvaluationScenarioResponse": ".types", "EvaluationScenariosResponse": ".types", "EvaluationStatus": ".types", "Evaluator": ".types", "EvaluatorArtifactFlags": ".types", "EvaluatorArtifactQueryFlags": ".types", "EvaluatorCatalogPreset": ".types", "EvaluatorCatalogPresetResponse": ".types", "EvaluatorCatalogPresetsResponse": ".types", "EvaluatorCatalogTemplate": ".types", "EvaluatorCatalogTemplateResponse": ".types", "EvaluatorCatalogTemplatesResponse": ".types", "EvaluatorCatalogType": ".types", "EvaluatorCatalogTypesResponse": ".types", "EvaluatorCreate": ".types", "EvaluatorEdit": ".types", "EvaluatorFlags": ".types", "EvaluatorFork": ".types", "EvaluatorQuery": ".types", "EvaluatorResponse": ".types", "EvaluatorRevision": ".types", "EvaluatorRevisionCommit": ".types", "EvaluatorRevisionCreate": ".types", "EvaluatorRevisionDataInput": ".types", "EvaluatorRevisionDataInputHeadersValue": ".types", "EvaluatorRevisionDataInputRuntime": ".types", "EvaluatorRevisionDataOutput": ".types", "EvaluatorRevisionDataOutputHeadersValue": ".types", "EvaluatorRevisionDataOutputRuntime": ".types", "EvaluatorRevisionEdit": ".types", "EvaluatorRevisionFlags": ".types", "EvaluatorRevisionFork": ".types", "EvaluatorRevisionQuery": ".types", "EvaluatorRevisionQueryFlags": ".types", "EvaluatorRevisionResolveResponse": ".types", "EvaluatorRevisionResponse": ".types", "EvaluatorRevisionsLog": ".types", "EvaluatorRevisionsResponse": ".types", "EvaluatorTemplate": ".types", "EvaluatorTemplatesResponse": ".types", "EvaluatorVariant": ".types", "EvaluatorVariantCreate": ".types", "EvaluatorVariantEdit": ".types", "EvaluatorVariantFlags": ".types", "EvaluatorVariantFork": ".types", "EvaluatorVariantResponse": ".types", "EvaluatorVariantsResponse": ".types", "EvaluatorsResponse": ".types", "Event": ".types", "EventQuery": ".types", "EventType": ".types", "EventsQueryResponse": ".types", "ExistenceOperator": ".types", "FetchLegacyAnalyticsRequestNewest": ".legacy", "FetchLegacyAnalyticsRequestOldest": ".legacy", "FetchSimpleTestsetToFileRequestFileType": ".testsets", "FetchTestsetRevisionToFileRequestFileType": ".testsets", "FilteringInput": ".types", "FilteringInputConditionsItem": ".types", "FilteringOutput": ".types", "FilteringOutputConditionsItem": ".types", "Focus": ".types", "Folder": ".types", "FolderCreate": ".types", "FolderEdit": ".types", "FolderIdResponse": ".types", "FolderKind": ".types", "FolderQuery": ".types", "FolderQueryKinds": ".types", "FolderResponse": ".types", "FoldersResponse": ".types", "Format": ".types", "Formatting": ".types", typing.Any: ".types", typing.Any: ".types", "Header": ".types", "HttpValidationError": ".types", "InviteRequest": ".types", "Invocation": ".types", "InvocationCreate": ".types", "InvocationCreateLinks": ".types", "InvocationEdit": ".types", "InvocationEditLinks": ".types", "InvocationLinkResponse": ".types", "InvocationLinks": ".types", "InvocationQuery": ".types", "InvocationQueryLinks": ".types", "InvocationResponse": ".types", "InvocationsResponse": ".types", "JsonSchemasInput": ".types", "JsonSchemasOutput": ".types", typing.Any: ".types", typing.Any: ".types", "LegacyLifecycleDto": ".types", "ListApiKeysResponse": ".types", "ListOperator": ".types", "ListOptions": ".types", "LogicalOperator": ".types", "MetricSpec": ".types", "MetricType": ".types", "MetricsBucket": ".types", "NumericOperator": ".types", "OTelEventInput": ".types", "OTelEventInputTimestamp": ".types", "OTelEventOutput": ".types", "OTelEventOutputTimestamp": ".types", "OTelHashInput": ".types", "OTelHashOutput": ".types", "OTelLinkInput": ".types", "OTelLinkOutput": ".types", "OTelLinksResponse": ".types", "OTelReferenceInput": ".types", "OTelReferenceOutput": ".types", "OTelSpanKind": ".types", "OTelStatusCode": ".types", "OTelTracingRequest": ".types", "OTelTracingResponse": ".types", "OldAnalyticsResponse": ".types", "OrganizationDetails": ".types", "OrganizationDomainResponse": ".types", "OrganizationProviderResponse": ".types", "OrganizationUpdate": ".types", "OssSrcModelsApiOrganizationModelsOrganization": ".types", "Permission": ".types", "ProjectsResponse": ".types", "QueriesResponse": ".types", "Query": ".types", "QueryApplicationVariantsRequestOrder": ".applications", "QueryCreate": ".types", "QueryEdit": ".types", "QueryEnvironmentRevisionsRequestOrder": ".environments", "QueryEnvironmentVariantsRequestOrder": ".environments", "QueryEnvironmentsRequestOrder": ".environments", "QueryEvaluatorVariantsRequestOrder": ".evaluators", "QueryFlags": ".types", "QueryQueriesRequestOrder": ".queries", "QueryQueryFlags": ".types", "QueryResponse": ".types", "QueryRevision": ".types", "QueryRevisionCommit": ".types", "QueryRevisionCreate": ".types", "QueryRevisionDataInput": ".types", "QueryRevisionDataOutput": ".types", "QueryRevisionEdit": ".types", "QueryRevisionQuery": ".types", "QueryRevisionResponse": ".types", "QueryRevisionsLog": ".types", "QueryRevisionsResponse": ".types", "QuerySpansAnalyticsRequestNewest": ".traces", "QuerySpansAnalyticsRequestOldest": ".traces", "QueryVariant": ".types", "QueryVariantCreate": ".types", "QueryVariantEdit": ".types", "QueryVariantQuery": ".types", "QueryVariantResponse": ".types", "QueryVariantsResponse": ".types", "QueryWorkflowRevisionsRequestOrder": ".workflows", "QueryWorkflowVariantsRequestOrder": ".workflows", "QueryWorkflowsRequestOrder": ".workflows", "Reference": ".types", "ReferenceRequestModelInput": ".types", "ReferenceRequestModelOutput": ".types", "RequestType": ".types", "ResolutionInfo": ".types", "RetrievalInfo": ".types", "RevisionFork": ".types", "SecretDto": ".types", "SecretDtoData": ".types", "SecretKind": ".types", "SecretResponseDto": ".types", "SecretResponseDtoData": ".types", "SessionIdsResponse": ".types", "SimpleApplication": ".types", "SimpleApplicationCreate": ".types", "SimpleApplicationDataInput": ".types", "SimpleApplicationDataInputHeadersValue": ".types", "SimpleApplicationDataInputRuntime": ".types", "SimpleApplicationDataOutput": ".types", "SimpleApplicationDataOutputHeadersValue": ".types", "SimpleApplicationDataOutputRuntime": ".types", "SimpleApplicationEdit": ".types", "SimpleApplicationFlags": ".types", "SimpleApplicationQuery": ".types", "SimpleApplicationQueryFlags": ".types", "SimpleApplicationResponse": ".types", "SimpleApplicationsResponse": ".types", "SimpleEnvironment": ".types", "SimpleEnvironmentCreate": ".types", "SimpleEnvironmentEdit": ".types", "SimpleEnvironmentQuery": ".types", "SimpleEnvironmentResponse": ".types", "SimpleEnvironmentsResponse": ".types", "SimpleEvaluation": ".types", "SimpleEvaluationCreate": ".types", "SimpleEvaluationData": ".types", "SimpleEvaluationDataApplicationSteps": ".types", "SimpleEvaluationDataApplicationStepsOneValue": ".types", "SimpleEvaluationDataEvaluatorSteps": ".types", "SimpleEvaluationDataEvaluatorStepsOneValue": ".types", "SimpleEvaluationDataQuerySteps": ".types", "SimpleEvaluationDataQueryStepsOneValue": ".types", "SimpleEvaluationDataTestsetSteps": ".types", "SimpleEvaluationDataTestsetStepsOneValue": ".types", "SimpleEvaluationEdit": ".types", "SimpleEvaluationIdResponse": ".types", "SimpleEvaluationQuery": ".types", "SimpleEvaluationResponse": ".types", "SimpleEvaluationsResponse": ".types", "SimpleEvaluator": ".types", "SimpleEvaluatorCreate": ".types", "SimpleEvaluatorDataInput": ".types", "SimpleEvaluatorDataInputHeadersValue": ".types", "SimpleEvaluatorDataInputRuntime": ".types", "SimpleEvaluatorDataOutput": ".types", "SimpleEvaluatorDataOutputHeadersValue": ".types", "SimpleEvaluatorDataOutputRuntime": ".types", "SimpleEvaluatorEdit": ".types", "SimpleEvaluatorFlags": ".types", "SimpleEvaluatorQuery": ".types", "SimpleEvaluatorQueryFlags": ".types", "SimpleEvaluatorResponse": ".types", "SimpleEvaluatorsResponse": ".types", "SimpleQueriesResponse": ".types", "SimpleQuery": ".types", "SimpleQueryCreate": ".types", "SimpleQueryEdit": ".types", "SimpleQueryQuery": ".types", "SimpleQueryResponse": ".types", "SimpleQueue": ".types", "SimpleQueueCreate": ".types", "SimpleQueueData": ".types", "SimpleQueueDataEvaluators": ".types", "SimpleQueueDataEvaluatorsOneValue": ".types", "SimpleQueueIdResponse": ".types", "SimpleQueueKind": ".types", "SimpleQueueQuery": ".types", "SimpleQueueResponse": ".types", "SimpleQueueScenariosQuery": ".types", "SimpleQueueScenariosResponse": ".types", "SimpleQueueSettings": ".types", "SimpleQueuesResponse": ".types", "SimpleTestset": ".types", "SimpleTestsetCreate": ".types", "SimpleTestsetEdit": ".types", "SimpleTestsetQuery": ".types", "SimpleTestsetResponse": ".types", "SimpleTestsetsResponse": ".types", "SimpleTrace": ".types", "SimpleTraceChannel": ".types", "SimpleTraceCreate": ".types", "SimpleTraceCreateLinks": ".types", "SimpleTraceEdit": ".types", "SimpleTraceEditLinks": ".types", "SimpleTraceKind": ".types", "SimpleTraceLinkResponse": ".types", "SimpleTraceLinks": ".types", "SimpleTraceOrigin": ".types", "SimpleTraceQuery": ".types", "SimpleTraceQueryLinks": ".types", "SimpleTraceReferences": ".types", "SimpleTraceResponse": ".types", "SimpleTracesResponse": ".types", "SimpleWorkflow": ".types", "SimpleWorkflowCreate": ".types", "SimpleWorkflowDataInput": ".types", "SimpleWorkflowDataInputHeadersValue": ".types", "SimpleWorkflowDataInputRuntime": ".types", "SimpleWorkflowDataOutput": ".types", "SimpleWorkflowDataOutputHeadersValue": ".types", "SimpleWorkflowDataOutputRuntime": ".types", "SimpleWorkflowEdit": ".types", "SimpleWorkflowFlags": ".types", "SimpleWorkflowQuery": ".types", "SimpleWorkflowQueryFlags": ".types", "SimpleWorkflowResponse": ".types", "SimpleWorkflowsResponse": ".types", "SpanInput": ".types", "SpanInputEndTime": ".types", "SpanInputStartTime": ".types", "SpanOutput": ".types", "SpanOutputEndTime": ".types", "SpanOutputStartTime": ".types", "SpanResponse": ".types", "SpanType": ".types", "SpansNodeInput": ".types", "SpansNodeInputEndTime": ".types", "SpansNodeInputSpansValue": ".types", "SpansNodeInputStartTime": ".types", "SpansNodeOutput": ".types", "SpansNodeOutputEndTime": ".types", "SpansNodeOutputSpansValue": ".types", "SpansNodeOutputStartTime": ".types", "SpansResponse": ".types", "SpansTreeInput": ".types", "SpansTreeInputSpansValue": ".types", "SpansTreeOutput": ".types", "SpansTreeOutputSpansValue": ".types", "SsoProviderDto": ".types", "SsoProviderInfo": ".types", "SsoProviderSettingsDto": ".types", "SsoProviders": ".types", "StandardProviderDto": ".types", "StandardProviderKind": ".types", "StandardProviderSettingsDto": ".types", "Status": ".types", "StringOperator": ".types", "TestcaseInput": ".types", "TestcaseOutput": ".types", "TestcaseResponse": ".types", "TestcasesResponse": ".types", "Testset": ".types", "TestsetCreate": ".types", "TestsetEdit": ".types", "TestsetFlags": ".types", "TestsetQuery": ".types", "TestsetResponse": ".types", "TestsetRevision": ".types", "TestsetRevisionCommit": ".types", "TestsetRevisionCreate": ".types", "TestsetRevisionDataInput": ".types", "TestsetRevisionDataOutput": ".types", "TestsetRevisionDelta": ".types", "TestsetRevisionDeltaColumns": ".types", "TestsetRevisionDeltaRows": ".types", "TestsetRevisionEdit": ".types", "TestsetRevisionQuery": ".types", "TestsetRevisionResponse": ".types", "TestsetRevisionsLog": ".types", "TestsetRevisionsResponse": ".types", "TestsetVariant": ".types", "TestsetVariantCreate": ".types", "TestsetVariantEdit": ".types", "TestsetVariantQuery": ".types", "TestsetVariantResponse": ".types", "TestsetVariantsResponse": ".types", "TestsetsResponse": ".types", "TextOptions": ".types", "ToolAuthScheme": ".types", "ToolCallData": ".types", "ToolCallFunction": ".types", "ToolCallResponse": ".types", "ToolCatalogAction": ".types", "ToolCatalogActionDetails": ".types", "ToolCatalogActionResponse": ".types", "ToolCatalogActionResponseAction": ".types", "ToolCatalogActionsResponse": ".types", "ToolCatalogActionsResponseActionsItem": ".types", "ToolCatalogIntegration": ".types", "ToolCatalogIntegrationDetails": ".types", "ToolCatalogIntegrationResponse": ".types", "ToolCatalogIntegrationResponseIntegration": ".types", "ToolCatalogIntegrationsResponse": ".types", "ToolCatalogIntegrationsResponseIntegrationsItem": ".types", "ToolCatalogProvider": ".types", "ToolCatalogProviderDetails": ".types", "ToolCatalogProviderResponse": ".types", "ToolCatalogProviderResponseProvider": ".types", "ToolCatalogProvidersResponse": ".types", "ToolCatalogProvidersResponseProvidersItem": ".types", "ToolConnection": ".types", "ToolConnectionCreate": ".types", "ToolConnectionCreateData": ".types", "ToolConnectionResponse": ".types", "ToolConnectionStatus": ".types", "ToolConnectionsResponse": ".types", "ToolProviderKind": ".types", "ToolResult": ".types", "ToolResultData": ".types", "TraceIdResponse": ".types", "TraceIdsResponse": ".types", "TraceInput": ".types", "TraceInputSpansValue": ".types", "TraceOutput": ".types", "TraceOutputSpansValue": ".types", "TraceRequest": ".types", "TraceResponse": ".types", "TraceType": ".types", "TracesRequest": ".types", "TracesResponse": ".types", "TracingQuery": ".types", "UnprocessableEntityError": ".errors", "UserIdsResponse": ".types", "ValidationError": ".types", "ValidationErrorLocItem": ".types", "VariantFork": ".types", "WebhookDeliveriesResponse": ".types", "WebhookDelivery": ".types", "WebhookDeliveryCreate": ".types", "WebhookDeliveryData": ".types", "WebhookDeliveryQuery": ".types", "WebhookDeliveryResponse": ".types", "WebhookDeliveryResponseInfo": ".types", "WebhookEventType": ".types", "WebhookProviderDto": ".types", "WebhookProviderSettingsDto": ".types", "WebhookSubscription": ".types", "WebhookSubscriptionCreate": ".types", "WebhookSubscriptionData": ".types", "WebhookSubscriptionDataAuthMode": ".types", "WebhookSubscriptionEdit": ".types", "WebhookSubscriptionQuery": ".types", "WebhookSubscriptionResponse": ".types", "WebhookSubscriptionTestRequestSubscription": ".webhooks", "WebhookSubscriptionsResponse": ".types", "Windowing": ".types", "WindowingOrder": ".types", "Workflow": ".types", "WorkflowArtifactFlags": ".types", "WorkflowCatalogFlags": ".types", "WorkflowCatalogPreset": ".types", "WorkflowCatalogPresetResponse": ".types", "WorkflowCatalogPresetsResponse": ".types", "WorkflowCatalogTemplate": ".types", "WorkflowCatalogTemplateResponse": ".types", "WorkflowCatalogTemplatesResponse": ".types", "WorkflowCatalogType": ".types", "WorkflowCatalogTypeResponse": ".types", "WorkflowCatalogTypesResponse": ".types", "WorkflowCreate": ".types", "WorkflowEdit": ".types", "WorkflowFlags": ".types", "WorkflowFork": ".types", "WorkflowResponse": ".types", "WorkflowRevisionCommit": ".types", "WorkflowRevisionCreate": ".types", "WorkflowRevisionDataInput": ".types", "WorkflowRevisionDataInputHeadersValue": ".types", "WorkflowRevisionDataInputRuntime": ".types", "WorkflowRevisionDataOutput": ".types", "WorkflowRevisionDataOutputHeadersValue": ".types", "WorkflowRevisionDataOutputRuntime": ".types", "WorkflowRevisionEdit": ".types", "WorkflowRevisionFlags": ".types", "WorkflowRevisionFork": ".types", "WorkflowRevisionInput": ".types", "WorkflowRevisionOutput": ".types", "WorkflowRevisionResolveResponse": ".types", "WorkflowRevisionResponse": ".types", "WorkflowRevisionsLog": ".types", "WorkflowRevisionsResponse": ".types", "WorkflowVariant": ".types", "WorkflowVariantCreate": ".types", "WorkflowVariantEdit": ".types", "WorkflowVariantFlags": ".types", "WorkflowVariantFork": ".types", "WorkflowVariantResponse": ".types", "WorkflowVariantsResponse": ".types", "WorkflowsResponse": ".types", "Workspace": ".types", "WorkspaceMemberResponse": ".types", "WorkspacePermission": ".types", "WorkspaceResponse": ".types", "access": ".access", "annotations": ".annotations", "applications": ".applications", "billing": ".billing", "environments": ".environments", "evaluations": ".evaluations", "evaluators": ".evaluators", "events": ".events", "folders": ".folders", "invocations": ".invocations", "keys": ".keys", "legacy": ".legacy", "organizations": ".organizations", "projects": ".projects", "queries": ".queries", "secrets": ".secrets", "status": ".status", "testcases": ".testcases", "testsets": ".testsets", "tools": ".tools", "traces": ".traces", "users": ".users", "webhooks": ".webhooks", "workflows": ".workflows", "workspaces": ".workspaces"} +_dynamic_imports: typing.Dict[str, str] = {"AdminAccountCreateOptions": ".types", "AdminAccountRead": ".types", "AdminAccountsCreate": ".types", "AdminAccountsDelete": ".types", "AdminAccountsDeleteTarget": ".types", "AdminAccountsResponse": ".types", "AdminApiKeyCreate": ".types", "AdminApiKeyResponse": ".types", "AdminDeleteResponse": ".types", "AdminDeletedEntities": ".types", "AdminDeletedEntity": ".types", "AdminOrganizationCreate": ".types", "AdminOrganizationMembershipCreate": ".types", "AdminOrganizationMembershipRead": ".types", "AdminOrganizationRead": ".types", "AdminProjectCreate": ".types", "AdminProjectMembershipCreate": ".types", "AdminProjectMembershipRead": ".types", "AdminProjectRead": ".types", "AdminSimpleAccountCreate": ".types", "AdminSimpleAccountDeleteEntry": ".types", "AdminSimpleAccountRead": ".types", "AdminSimpleAccountsApiKeysCreate": ".types", "AdminSimpleAccountsCreate": ".types", "AdminSimpleAccountsDelete": ".types", "AdminSimpleAccountsOrganizationsCreate": ".types", "AdminSimpleAccountsOrganizationsMembershipsCreate": ".types", "AdminSimpleAccountsOrganizationsTransferOwnership": ".types", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjects": ".types", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjectsZero": ".types", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspaces": ".types", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspacesZero": ".types", "AdminSimpleAccountsOrganizationsTransferOwnershipResponse": ".types", "AdminSimpleAccountsProjectsCreate": ".types", "AdminSimpleAccountsProjectsMembershipsCreate": ".types", "AdminSimpleAccountsResponse": ".types", "AdminSimpleAccountsUsersCreate": ".types", "AdminSimpleAccountsUsersIdentitiesCreate": ".types", "AdminSimpleAccountsUsersResetPassword": ".types", "AdminSimpleAccountsWorkspacesCreate": ".types", "AdminSimpleAccountsWorkspacesMembershipsCreate": ".types", "AdminStructuredError": ".types", "AdminSubscriptionCreate": ".types", "AdminSubscriptionRead": ".types", "AdminUserCreate": ".types", "AdminUserIdentityCreate": ".types", "AdminUserIdentityRead": ".types", "AdminUserIdentityReadStatus": ".types", "AdminUserRead": ".types", "AdminWorkspaceCreate": ".types", "AdminWorkspaceMembershipCreate": ".types", "AdminWorkspaceMembershipRead": ".types", "AdminWorkspaceRead": ".types", "AgentaApi": ".client", "AgentaApiEnvironment": ".environment", "Analytics": ".types", "AnalyticsResponse": ".types", "Annotation": ".types", "AnnotationCreate": ".types", "AnnotationCreateLinks": ".types", "AnnotationEdit": ".types", "AnnotationEditLinks": ".types", "AnnotationLinkResponse": ".types", "AnnotationLinks": ".types", "AnnotationQuery": ".types", "AnnotationQueryLinks": ".types", "AnnotationResponse": ".types", "AnnotationsResponse": ".types", "Application": ".types", "ApplicationArtifactFlags": ".types", "ApplicationArtifactQueryFlags": ".types", "ApplicationCatalogPreset": ".types", "ApplicationCatalogPresetResponse": ".types", "ApplicationCatalogPresetsResponse": ".types", "ApplicationCatalogTemplate": ".types", "ApplicationCatalogTemplateResponse": ".types", "ApplicationCatalogTemplatesResponse": ".types", "ApplicationCatalogType": ".types", "ApplicationCatalogTypesResponse": ".types", "ApplicationCreate": ".types", "ApplicationEdit": ".types", "ApplicationFlags": ".types", "ApplicationFork": ".types", "ApplicationQuery": ".types", "ApplicationResponse": ".types", "ApplicationRevision": ".types", "ApplicationRevisionCommit": ".types", "ApplicationRevisionCreate": ".types", "ApplicationRevisionDataInput": ".types", "ApplicationRevisionDataInputHeadersValue": ".types", "ApplicationRevisionDataInputRuntime": ".types", "ApplicationRevisionDataOutput": ".types", "ApplicationRevisionDataOutputHeadersValue": ".types", "ApplicationRevisionDataOutputRuntime": ".types", "ApplicationRevisionEdit": ".types", "ApplicationRevisionFlags": ".types", "ApplicationRevisionFork": ".types", "ApplicationRevisionQuery": ".types", "ApplicationRevisionQueryFlags": ".types", "ApplicationRevisionResolveResponse": ".types", "ApplicationRevisionResponse": ".types", "ApplicationRevisionsLog": ".types", "ApplicationRevisionsResponse": ".types", "ApplicationVariant": ".types", "ApplicationVariantCreate": ".types", "ApplicationVariantEdit": ".types", "ApplicationVariantFlags": ".types", "ApplicationVariantFork": ".types", "ApplicationVariantResponse": ".types", "ApplicationVariantsResponse": ".types", "ApplicationsResponse": ".types", "AsyncAgentaApi": ".client", "BodyConfigsFetchVariantsConfigsFetchPost": ".types", "Bucket": ".types", "CollectStatusResponse": ".types", "ComparisonOperator": ".types", "Condition": ".types", "ConditionOperator": ".types", "ConditionOptions": ".types", "ConditionValue": ".types", "ConfigResponseModel": ".types", "CreateSimpleTestsetFromFileRequestFileType": ".testsets", "CreateTestsetRevisionFromFileRequestFileType": ".testsets", "CustomModelSettingsDto": ".types", "CustomProviderDto": ".types", "CustomProviderKind": ".types", "CustomProviderSettingsDto": ".types", "DictOperator": ".types", "DiscoverResponse": ".types", "DiscoverResponseMethodsValue": ".types", "EditSimpleTestsetFromFileRequestFileType": ".testsets", "EeSrcModelsApiOrganizationModelsOrganization": ".types", "EntityRef": ".types", "Environment": ".types", "EnvironmentCreate": ".types", "EnvironmentEdit": ".types", "EnvironmentFlags": ".types", "EnvironmentQueryFlags": ".types", "EnvironmentResponse": ".types", "EnvironmentRevision": ".types", "EnvironmentRevisionCommit": ".types", "EnvironmentRevisionCreate": ".types", "EnvironmentRevisionData": ".types", "EnvironmentRevisionDelta": ".types", "EnvironmentRevisionEdit": ".types", "EnvironmentRevisionResolveResponse": ".types", "EnvironmentRevisionResponse": ".types", "EnvironmentRevisionsLog": ".types", "EnvironmentRevisionsResponse": ".types", "EnvironmentVariant": ".types", "EnvironmentVariantCreate": ".types", "EnvironmentVariantEdit": ".types", "EnvironmentVariantResponse": ".types", "EnvironmentVariantsResponse": ".types", "EnvironmentsResponse": ".types", "ErrorPolicy": ".types", "EvaluationMetrics": ".types", "EvaluationMetricsCreate": ".types", "EvaluationMetricsIdsResponse": ".types", "EvaluationMetricsQuery": ".types", "EvaluationMetricsQueryScenarioIds": ".types", "EvaluationMetricsQueryTimestamps": ".types", "EvaluationMetricsRefresh": ".types", "EvaluationMetricsResponse": ".types", "EvaluationQueue": ".types", "EvaluationQueueCreate": ".types", "EvaluationQueueData": ".types", "EvaluationQueueEdit": ".types", "EvaluationQueueFlags": ".types", "EvaluationQueueIdResponse": ".types", "EvaluationQueueIdsResponse": ".types", "EvaluationQueueQuery": ".types", "EvaluationQueueQueryFlags": ".types", "EvaluationQueueResponse": ".types", "EvaluationQueueScenariosQuery": ".types", "EvaluationQueuesResponse": ".types", "EvaluationResult": ".types", "EvaluationResultCreate": ".types", "EvaluationResultIdResponse": ".types", "EvaluationResultIdsResponse": ".types", "EvaluationResultQuery": ".types", "EvaluationResultResponse": ".types", "EvaluationResultsResponse": ".types", "EvaluationRun": ".types", "EvaluationRunCreate": ".types", "EvaluationRunData": ".types", "EvaluationRunDataConcurrency": ".types", "EvaluationRunDataMapping": ".types", "EvaluationRunDataMappingColumn": ".types", "EvaluationRunDataMappingStep": ".types", "EvaluationRunDataStep": ".types", "EvaluationRunDataStepInput": ".types", "EvaluationRunDataStepOrigin": ".types", "EvaluationRunDataStepType": ".types", "EvaluationRunEdit": ".types", "EvaluationRunFlags": ".types", "EvaluationRunIdResponse": ".types", "EvaluationRunIdsRequest": ".types", "EvaluationRunIdsResponse": ".types", "EvaluationRunQuery": ".types", "EvaluationRunQueryFlags": ".types", "EvaluationRunResponse": ".types", "EvaluationRunsResponse": ".types", "EvaluationScenario": ".types", "EvaluationScenarioCreate": ".types", "EvaluationScenarioEdit": ".types", "EvaluationScenarioIdResponse": ".types", "EvaluationScenarioIdsResponse": ".types", "EvaluationScenarioQuery": ".types", "EvaluationScenarioResponse": ".types", "EvaluationScenariosResponse": ".types", "EvaluationStatus": ".types", "Evaluator": ".types", "EvaluatorArtifactFlags": ".types", "EvaluatorArtifactQueryFlags": ".types", "EvaluatorCatalogPreset": ".types", "EvaluatorCatalogPresetResponse": ".types", "EvaluatorCatalogPresetsResponse": ".types", "EvaluatorCatalogTemplate": ".types", "EvaluatorCatalogTemplateResponse": ".types", "EvaluatorCatalogTemplatesResponse": ".types", "EvaluatorCatalogType": ".types", "EvaluatorCatalogTypesResponse": ".types", "EvaluatorCreate": ".types", "EvaluatorEdit": ".types", "EvaluatorFlags": ".types", "EvaluatorFork": ".types", "EvaluatorQuery": ".types", "EvaluatorResponse": ".types", "EvaluatorRevision": ".types", "EvaluatorRevisionCommit": ".types", "EvaluatorRevisionCreate": ".types", "EvaluatorRevisionDataInput": ".types", "EvaluatorRevisionDataInputHeadersValue": ".types", "EvaluatorRevisionDataInputRuntime": ".types", "EvaluatorRevisionDataOutput": ".types", "EvaluatorRevisionDataOutputHeadersValue": ".types", "EvaluatorRevisionDataOutputRuntime": ".types", "EvaluatorRevisionEdit": ".types", "EvaluatorRevisionFlags": ".types", "EvaluatorRevisionFork": ".types", "EvaluatorRevisionQuery": ".types", "EvaluatorRevisionQueryFlags": ".types", "EvaluatorRevisionResolveResponse": ".types", "EvaluatorRevisionResponse": ".types", "EvaluatorRevisionsLog": ".types", "EvaluatorRevisionsResponse": ".types", "EvaluatorTemplate": ".types", "EvaluatorTemplatesResponse": ".types", "EvaluatorVariant": ".types", "EvaluatorVariantCreate": ".types", "EvaluatorVariantEdit": ".types", "EvaluatorVariantFlags": ".types", "EvaluatorVariantFork": ".types", "EvaluatorVariantResponse": ".types", "EvaluatorVariantsResponse": ".types", "EvaluatorsResponse": ".types", "Event": ".types", "EventQuery": ".types", "EventType": ".types", "EventsQueryResponse": ".types", "ExistenceOperator": ".types", "FetchLegacyAnalyticsRequestNewest": ".legacy", "FetchLegacyAnalyticsRequestOldest": ".legacy", "FetchSimpleTestsetToFileRequestFileType": ".testsets", "FetchTestsetRevisionToFileRequestFileType": ".testsets", "FilteringInput": ".types", "FilteringInputConditionsItem": ".types", "FilteringOutput": ".types", "FilteringOutputConditionsItem": ".types", "Focus": ".types", "Folder": ".types", "FolderCreate": ".types", "FolderEdit": ".types", "FolderIdResponse": ".types", "FolderKind": ".types", "FolderQuery": ".types", "FolderQueryKinds": ".types", "FolderResponse": ".types", "FoldersResponse": ".types", "Format": ".types", "Formatting": ".types", typing.Any: ".types", typing.Any: ".types", "Header": ".types", "HttpValidationError": ".types", "InviteRequest": ".types", "Invocation": ".types", "InvocationCreate": ".types", "InvocationCreateLinks": ".types", "InvocationEdit": ".types", "InvocationEditLinks": ".types", "InvocationLinkResponse": ".types", "InvocationLinks": ".types", "InvocationQuery": ".types", "InvocationQueryLinks": ".types", "InvocationResponse": ".types", "InvocationsResponse": ".types", "JsonSchemasInput": ".types", "JsonSchemasOutput": ".types", typing.Any: ".types", typing.Any: ".types", "LegacyLifecycleDto": ".types", "ListApiKeysResponse": ".types", "ListOperator": ".types", "ListOptions": ".types", "LogicalOperator": ".types", "MetricSpec": ".types", "MetricType": ".types", "MetricsBucket": ".types", "NumericOperator": ".types", "OTelEventInput": ".types", "OTelEventInputTimestamp": ".types", "OTelEventOutput": ".types", "OTelEventOutputTimestamp": ".types", "OTelHashInput": ".types", "OTelHashOutput": ".types", "OTelLinkInput": ".types", "OTelLinkOutput": ".types", "OTelLinksResponse": ".types", "OTelReferenceInput": ".types", "OTelReferenceOutput": ".types", "OTelSpanKind": ".types", "OTelStatusCode": ".types", "OTelTracingRequest": ".types", "OTelTracingResponse": ".types", "OldAnalyticsResponse": ".types", "OrganizationDetails": ".types", "OrganizationDomainResponse": ".types", "OrganizationProviderResponse": ".types", "OrganizationUpdate": ".types", "OssSrcModelsApiOrganizationModelsOrganization": ".types", "Permission": ".types", "ProjectsResponse": ".types", "QueriesResponse": ".types", "Query": ".types", "QueryApplicationVariantsRequestOrder": ".applications", "QueryCreate": ".types", "QueryEdit": ".types", "QueryEnvironmentRevisionsRequestOrder": ".environments", "QueryEnvironmentVariantsRequestOrder": ".environments", "QueryEnvironmentsRequestOrder": ".environments", "QueryEvaluatorVariantsRequestOrder": ".evaluators", "QueryFlags": ".types", "QueryQueriesRequestOrder": ".queries", "QueryQueryFlags": ".types", "QueryResponse": ".types", "QueryRevision": ".types", "QueryRevisionCommit": ".types", "QueryRevisionCreate": ".types", "QueryRevisionDataInput": ".types", "QueryRevisionDataOutput": ".types", "QueryRevisionEdit": ".types", "QueryRevisionQuery": ".types", "QueryRevisionResponse": ".types", "QueryRevisionsLog": ".types", "QueryRevisionsResponse": ".types", "QuerySpansAnalyticsRequestNewest": ".traces", "QuerySpansAnalyticsRequestOldest": ".traces", "QueryVariant": ".types", "QueryVariantCreate": ".types", "QueryVariantEdit": ".types", "QueryVariantQuery": ".types", "QueryVariantResponse": ".types", "QueryVariantsResponse": ".types", "QueryWorkflowRevisionsRequestOrder": ".workflows", "QueryWorkflowVariantsRequestOrder": ".workflows", "QueryWorkflowsRequestOrder": ".workflows", "Reference": ".types", "ReferenceRequestModelInput": ".types", "ReferenceRequestModelOutput": ".types", "RequestType": ".types", "ResolutionInfo": ".types", "RetrievalInfo": ".types", "RevisionFork": ".types", "SecretDto": ".types", "SecretDtoData": ".types", "SecretKind": ".types", "SecretResponseDto": ".types", "SecretResponseDtoData": ".types", "SessionIdsResponse": ".types", "SimpleApplication": ".types", "SimpleApplicationCreate": ".types", "SimpleApplicationDataInput": ".types", "SimpleApplicationDataInputHeadersValue": ".types", "SimpleApplicationDataInputRuntime": ".types", "SimpleApplicationDataOutput": ".types", "SimpleApplicationDataOutputHeadersValue": ".types", "SimpleApplicationDataOutputRuntime": ".types", "SimpleApplicationEdit": ".types", "SimpleApplicationFlags": ".types", "SimpleApplicationQuery": ".types", "SimpleApplicationQueryFlags": ".types", "SimpleApplicationResponse": ".types", "SimpleApplicationsResponse": ".types", "SimpleEnvironment": ".types", "SimpleEnvironmentCreate": ".types", "SimpleEnvironmentEdit": ".types", "SimpleEnvironmentQuery": ".types", "SimpleEnvironmentResponse": ".types", "SimpleEnvironmentsResponse": ".types", "SimpleEvaluation": ".types", "SimpleEvaluationCreate": ".types", "SimpleEvaluationData": ".types", "SimpleEvaluationDataApplicationSteps": ".types", "SimpleEvaluationDataApplicationStepsOneValue": ".types", "SimpleEvaluationDataEvaluatorSteps": ".types", "SimpleEvaluationDataEvaluatorStepsOneValue": ".types", "SimpleEvaluationDataQuerySteps": ".types", "SimpleEvaluationDataQueryStepsOneValue": ".types", "SimpleEvaluationDataTestsetSteps": ".types", "SimpleEvaluationDataTestsetStepsOneValue": ".types", "SimpleEvaluationEdit": ".types", "SimpleEvaluationIdResponse": ".types", "SimpleEvaluationQuery": ".types", "SimpleEvaluationResponse": ".types", "SimpleEvaluationsResponse": ".types", "SimpleEvaluator": ".types", "SimpleEvaluatorCreate": ".types", "SimpleEvaluatorDataInput": ".types", "SimpleEvaluatorDataInputHeadersValue": ".types", "SimpleEvaluatorDataInputRuntime": ".types", "SimpleEvaluatorDataOutput": ".types", "SimpleEvaluatorDataOutputHeadersValue": ".types", "SimpleEvaluatorDataOutputRuntime": ".types", "SimpleEvaluatorEdit": ".types", "SimpleEvaluatorFlags": ".types", "SimpleEvaluatorQuery": ".types", "SimpleEvaluatorQueryFlags": ".types", "SimpleEvaluatorResponse": ".types", "SimpleEvaluatorsResponse": ".types", "SimpleQueriesResponse": ".types", "SimpleQuery": ".types", "SimpleQueryCreate": ".types", "SimpleQueryEdit": ".types", "SimpleQueryQuery": ".types", "SimpleQueryResponse": ".types", "SimpleQueue": ".types", "SimpleQueueCreate": ".types", "SimpleQueueData": ".types", "SimpleQueueDataEvaluators": ".types", "SimpleQueueDataEvaluatorsOneValue": ".types", "SimpleQueueIdResponse": ".types", "SimpleQueueKind": ".types", "SimpleQueueQuery": ".types", "SimpleQueueResponse": ".types", "SimpleQueueScenariosQuery": ".types", "SimpleQueueScenariosResponse": ".types", "SimpleQueueSettings": ".types", "SimpleQueuesResponse": ".types", "SimpleTestset": ".types", "SimpleTestsetCreate": ".types", "SimpleTestsetEdit": ".types", "SimpleTestsetQuery": ".types", "SimpleTestsetResponse": ".types", "SimpleTestsetsResponse": ".types", "SimpleTrace": ".types", "SimpleTraceChannel": ".types", "SimpleTraceCreate": ".types", "SimpleTraceCreateLinks": ".types", "SimpleTraceEdit": ".types", "SimpleTraceEditLinks": ".types", "SimpleTraceKind": ".types", "SimpleTraceLinkResponse": ".types", "SimpleTraceLinks": ".types", "SimpleTraceOrigin": ".types", "SimpleTraceQuery": ".types", "SimpleTraceQueryLinks": ".types", "SimpleTraceReferences": ".types", "SimpleTraceResponse": ".types", "SimpleTracesResponse": ".types", "SimpleWorkflow": ".types", "SimpleWorkflowCreate": ".types", "SimpleWorkflowDataInput": ".types", "SimpleWorkflowDataInputHeadersValue": ".types", "SimpleWorkflowDataInputRuntime": ".types", "SimpleWorkflowDataOutput": ".types", "SimpleWorkflowDataOutputHeadersValue": ".types", "SimpleWorkflowDataOutputRuntime": ".types", "SimpleWorkflowEdit": ".types", "SimpleWorkflowFlags": ".types", "SimpleWorkflowQuery": ".types", "SimpleWorkflowQueryFlags": ".types", "SimpleWorkflowResponse": ".types", "SimpleWorkflowsResponse": ".types", "SpanInput": ".types", "SpanInputEndTime": ".types", "SpanInputStartTime": ".types", "SpanOutput": ".types", "SpanOutputEndTime": ".types", "SpanOutputStartTime": ".types", "SpanResponse": ".types", "SpanType": ".types", "SpansNodeInput": ".types", "SpansNodeInputEndTime": ".types", "SpansNodeInputSpansValue": ".types", "SpansNodeInputStartTime": ".types", "SpansNodeOutput": ".types", "SpansNodeOutputEndTime": ".types", "SpansNodeOutputSpansValue": ".types", "SpansNodeOutputStartTime": ".types", "SpansResponse": ".types", "SpansTreeInput": ".types", "SpansTreeInputSpansValue": ".types", "SpansTreeOutput": ".types", "SpansTreeOutputSpansValue": ".types", "SsoProviderDto": ".types", "SsoProviderInfo": ".types", "SsoProviderSettingsDto": ".types", "SsoProviders": ".types", "StandardProviderDto": ".types", "StandardProviderKind": ".types", "StandardProviderSettingsDto": ".types", "Status": ".types", "StringOperator": ".types", "TestcaseInput": ".types", "TestcaseOutput": ".types", "TestcaseResponse": ".types", "TestcasesResponse": ".types", "Testset": ".types", "TestsetCreate": ".types", "TestsetEdit": ".types", "TestsetFlags": ".types", "TestsetQuery": ".types", "TestsetResponse": ".types", "TestsetRevision": ".types", "TestsetRevisionCommit": ".types", "TestsetRevisionCreate": ".types", "TestsetRevisionDataInput": ".types", "TestsetRevisionDataOutput": ".types", "TestsetRevisionDelta": ".types", "TestsetRevisionDeltaColumns": ".types", "TestsetRevisionDeltaRows": ".types", "TestsetRevisionEdit": ".types", "TestsetRevisionQuery": ".types", "TestsetRevisionResponse": ".types", "TestsetRevisionsLog": ".types", "TestsetRevisionsResponse": ".types", "TestsetVariant": ".types", "TestsetVariantCreate": ".types", "TestsetVariantEdit": ".types", "TestsetVariantQuery": ".types", "TestsetVariantResponse": ".types", "TestsetVariantsResponse": ".types", "TestsetsResponse": ".types", "TextOptions": ".types", "ToolAuthScheme": ".types", "ToolCallData": ".types", "ToolCallFunction": ".types", "ToolCallResponse": ".types", "ToolCatalogAction": ".types", "ToolCatalogActionDetails": ".types", "ToolCatalogActionResponse": ".types", "ToolCatalogActionResponseAction": ".types", "ToolCatalogActionsResponse": ".types", "ToolCatalogActionsResponseActionsItem": ".types", "ToolCatalogIntegration": ".types", "ToolCatalogIntegrationDetails": ".types", "ToolCatalogIntegrationResponse": ".types", "ToolCatalogIntegrationResponseIntegration": ".types", "ToolCatalogIntegrationsResponse": ".types", "ToolCatalogIntegrationsResponseIntegrationsItem": ".types", "ToolCatalogProvider": ".types", "ToolCatalogProviderDetails": ".types", "ToolCatalogProviderResponse": ".types", "ToolCatalogProviderResponseProvider": ".types", "ToolCatalogProvidersResponse": ".types", "ToolCatalogProvidersResponseProvidersItem": ".types", "ToolConnection": ".types", "ToolConnectionCreate": ".types", "ToolConnectionCreateData": ".types", "ToolConnectionResponse": ".types", "ToolConnectionStatus": ".types", "ToolConnectionsResponse": ".types", "ToolProviderKind": ".types", "ToolResult": ".types", "ToolResultData": ".types", "TraceIdResponse": ".types", "TraceIdsResponse": ".types", "TraceInput": ".types", "TraceInputSpansValue": ".types", "TraceOutput": ".types", "TraceOutputSpansValue": ".types", "TraceRequest": ".types", "TraceResponse": ".types", "TraceType": ".types", "TracesRequest": ".types", "TracesResponse": ".types", "TracingQuery": ".types", "UnprocessableEntityError": ".errors", "UserIdsResponse": ".types", "ValidationError": ".types", "ValidationErrorLocItem": ".types", "VariantFork": ".types", "WebhookDeliveriesResponse": ".types", "WebhookDelivery": ".types", "WebhookDeliveryCreate": ".types", "WebhookDeliveryData": ".types", "WebhookDeliveryQuery": ".types", "WebhookDeliveryResponse": ".types", "WebhookDeliveryResponseInfo": ".types", "WebhookEventType": ".types", "WebhookProviderDto": ".types", "WebhookProviderSettingsDto": ".types", "WebhookSubscription": ".types", "WebhookSubscriptionCreate": ".types", "WebhookSubscriptionData": ".types", "WebhookSubscriptionDataAuthMode": ".types", "WebhookSubscriptionEdit": ".types", "WebhookSubscriptionQuery": ".types", "WebhookSubscriptionResponse": ".types", "WebhookSubscriptionTestRequestSubscription": ".webhooks", "WebhookSubscriptionsResponse": ".types", "Windowing": ".types", "WindowingOrder": ".types", "Workflow": ".types", "WorkflowArtifactFlags": ".types", "WorkflowCatalogFlags": ".types", "WorkflowCatalogPreset": ".types", "WorkflowCatalogPresetResponse": ".types", "WorkflowCatalogPresetsResponse": ".types", "WorkflowCatalogTemplate": ".types", "WorkflowCatalogTemplateResponse": ".types", "WorkflowCatalogTemplatesResponse": ".types", "WorkflowCatalogType": ".types", "WorkflowCatalogTypeResponse": ".types", "WorkflowCatalogTypesResponse": ".types", "WorkflowCreate": ".types", "WorkflowEdit": ".types", "WorkflowFlags": ".types", "WorkflowFork": ".types", "WorkflowResponse": ".types", "WorkflowRevisionCommit": ".types", "WorkflowRevisionCreate": ".types", "WorkflowRevisionDataInput": ".types", "WorkflowRevisionDataInputHeadersValue": ".types", "WorkflowRevisionDataInputRuntime": ".types", "WorkflowRevisionDataOutput": ".types", "WorkflowRevisionDataOutputHeadersValue": ".types", "WorkflowRevisionDataOutputRuntime": ".types", "WorkflowRevisionEdit": ".types", "WorkflowRevisionFlags": ".types", "WorkflowRevisionFork": ".types", "WorkflowRevisionInput": ".types", "WorkflowRevisionOutput": ".types", "WorkflowRevisionResolveResponse": ".types", "WorkflowRevisionResponse": ".types", "WorkflowRevisionsLog": ".types", "WorkflowRevisionsResponse": ".types", "WorkflowVariant": ".types", "WorkflowVariantCreate": ".types", "WorkflowVariantEdit": ".types", "WorkflowVariantFlags": ".types", "WorkflowVariantFork": ".types", "WorkflowVariantResponse": ".types", "WorkflowVariantsResponse": ".types", "WorkflowsResponse": ".types", "Workspace": ".types", "WorkspaceMemberResponse": ".types", "WorkspacePermission": ".types", "WorkspaceResponse": ".types", "access": ".access", "annotations": ".annotations", "applications": ".applications", "billing": ".billing", "environments": ".environments", "evaluations": ".evaluations", "evaluators": ".evaluators", "events": ".events", "folders": ".folders", "invocations": ".invocations", "keys": ".keys", "legacy": ".legacy", "organizations": ".organizations", "projects": ".projects", "queries": ".queries", "secrets": ".secrets", "status": ".status", "testcases": ".testcases", "testsets": ".testsets", "tools": ".tools", "traces": ".traces", "users": ".users", "webhooks": ".webhooks", "workflows": ".workflows", "workspaces": ".workspaces"} def __getattr__(attr_name: str) -> typing.Any: module_name = _dynamic_imports.get(attr_name) if module_name is None: @@ -37,4 +37,4 @@ def __getattr__(attr_name: str) -> typing.Any: def __dir__(): lazy_attrs = list(_dynamic_imports.keys()) return sorted(lazy_attrs) -__all__ = ["AdminAccountCreateOptions", "AdminAccountRead", "AdminAccountsCreate", "AdminAccountsDelete", "AdminAccountsDeleteTarget", "AdminAccountsResponse", "AdminApiKeyCreate", "AdminApiKeyResponse", "AdminDeleteResponse", "AdminDeletedEntities", "AdminDeletedEntity", "AdminOrganizationCreate", "AdminOrganizationMembershipCreate", "AdminOrganizationMembershipRead", "AdminOrganizationRead", "AdminProjectCreate", "AdminProjectMembershipCreate", "AdminProjectMembershipRead", "AdminProjectRead", "AdminSimpleAccountCreate", "AdminSimpleAccountDeleteEntry", "AdminSimpleAccountRead", "AdminSimpleAccountsApiKeysCreate", "AdminSimpleAccountsCreate", "AdminSimpleAccountsDelete", "AdminSimpleAccountsOrganizationsCreate", "AdminSimpleAccountsOrganizationsMembershipsCreate", "AdminSimpleAccountsOrganizationsTransferOwnership", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjects", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjectsZero", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspaces", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspacesZero", "AdminSimpleAccountsOrganizationsTransferOwnershipResponse", "AdminSimpleAccountsProjectsCreate", "AdminSimpleAccountsProjectsMembershipsCreate", "AdminSimpleAccountsResponse", "AdminSimpleAccountsUsersCreate", "AdminSimpleAccountsUsersIdentitiesCreate", "AdminSimpleAccountsUsersResetPassword", "AdminSimpleAccountsWorkspacesCreate", "AdminSimpleAccountsWorkspacesMembershipsCreate", "AdminStructuredError", "AdminSubscriptionCreate", "AdminSubscriptionRead", "AdminUserCreate", "AdminUserIdentityCreate", "AdminUserIdentityRead", "AdminUserIdentityReadStatus", "AdminUserRead", "AdminWorkspaceCreate", "AdminWorkspaceMembershipCreate", "AdminWorkspaceMembershipRead", "AdminWorkspaceRead", "AgentaApi", "AgentaApiEnvironment", "Analytics", "AnalyticsResponse", "Annotation", "AnnotationCreate", "AnnotationCreateLinks", "AnnotationEdit", "AnnotationEditLinks", "AnnotationLinkResponse", "AnnotationLinks", "AnnotationQuery", "AnnotationQueryLinks", "AnnotationResponse", "AnnotationsResponse", "Application", "ApplicationArtifactFlags", "ApplicationArtifactQueryFlags", "ApplicationCatalogPreset", "ApplicationCatalogPresetResponse", "ApplicationCatalogPresetsResponse", "ApplicationCatalogTemplate", "ApplicationCatalogTemplateResponse", "ApplicationCatalogTemplatesResponse", "ApplicationCatalogType", "ApplicationCatalogTypesResponse", "ApplicationCreate", "ApplicationEdit", "ApplicationFlags", "ApplicationFork", "ApplicationQuery", "ApplicationResponse", "ApplicationRevision", "ApplicationRevisionCommit", "ApplicationRevisionCreate", "ApplicationRevisionDataInput", "ApplicationRevisionDataInputHeadersValue", "ApplicationRevisionDataInputRuntime", "ApplicationRevisionDataOutput", "ApplicationRevisionDataOutputHeadersValue", "ApplicationRevisionDataOutputRuntime", "ApplicationRevisionEdit", "ApplicationRevisionFlags", "ApplicationRevisionFork", "ApplicationRevisionQuery", "ApplicationRevisionQueryFlags", "ApplicationRevisionResolveResponse", "ApplicationRevisionResponse", "ApplicationRevisionsLog", "ApplicationRevisionsResponse", "ApplicationVariant", "ApplicationVariantCreate", "ApplicationVariantEdit", "ApplicationVariantFlags", "ApplicationVariantFork", "ApplicationVariantResponse", "ApplicationVariantsResponse", "ApplicationsResponse", "AsyncAgentaApi", "BodyConfigsFetchVariantsConfigsFetchPost", "Bucket", "CollectStatusResponse", "ComparisonOperator", "Condition", "ConditionOperator", "ConditionOptions", "ConditionValue", "ConfigResponseModel", "CreateSimpleTestsetFromFileRequestFileType", "CreateTestsetRevisionFromFileRequestFileType", "CustomModelSettingsDto", "CustomProviderDto", "CustomProviderKind", "CustomProviderSettingsDto", "DictOperator", "DiscoverResponse", "DiscoverResponseMethodsValue", "EditSimpleTestsetFromFileRequestFileType", "EeSrcModelsApiOrganizationModelsOrganization", "EntityRef", "Environment", "EnvironmentCreate", "EnvironmentEdit", "EnvironmentFlags", "EnvironmentQueryFlags", "EnvironmentResponse", "EnvironmentRevision", "EnvironmentRevisionCommit", "EnvironmentRevisionCreate", "EnvironmentRevisionData", "EnvironmentRevisionDelta", "EnvironmentRevisionEdit", "EnvironmentRevisionResolveResponse", "EnvironmentRevisionResponse", "EnvironmentRevisionsLog", "EnvironmentRevisionsResponse", "EnvironmentVariant", "EnvironmentVariantCreate", "EnvironmentVariantEdit", "EnvironmentVariantResponse", "EnvironmentVariantsResponse", "EnvironmentsResponse", "ErrorPolicy", "EvaluationMetrics", "EvaluationMetricsCreate", "EvaluationMetricsEdit", "EvaluationMetricsIdsResponse", "EvaluationMetricsQuery", "EvaluationMetricsQueryScenarioIds", "EvaluationMetricsQueryTimestamps", "EvaluationMetricsRefresh", "EvaluationMetricsResponse", "EvaluationQueue", "EvaluationQueueCreate", "EvaluationQueueData", "EvaluationQueueEdit", "EvaluationQueueFlags", "EvaluationQueueIdResponse", "EvaluationQueueIdsResponse", "EvaluationQueueQuery", "EvaluationQueueQueryFlags", "EvaluationQueueResponse", "EvaluationQueueScenariosQuery", "EvaluationQueuesResponse", "EvaluationResult", "EvaluationResultCreate", "EvaluationResultEdit", "EvaluationResultIdResponse", "EvaluationResultIdsResponse", "EvaluationResultQuery", "EvaluationResultResponse", "EvaluationResultsResponse", "EvaluationRun", "EvaluationRunCreate", "EvaluationRunData", "EvaluationRunDataMapping", "EvaluationRunDataMappingColumn", "EvaluationRunDataMappingStep", "EvaluationRunDataStep", "EvaluationRunDataStepInput", "EvaluationRunDataStepOrigin", "EvaluationRunDataStepType", "EvaluationRunEdit", "EvaluationRunFlags", "EvaluationRunIdResponse", "EvaluationRunIdsRequest", "EvaluationRunIdsResponse", "EvaluationRunQuery", "EvaluationRunQueryFlags", "EvaluationRunResponse", "EvaluationRunsResponse", "EvaluationScenario", "EvaluationScenarioCreate", "EvaluationScenarioEdit", "EvaluationScenarioIdResponse", "EvaluationScenarioIdsResponse", "EvaluationScenarioQuery", "EvaluationScenarioResponse", "EvaluationScenariosResponse", "EvaluationStatus", "Evaluator", "EvaluatorArtifactFlags", "EvaluatorArtifactQueryFlags", "EvaluatorCatalogPreset", "EvaluatorCatalogPresetResponse", "EvaluatorCatalogPresetsResponse", "EvaluatorCatalogTemplate", "EvaluatorCatalogTemplateResponse", "EvaluatorCatalogTemplatesResponse", "EvaluatorCatalogType", "EvaluatorCatalogTypesResponse", "EvaluatorCreate", "EvaluatorEdit", "EvaluatorFlags", "EvaluatorFork", "EvaluatorQuery", "EvaluatorResponse", "EvaluatorRevision", "EvaluatorRevisionCommit", "EvaluatorRevisionCreate", "EvaluatorRevisionDataInput", "EvaluatorRevisionDataInputHeadersValue", "EvaluatorRevisionDataInputRuntime", "EvaluatorRevisionDataOutput", "EvaluatorRevisionDataOutputHeadersValue", "EvaluatorRevisionDataOutputRuntime", "EvaluatorRevisionEdit", "EvaluatorRevisionFlags", "EvaluatorRevisionFork", "EvaluatorRevisionQuery", "EvaluatorRevisionQueryFlags", "EvaluatorRevisionResolveResponse", "EvaluatorRevisionResponse", "EvaluatorRevisionsLog", "EvaluatorRevisionsResponse", "EvaluatorTemplate", "EvaluatorTemplatesResponse", "EvaluatorVariant", "EvaluatorVariantCreate", "EvaluatorVariantEdit", "EvaluatorVariantFlags", "EvaluatorVariantFork", "EvaluatorVariantResponse", "EvaluatorVariantsResponse", "EvaluatorsResponse", "Event", "EventQuery", "EventType", "EventsQueryResponse", "ExistenceOperator", "FetchLegacyAnalyticsRequestNewest", "FetchLegacyAnalyticsRequestOldest", "FetchSimpleTestsetToFileRequestFileType", "FetchTestsetRevisionToFileRequestFileType", "FilteringInput", "FilteringInputConditionsItem", "FilteringOutput", "FilteringOutputConditionsItem", "Focus", "Folder", "FolderCreate", "FolderEdit", "FolderIdResponse", "FolderKind", "FolderQuery", "FolderQueryKinds", "FolderResponse", "FoldersResponse", "Format", "Formatting", typing.Any, typing.Any, "Header", "HttpValidationError", "InviteRequest", "Invocation", "InvocationCreate", "InvocationCreateLinks", "InvocationEdit", "InvocationEditLinks", "InvocationLinkResponse", "InvocationLinks", "InvocationQuery", "InvocationQueryLinks", "InvocationResponse", "InvocationsResponse", "JsonSchemasInput", "JsonSchemasOutput", typing.Any, typing.Any, "LegacyLifecycleDto", "ListApiKeysResponse", "ListOperator", "ListOptions", "LogicalOperator", "MetricSpec", "MetricType", "MetricsBucket", "NumericOperator", "OTelEventInput", "OTelEventInputTimestamp", "OTelEventOutput", "OTelEventOutputTimestamp", "OTelHashInput", "OTelHashOutput", "OTelLinkInput", "OTelLinkOutput", "OTelLinksResponse", "OTelReferenceInput", "OTelReferenceOutput", "OTelSpanKind", "OTelStatusCode", "OTelTracingRequest", "OTelTracingResponse", "OldAnalyticsResponse", "OrganizationDetails", "OrganizationDomainResponse", "OrganizationProviderResponse", "OrganizationUpdate", "OssSrcModelsApiOrganizationModelsOrganization", "Permission", "ProjectsResponse", "QueriesResponse", "Query", "QueryApplicationVariantsRequestOrder", "QueryCreate", "QueryEdit", "QueryEnvironmentRevisionsRequestOrder", "QueryEnvironmentVariantsRequestOrder", "QueryEnvironmentsRequestOrder", "QueryEvaluatorVariantsRequestOrder", "QueryFlags", "QueryQueriesRequestOrder", "QueryQueryFlags", "QueryResponse", "QueryRevision", "QueryRevisionCommit", "QueryRevisionCreate", "QueryRevisionDataInput", "QueryRevisionDataOutput", "QueryRevisionEdit", "QueryRevisionQuery", "QueryRevisionResponse", "QueryRevisionsLog", "QueryRevisionsResponse", "QuerySpansAnalyticsRequestNewest", "QuerySpansAnalyticsRequestOldest", "QueryVariant", "QueryVariantCreate", "QueryVariantEdit", "QueryVariantQuery", "QueryVariantResponse", "QueryVariantsResponse", "QueryWorkflowRevisionsRequestOrder", "QueryWorkflowVariantsRequestOrder", "QueryWorkflowsRequestOrder", "Reference", "ReferenceRequestModelInput", "ReferenceRequestModelOutput", "RequestType", "ResolutionInfo", "RetrievalInfo", "RevisionFork", "SecretDto", "SecretDtoData", "SecretKind", "SecretResponseDto", "SecretResponseDtoData", "SessionIdsResponse", "SimpleApplication", "SimpleApplicationCreate", "SimpleApplicationDataInput", "SimpleApplicationDataInputHeadersValue", "SimpleApplicationDataInputRuntime", "SimpleApplicationDataOutput", "SimpleApplicationDataOutputHeadersValue", "SimpleApplicationDataOutputRuntime", "SimpleApplicationEdit", "SimpleApplicationFlags", "SimpleApplicationQuery", "SimpleApplicationQueryFlags", "SimpleApplicationResponse", "SimpleApplicationsResponse", "SimpleEnvironment", "SimpleEnvironmentCreate", "SimpleEnvironmentEdit", "SimpleEnvironmentQuery", "SimpleEnvironmentResponse", "SimpleEnvironmentsResponse", "SimpleEvaluation", "SimpleEvaluationCreate", "SimpleEvaluationData", "SimpleEvaluationDataApplicationSteps", "SimpleEvaluationDataApplicationStepsOneValue", "SimpleEvaluationDataEvaluatorSteps", "SimpleEvaluationDataEvaluatorStepsOneValue", "SimpleEvaluationDataQuerySteps", "SimpleEvaluationDataQueryStepsOneValue", "SimpleEvaluationDataTestsetSteps", "SimpleEvaluationDataTestsetStepsOneValue", "SimpleEvaluationEdit", "SimpleEvaluationIdResponse", "SimpleEvaluationQuery", "SimpleEvaluationResponse", "SimpleEvaluationsResponse", "SimpleEvaluator", "SimpleEvaluatorCreate", "SimpleEvaluatorDataInput", "SimpleEvaluatorDataInputHeadersValue", "SimpleEvaluatorDataInputRuntime", "SimpleEvaluatorDataOutput", "SimpleEvaluatorDataOutputHeadersValue", "SimpleEvaluatorDataOutputRuntime", "SimpleEvaluatorEdit", "SimpleEvaluatorFlags", "SimpleEvaluatorQuery", "SimpleEvaluatorQueryFlags", "SimpleEvaluatorResponse", "SimpleEvaluatorsResponse", "SimpleQueriesResponse", "SimpleQuery", "SimpleQueryCreate", "SimpleQueryEdit", "SimpleQueryQuery", "SimpleQueryResponse", "SimpleQueue", "SimpleQueueCreate", "SimpleQueueData", "SimpleQueueDataEvaluators", "SimpleQueueDataEvaluatorsOneValue", "SimpleQueueIdResponse", "SimpleQueueKind", "SimpleQueueQuery", "SimpleQueueResponse", "SimpleQueueScenariosQuery", "SimpleQueueScenariosResponse", "SimpleQueueSettings", "SimpleQueuesResponse", "SimpleTestset", "SimpleTestsetCreate", "SimpleTestsetEdit", "SimpleTestsetQuery", "SimpleTestsetResponse", "SimpleTestsetsResponse", "SimpleTrace", "SimpleTraceChannel", "SimpleTraceCreate", "SimpleTraceCreateLinks", "SimpleTraceEdit", "SimpleTraceEditLinks", "SimpleTraceKind", "SimpleTraceLinkResponse", "SimpleTraceLinks", "SimpleTraceOrigin", "SimpleTraceQuery", "SimpleTraceQueryLinks", "SimpleTraceReferences", "SimpleTraceResponse", "SimpleTracesResponse", "SimpleWorkflow", "SimpleWorkflowCreate", "SimpleWorkflowDataInput", "SimpleWorkflowDataInputHeadersValue", "SimpleWorkflowDataInputRuntime", "SimpleWorkflowDataOutput", "SimpleWorkflowDataOutputHeadersValue", "SimpleWorkflowDataOutputRuntime", "SimpleWorkflowEdit", "SimpleWorkflowFlags", "SimpleWorkflowQuery", "SimpleWorkflowQueryFlags", "SimpleWorkflowResponse", "SimpleWorkflowsResponse", "SpanInput", "SpanInputEndTime", "SpanInputStartTime", "SpanOutput", "SpanOutputEndTime", "SpanOutputStartTime", "SpanResponse", "SpanType", "SpansNodeInput", "SpansNodeInputEndTime", "SpansNodeInputSpansValue", "SpansNodeInputStartTime", "SpansNodeOutput", "SpansNodeOutputEndTime", "SpansNodeOutputSpansValue", "SpansNodeOutputStartTime", "SpansResponse", "SpansTreeInput", "SpansTreeInputSpansValue", "SpansTreeOutput", "SpansTreeOutputSpansValue", "SsoProviderDto", "SsoProviderInfo", "SsoProviderSettingsDto", "SsoProviders", "StandardProviderDto", "StandardProviderKind", "StandardProviderSettingsDto", "Status", "StringOperator", "TestcaseInput", "TestcaseOutput", "TestcaseResponse", "TestcasesResponse", "Testset", "TestsetCreate", "TestsetEdit", "TestsetFlags", "TestsetQuery", "TestsetResponse", "TestsetRevision", "TestsetRevisionCommit", "TestsetRevisionCreate", "TestsetRevisionDataInput", "TestsetRevisionDataOutput", "TestsetRevisionDelta", "TestsetRevisionDeltaColumns", "TestsetRevisionDeltaRows", "TestsetRevisionEdit", "TestsetRevisionQuery", "TestsetRevisionResponse", "TestsetRevisionsLog", "TestsetRevisionsResponse", "TestsetVariant", "TestsetVariantCreate", "TestsetVariantEdit", "TestsetVariantQuery", "TestsetVariantResponse", "TestsetVariantsResponse", "TestsetsResponse", "TextOptions", "ToolAuthScheme", "ToolCallData", "ToolCallFunction", "ToolCallResponse", "ToolCatalogAction", "ToolCatalogActionDetails", "ToolCatalogActionResponse", "ToolCatalogActionResponseAction", "ToolCatalogActionsResponse", "ToolCatalogActionsResponseActionsItem", "ToolCatalogIntegration", "ToolCatalogIntegrationDetails", "ToolCatalogIntegrationResponse", "ToolCatalogIntegrationResponseIntegration", "ToolCatalogIntegrationsResponse", "ToolCatalogIntegrationsResponseIntegrationsItem", "ToolCatalogProvider", "ToolCatalogProviderDetails", "ToolCatalogProviderResponse", "ToolCatalogProviderResponseProvider", "ToolCatalogProvidersResponse", "ToolCatalogProvidersResponseProvidersItem", "ToolConnection", "ToolConnectionCreate", "ToolConnectionCreateData", "ToolConnectionResponse", "ToolConnectionStatus", "ToolConnectionsResponse", "ToolProviderKind", "ToolResult", "ToolResultData", "TraceIdResponse", "TraceIdsResponse", "TraceInput", "TraceInputSpansValue", "TraceOutput", "TraceOutputSpansValue", "TraceRequest", "TraceResponse", "TraceType", "TracesRequest", "TracesResponse", "TracingQuery", "UnprocessableEntityError", "UserIdsResponse", "ValidationError", "ValidationErrorLocItem", "VariantFork", "WebhookDeliveriesResponse", "WebhookDelivery", "WebhookDeliveryCreate", "WebhookDeliveryData", "WebhookDeliveryQuery", "WebhookDeliveryResponse", "WebhookDeliveryResponseInfo", "WebhookEventType", "WebhookProviderDto", "WebhookProviderSettingsDto", "WebhookSubscription", "WebhookSubscriptionCreate", "WebhookSubscriptionData", "WebhookSubscriptionDataAuthMode", "WebhookSubscriptionEdit", "WebhookSubscriptionQuery", "WebhookSubscriptionResponse", "WebhookSubscriptionTestRequestSubscription", "WebhookSubscriptionsResponse", "Windowing", "WindowingOrder", "Workflow", "WorkflowArtifactFlags", "WorkflowCatalogFlags", "WorkflowCatalogPreset", "WorkflowCatalogPresetResponse", "WorkflowCatalogPresetsResponse", "WorkflowCatalogTemplate", "WorkflowCatalogTemplateResponse", "WorkflowCatalogTemplatesResponse", "WorkflowCatalogType", "WorkflowCatalogTypeResponse", "WorkflowCatalogTypesResponse", "WorkflowCreate", "WorkflowEdit", "WorkflowFlags", "WorkflowFork", "WorkflowResponse", "WorkflowRevisionCommit", "WorkflowRevisionCreate", "WorkflowRevisionDataInput", "WorkflowRevisionDataInputHeadersValue", "WorkflowRevisionDataInputRuntime", "WorkflowRevisionDataOutput", "WorkflowRevisionDataOutputHeadersValue", "WorkflowRevisionDataOutputRuntime", "WorkflowRevisionEdit", "WorkflowRevisionFlags", "WorkflowRevisionFork", "WorkflowRevisionInput", "WorkflowRevisionOutput", "WorkflowRevisionResolveResponse", "WorkflowRevisionResponse", "WorkflowRevisionsLog", "WorkflowRevisionsResponse", "WorkflowVariant", "WorkflowVariantCreate", "WorkflowVariantEdit", "WorkflowVariantFlags", "WorkflowVariantFork", "WorkflowVariantResponse", "WorkflowVariantsResponse", "WorkflowsResponse", "Workspace", "WorkspaceMemberResponse", "WorkspacePermission", "WorkspaceResponse", "access", "annotations", "applications", "billing", "environments", "evaluations", "evaluators", "events", "folders", "invocations", "keys", "legacy", "organizations", "projects", "queries", "secrets", "status", "testcases", "testsets", "tools", "traces", "users", "webhooks", "workflows", "workspaces"] +__all__ = ["AdminAccountCreateOptions", "AdminAccountRead", "AdminAccountsCreate", "AdminAccountsDelete", "AdminAccountsDeleteTarget", "AdminAccountsResponse", "AdminApiKeyCreate", "AdminApiKeyResponse", "AdminDeleteResponse", "AdminDeletedEntities", "AdminDeletedEntity", "AdminOrganizationCreate", "AdminOrganizationMembershipCreate", "AdminOrganizationMembershipRead", "AdminOrganizationRead", "AdminProjectCreate", "AdminProjectMembershipCreate", "AdminProjectMembershipRead", "AdminProjectRead", "AdminSimpleAccountCreate", "AdminSimpleAccountDeleteEntry", "AdminSimpleAccountRead", "AdminSimpleAccountsApiKeysCreate", "AdminSimpleAccountsCreate", "AdminSimpleAccountsDelete", "AdminSimpleAccountsOrganizationsCreate", "AdminSimpleAccountsOrganizationsMembershipsCreate", "AdminSimpleAccountsOrganizationsTransferOwnership", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjects", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjectsZero", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspaces", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspacesZero", "AdminSimpleAccountsOrganizationsTransferOwnershipResponse", "AdminSimpleAccountsProjectsCreate", "AdminSimpleAccountsProjectsMembershipsCreate", "AdminSimpleAccountsResponse", "AdminSimpleAccountsUsersCreate", "AdminSimpleAccountsUsersIdentitiesCreate", "AdminSimpleAccountsUsersResetPassword", "AdminSimpleAccountsWorkspacesCreate", "AdminSimpleAccountsWorkspacesMembershipsCreate", "AdminStructuredError", "AdminSubscriptionCreate", "AdminSubscriptionRead", "AdminUserCreate", "AdminUserIdentityCreate", "AdminUserIdentityRead", "AdminUserIdentityReadStatus", "AdminUserRead", "AdminWorkspaceCreate", "AdminWorkspaceMembershipCreate", "AdminWorkspaceMembershipRead", "AdminWorkspaceRead", "AgentaApi", "AgentaApiEnvironment", "Analytics", "AnalyticsResponse", "Annotation", "AnnotationCreate", "AnnotationCreateLinks", "AnnotationEdit", "AnnotationEditLinks", "AnnotationLinkResponse", "AnnotationLinks", "AnnotationQuery", "AnnotationQueryLinks", "AnnotationResponse", "AnnotationsResponse", "Application", "ApplicationArtifactFlags", "ApplicationArtifactQueryFlags", "ApplicationCatalogPreset", "ApplicationCatalogPresetResponse", "ApplicationCatalogPresetsResponse", "ApplicationCatalogTemplate", "ApplicationCatalogTemplateResponse", "ApplicationCatalogTemplatesResponse", "ApplicationCatalogType", "ApplicationCatalogTypesResponse", "ApplicationCreate", "ApplicationEdit", "ApplicationFlags", "ApplicationFork", "ApplicationQuery", "ApplicationResponse", "ApplicationRevision", "ApplicationRevisionCommit", "ApplicationRevisionCreate", "ApplicationRevisionDataInput", "ApplicationRevisionDataInputHeadersValue", "ApplicationRevisionDataInputRuntime", "ApplicationRevisionDataOutput", "ApplicationRevisionDataOutputHeadersValue", "ApplicationRevisionDataOutputRuntime", "ApplicationRevisionEdit", "ApplicationRevisionFlags", "ApplicationRevisionFork", "ApplicationRevisionQuery", "ApplicationRevisionQueryFlags", "ApplicationRevisionResolveResponse", "ApplicationRevisionResponse", "ApplicationRevisionsLog", "ApplicationRevisionsResponse", "ApplicationVariant", "ApplicationVariantCreate", "ApplicationVariantEdit", "ApplicationVariantFlags", "ApplicationVariantFork", "ApplicationVariantResponse", "ApplicationVariantsResponse", "ApplicationsResponse", "AsyncAgentaApi", "BodyConfigsFetchVariantsConfigsFetchPost", "Bucket", "CollectStatusResponse", "ComparisonOperator", "Condition", "ConditionOperator", "ConditionOptions", "ConditionValue", "ConfigResponseModel", "CreateSimpleTestsetFromFileRequestFileType", "CreateTestsetRevisionFromFileRequestFileType", "CustomModelSettingsDto", "CustomProviderDto", "CustomProviderKind", "CustomProviderSettingsDto", "DictOperator", "DiscoverResponse", "DiscoverResponseMethodsValue", "EditSimpleTestsetFromFileRequestFileType", "EeSrcModelsApiOrganizationModelsOrganization", "EntityRef", "Environment", "EnvironmentCreate", "EnvironmentEdit", "EnvironmentFlags", "EnvironmentQueryFlags", "EnvironmentResponse", "EnvironmentRevision", "EnvironmentRevisionCommit", "EnvironmentRevisionCreate", "EnvironmentRevisionData", "EnvironmentRevisionDelta", "EnvironmentRevisionEdit", "EnvironmentRevisionResolveResponse", "EnvironmentRevisionResponse", "EnvironmentRevisionsLog", "EnvironmentRevisionsResponse", "EnvironmentVariant", "EnvironmentVariantCreate", "EnvironmentVariantEdit", "EnvironmentVariantResponse", "EnvironmentVariantsResponse", "EnvironmentsResponse", "ErrorPolicy", "EvaluationMetrics", "EvaluationMetricsCreate", "EvaluationMetricsIdsResponse", "EvaluationMetricsQuery", "EvaluationMetricsQueryScenarioIds", "EvaluationMetricsQueryTimestamps", "EvaluationMetricsRefresh", "EvaluationMetricsResponse", "EvaluationQueue", "EvaluationQueueCreate", "EvaluationQueueData", "EvaluationQueueEdit", "EvaluationQueueFlags", "EvaluationQueueIdResponse", "EvaluationQueueIdsResponse", "EvaluationQueueQuery", "EvaluationQueueQueryFlags", "EvaluationQueueResponse", "EvaluationQueueScenariosQuery", "EvaluationQueuesResponse", "EvaluationResult", "EvaluationResultCreate", "EvaluationResultIdResponse", "EvaluationResultIdsResponse", "EvaluationResultQuery", "EvaluationResultResponse", "EvaluationResultsResponse", "EvaluationRun", "EvaluationRunCreate", "EvaluationRunData", "EvaluationRunDataConcurrency", "EvaluationRunDataMapping", "EvaluationRunDataMappingColumn", "EvaluationRunDataMappingStep", "EvaluationRunDataStep", "EvaluationRunDataStepInput", "EvaluationRunDataStepOrigin", "EvaluationRunDataStepType", "EvaluationRunEdit", "EvaluationRunFlags", "EvaluationRunIdResponse", "EvaluationRunIdsRequest", "EvaluationRunIdsResponse", "EvaluationRunQuery", "EvaluationRunQueryFlags", "EvaluationRunResponse", "EvaluationRunsResponse", "EvaluationScenario", "EvaluationScenarioCreate", "EvaluationScenarioEdit", "EvaluationScenarioIdResponse", "EvaluationScenarioIdsResponse", "EvaluationScenarioQuery", "EvaluationScenarioResponse", "EvaluationScenariosResponse", "EvaluationStatus", "Evaluator", "EvaluatorArtifactFlags", "EvaluatorArtifactQueryFlags", "EvaluatorCatalogPreset", "EvaluatorCatalogPresetResponse", "EvaluatorCatalogPresetsResponse", "EvaluatorCatalogTemplate", "EvaluatorCatalogTemplateResponse", "EvaluatorCatalogTemplatesResponse", "EvaluatorCatalogType", "EvaluatorCatalogTypesResponse", "EvaluatorCreate", "EvaluatorEdit", "EvaluatorFlags", "EvaluatorFork", "EvaluatorQuery", "EvaluatorResponse", "EvaluatorRevision", "EvaluatorRevisionCommit", "EvaluatorRevisionCreate", "EvaluatorRevisionDataInput", "EvaluatorRevisionDataInputHeadersValue", "EvaluatorRevisionDataInputRuntime", "EvaluatorRevisionDataOutput", "EvaluatorRevisionDataOutputHeadersValue", "EvaluatorRevisionDataOutputRuntime", "EvaluatorRevisionEdit", "EvaluatorRevisionFlags", "EvaluatorRevisionFork", "EvaluatorRevisionQuery", "EvaluatorRevisionQueryFlags", "EvaluatorRevisionResolveResponse", "EvaluatorRevisionResponse", "EvaluatorRevisionsLog", "EvaluatorRevisionsResponse", "EvaluatorTemplate", "EvaluatorTemplatesResponse", "EvaluatorVariant", "EvaluatorVariantCreate", "EvaluatorVariantEdit", "EvaluatorVariantFlags", "EvaluatorVariantFork", "EvaluatorVariantResponse", "EvaluatorVariantsResponse", "EvaluatorsResponse", "Event", "EventQuery", "EventType", "EventsQueryResponse", "ExistenceOperator", "FetchLegacyAnalyticsRequestNewest", "FetchLegacyAnalyticsRequestOldest", "FetchSimpleTestsetToFileRequestFileType", "FetchTestsetRevisionToFileRequestFileType", "FilteringInput", "FilteringInputConditionsItem", "FilteringOutput", "FilteringOutputConditionsItem", "Focus", "Folder", "FolderCreate", "FolderEdit", "FolderIdResponse", "FolderKind", "FolderQuery", "FolderQueryKinds", "FolderResponse", "FoldersResponse", "Format", "Formatting", typing.Any, typing.Any, "Header", "HttpValidationError", "InviteRequest", "Invocation", "InvocationCreate", "InvocationCreateLinks", "InvocationEdit", "InvocationEditLinks", "InvocationLinkResponse", "InvocationLinks", "InvocationQuery", "InvocationQueryLinks", "InvocationResponse", "InvocationsResponse", "JsonSchemasInput", "JsonSchemasOutput", typing.Any, typing.Any, "LegacyLifecycleDto", "ListApiKeysResponse", "ListOperator", "ListOptions", "LogicalOperator", "MetricSpec", "MetricType", "MetricsBucket", "NumericOperator", "OTelEventInput", "OTelEventInputTimestamp", "OTelEventOutput", "OTelEventOutputTimestamp", "OTelHashInput", "OTelHashOutput", "OTelLinkInput", "OTelLinkOutput", "OTelLinksResponse", "OTelReferenceInput", "OTelReferenceOutput", "OTelSpanKind", "OTelStatusCode", "OTelTracingRequest", "OTelTracingResponse", "OldAnalyticsResponse", "OrganizationDetails", "OrganizationDomainResponse", "OrganizationProviderResponse", "OrganizationUpdate", "OssSrcModelsApiOrganizationModelsOrganization", "Permission", "ProjectsResponse", "QueriesResponse", "Query", "QueryApplicationVariantsRequestOrder", "QueryCreate", "QueryEdit", "QueryEnvironmentRevisionsRequestOrder", "QueryEnvironmentVariantsRequestOrder", "QueryEnvironmentsRequestOrder", "QueryEvaluatorVariantsRequestOrder", "QueryFlags", "QueryQueriesRequestOrder", "QueryQueryFlags", "QueryResponse", "QueryRevision", "QueryRevisionCommit", "QueryRevisionCreate", "QueryRevisionDataInput", "QueryRevisionDataOutput", "QueryRevisionEdit", "QueryRevisionQuery", "QueryRevisionResponse", "QueryRevisionsLog", "QueryRevisionsResponse", "QuerySpansAnalyticsRequestNewest", "QuerySpansAnalyticsRequestOldest", "QueryVariant", "QueryVariantCreate", "QueryVariantEdit", "QueryVariantQuery", "QueryVariantResponse", "QueryVariantsResponse", "QueryWorkflowRevisionsRequestOrder", "QueryWorkflowVariantsRequestOrder", "QueryWorkflowsRequestOrder", "Reference", "ReferenceRequestModelInput", "ReferenceRequestModelOutput", "RequestType", "ResolutionInfo", "RetrievalInfo", "RevisionFork", "SecretDto", "SecretDtoData", "SecretKind", "SecretResponseDto", "SecretResponseDtoData", "SessionIdsResponse", "SimpleApplication", "SimpleApplicationCreate", "SimpleApplicationDataInput", "SimpleApplicationDataInputHeadersValue", "SimpleApplicationDataInputRuntime", "SimpleApplicationDataOutput", "SimpleApplicationDataOutputHeadersValue", "SimpleApplicationDataOutputRuntime", "SimpleApplicationEdit", "SimpleApplicationFlags", "SimpleApplicationQuery", "SimpleApplicationQueryFlags", "SimpleApplicationResponse", "SimpleApplicationsResponse", "SimpleEnvironment", "SimpleEnvironmentCreate", "SimpleEnvironmentEdit", "SimpleEnvironmentQuery", "SimpleEnvironmentResponse", "SimpleEnvironmentsResponse", "SimpleEvaluation", "SimpleEvaluationCreate", "SimpleEvaluationData", "SimpleEvaluationDataApplicationSteps", "SimpleEvaluationDataApplicationStepsOneValue", "SimpleEvaluationDataEvaluatorSteps", "SimpleEvaluationDataEvaluatorStepsOneValue", "SimpleEvaluationDataQuerySteps", "SimpleEvaluationDataQueryStepsOneValue", "SimpleEvaluationDataTestsetSteps", "SimpleEvaluationDataTestsetStepsOneValue", "SimpleEvaluationEdit", "SimpleEvaluationIdResponse", "SimpleEvaluationQuery", "SimpleEvaluationResponse", "SimpleEvaluationsResponse", "SimpleEvaluator", "SimpleEvaluatorCreate", "SimpleEvaluatorDataInput", "SimpleEvaluatorDataInputHeadersValue", "SimpleEvaluatorDataInputRuntime", "SimpleEvaluatorDataOutput", "SimpleEvaluatorDataOutputHeadersValue", "SimpleEvaluatorDataOutputRuntime", "SimpleEvaluatorEdit", "SimpleEvaluatorFlags", "SimpleEvaluatorQuery", "SimpleEvaluatorQueryFlags", "SimpleEvaluatorResponse", "SimpleEvaluatorsResponse", "SimpleQueriesResponse", "SimpleQuery", "SimpleQueryCreate", "SimpleQueryEdit", "SimpleQueryQuery", "SimpleQueryResponse", "SimpleQueue", "SimpleQueueCreate", "SimpleQueueData", "SimpleQueueDataEvaluators", "SimpleQueueDataEvaluatorsOneValue", "SimpleQueueIdResponse", "SimpleQueueKind", "SimpleQueueQuery", "SimpleQueueResponse", "SimpleQueueScenariosQuery", "SimpleQueueScenariosResponse", "SimpleQueueSettings", "SimpleQueuesResponse", "SimpleTestset", "SimpleTestsetCreate", "SimpleTestsetEdit", "SimpleTestsetQuery", "SimpleTestsetResponse", "SimpleTestsetsResponse", "SimpleTrace", "SimpleTraceChannel", "SimpleTraceCreate", "SimpleTraceCreateLinks", "SimpleTraceEdit", "SimpleTraceEditLinks", "SimpleTraceKind", "SimpleTraceLinkResponse", "SimpleTraceLinks", "SimpleTraceOrigin", "SimpleTraceQuery", "SimpleTraceQueryLinks", "SimpleTraceReferences", "SimpleTraceResponse", "SimpleTracesResponse", "SimpleWorkflow", "SimpleWorkflowCreate", "SimpleWorkflowDataInput", "SimpleWorkflowDataInputHeadersValue", "SimpleWorkflowDataInputRuntime", "SimpleWorkflowDataOutput", "SimpleWorkflowDataOutputHeadersValue", "SimpleWorkflowDataOutputRuntime", "SimpleWorkflowEdit", "SimpleWorkflowFlags", "SimpleWorkflowQuery", "SimpleWorkflowQueryFlags", "SimpleWorkflowResponse", "SimpleWorkflowsResponse", "SpanInput", "SpanInputEndTime", "SpanInputStartTime", "SpanOutput", "SpanOutputEndTime", "SpanOutputStartTime", "SpanResponse", "SpanType", "SpansNodeInput", "SpansNodeInputEndTime", "SpansNodeInputSpansValue", "SpansNodeInputStartTime", "SpansNodeOutput", "SpansNodeOutputEndTime", "SpansNodeOutputSpansValue", "SpansNodeOutputStartTime", "SpansResponse", "SpansTreeInput", "SpansTreeInputSpansValue", "SpansTreeOutput", "SpansTreeOutputSpansValue", "SsoProviderDto", "SsoProviderInfo", "SsoProviderSettingsDto", "SsoProviders", "StandardProviderDto", "StandardProviderKind", "StandardProviderSettingsDto", "Status", "StringOperator", "TestcaseInput", "TestcaseOutput", "TestcaseResponse", "TestcasesResponse", "Testset", "TestsetCreate", "TestsetEdit", "TestsetFlags", "TestsetQuery", "TestsetResponse", "TestsetRevision", "TestsetRevisionCommit", "TestsetRevisionCreate", "TestsetRevisionDataInput", "TestsetRevisionDataOutput", "TestsetRevisionDelta", "TestsetRevisionDeltaColumns", "TestsetRevisionDeltaRows", "TestsetRevisionEdit", "TestsetRevisionQuery", "TestsetRevisionResponse", "TestsetRevisionsLog", "TestsetRevisionsResponse", "TestsetVariant", "TestsetVariantCreate", "TestsetVariantEdit", "TestsetVariantQuery", "TestsetVariantResponse", "TestsetVariantsResponse", "TestsetsResponse", "TextOptions", "ToolAuthScheme", "ToolCallData", "ToolCallFunction", "ToolCallResponse", "ToolCatalogAction", "ToolCatalogActionDetails", "ToolCatalogActionResponse", "ToolCatalogActionResponseAction", "ToolCatalogActionsResponse", "ToolCatalogActionsResponseActionsItem", "ToolCatalogIntegration", "ToolCatalogIntegrationDetails", "ToolCatalogIntegrationResponse", "ToolCatalogIntegrationResponseIntegration", "ToolCatalogIntegrationsResponse", "ToolCatalogIntegrationsResponseIntegrationsItem", "ToolCatalogProvider", "ToolCatalogProviderDetails", "ToolCatalogProviderResponse", "ToolCatalogProviderResponseProvider", "ToolCatalogProvidersResponse", "ToolCatalogProvidersResponseProvidersItem", "ToolConnection", "ToolConnectionCreate", "ToolConnectionCreateData", "ToolConnectionResponse", "ToolConnectionStatus", "ToolConnectionsResponse", "ToolProviderKind", "ToolResult", "ToolResultData", "TraceIdResponse", "TraceIdsResponse", "TraceInput", "TraceInputSpansValue", "TraceOutput", "TraceOutputSpansValue", "TraceRequest", "TraceResponse", "TraceType", "TracesRequest", "TracesResponse", "TracingQuery", "UnprocessableEntityError", "UserIdsResponse", "ValidationError", "ValidationErrorLocItem", "VariantFork", "WebhookDeliveriesResponse", "WebhookDelivery", "WebhookDeliveryCreate", "WebhookDeliveryData", "WebhookDeliveryQuery", "WebhookDeliveryResponse", "WebhookDeliveryResponseInfo", "WebhookEventType", "WebhookProviderDto", "WebhookProviderSettingsDto", "WebhookSubscription", "WebhookSubscriptionCreate", "WebhookSubscriptionData", "WebhookSubscriptionDataAuthMode", "WebhookSubscriptionEdit", "WebhookSubscriptionQuery", "WebhookSubscriptionResponse", "WebhookSubscriptionTestRequestSubscription", "WebhookSubscriptionsResponse", "Windowing", "WindowingOrder", "Workflow", "WorkflowArtifactFlags", "WorkflowCatalogFlags", "WorkflowCatalogPreset", "WorkflowCatalogPresetResponse", "WorkflowCatalogPresetsResponse", "WorkflowCatalogTemplate", "WorkflowCatalogTemplateResponse", "WorkflowCatalogTemplatesResponse", "WorkflowCatalogType", "WorkflowCatalogTypeResponse", "WorkflowCatalogTypesResponse", "WorkflowCreate", "WorkflowEdit", "WorkflowFlags", "WorkflowFork", "WorkflowResponse", "WorkflowRevisionCommit", "WorkflowRevisionCreate", "WorkflowRevisionDataInput", "WorkflowRevisionDataInputHeadersValue", "WorkflowRevisionDataInputRuntime", "WorkflowRevisionDataOutput", "WorkflowRevisionDataOutputHeadersValue", "WorkflowRevisionDataOutputRuntime", "WorkflowRevisionEdit", "WorkflowRevisionFlags", "WorkflowRevisionFork", "WorkflowRevisionInput", "WorkflowRevisionOutput", "WorkflowRevisionResolveResponse", "WorkflowRevisionResponse", "WorkflowRevisionsLog", "WorkflowRevisionsResponse", "WorkflowVariant", "WorkflowVariantCreate", "WorkflowVariantEdit", "WorkflowVariantFlags", "WorkflowVariantFork", "WorkflowVariantResponse", "WorkflowVariantsResponse", "WorkflowsResponse", "Workspace", "WorkspaceMemberResponse", "WorkspacePermission", "WorkspaceResponse", "access", "annotations", "applications", "billing", "environments", "evaluations", "evaluators", "events", "folders", "invocations", "keys", "legacy", "organizations", "projects", "queries", "secrets", "status", "testcases", "testsets", "tools", "traces", "users", "webhooks", "workflows", "workspaces"] diff --git a/clients/python/agenta_client/access/client.py b/clients/python/agenta_client/access/client.py index fbf72584bc..57edb177e9 100644 --- a/clients/python/agenta_client/access/client.py +++ b/clients/python/agenta_client/access/client.py @@ -63,7 +63,7 @@ def fetch_access_roles(self, *, request_options: typing.Optional[RequestOptions] verbatim from access-controls, including the `"*"` wildcard for `owner` — callers that need to render the full permission list should expand the wildcard themselves (see - `ee.src.services.converters._expand_permissions`). + `ee.src.services.db_manager_ee._expand_permissions`). Parameters ---------- @@ -229,7 +229,7 @@ def sso_callback_redirect(self, organization_slug: str, provider_slug: str, *, r _response = self._raw_client.sso_callback_redirect(organization_slug, provider_slug, request_options=request_options) return _response.data - def verify_permissions(self, *, action: typing.Optional[str] = None, scope_type: typing.Optional[str] = None, scope_id: typing.Optional[str] = None, resource_type: typing.Optional[str] = None, resource_id: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None) -> typing.Any: + def check_permissions(self, *, action: typing.Optional[str] = None, scope_type: typing.Optional[str] = None, scope_id: typing.Optional[str] = None, resource_type: typing.Optional[str] = None, resource_id: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None) -> typing.Any: """ Parameters ---------- @@ -258,9 +258,9 @@ def verify_permissions(self, *, action: typing.Optional[str] = None, scope_type: client = AgentaApi( api_key="YOUR_API_KEY", ) - client.access.verify_permissions() + client.access.check_permissions() """ - _response = self._raw_client.verify_permissions(action=action, scope_type=scope_type, scope_id=scope_id, resource_type=resource_type, resource_id=resource_id, request_options=request_options) + _response = self._raw_client.check_permissions(action=action, scope_type=scope_type, scope_id=scope_id, resource_type=resource_type, resource_id=resource_id, request_options=request_options) return _response.data class AsyncAccessClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): @@ -324,7 +324,7 @@ async def fetch_access_roles(self, *, request_options: typing.Optional[RequestOp verbatim from access-controls, including the `"*"` wildcard for `owner` — callers that need to render the full permission list should expand the wildcard themselves (see - `ee.src.services.converters._expand_permissions`). + `ee.src.services.db_manager_ee._expand_permissions`). Parameters ---------- @@ -530,7 +530,7 @@ async def main() -> None: _response = await self._raw_client.sso_callback_redirect(organization_slug, provider_slug, request_options=request_options) return _response.data - async def verify_permissions(self, *, action: typing.Optional[str] = None, scope_type: typing.Optional[str] = None, scope_id: typing.Optional[str] = None, resource_type: typing.Optional[str] = None, resource_id: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None) -> typing.Any: + async def check_permissions(self, *, action: typing.Optional[str] = None, scope_type: typing.Optional[str] = None, scope_id: typing.Optional[str] = None, resource_type: typing.Optional[str] = None, resource_id: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None) -> typing.Any: """ Parameters ---------- @@ -564,10 +564,10 @@ async def verify_permissions(self, *, action: typing.Optional[str] = None, scope async def main() -> None: - await client.access.verify_permissions() + await client.access.check_permissions() asyncio.run(main()) """ - _response = await self._raw_client.verify_permissions(action=action, scope_type=scope_type, scope_id=scope_id, resource_type=resource_type, resource_id=resource_id, request_options=request_options) + _response = await self._raw_client.check_permissions(action=action, scope_type=scope_type, scope_id=scope_id, resource_type=resource_type, resource_id=resource_id, request_options=request_options) return _response.data diff --git a/clients/python/agenta_client/access/raw_client.py b/clients/python/agenta_client/access/raw_client.py index 8a2a1e32d3..deb8f88f23 100644 --- a/clients/python/agenta_client/access/raw_client.py +++ b/clients/python/agenta_client/access/raw_client.py @@ -64,7 +64,7 @@ def fetch_access_roles(self, *, request_options: typing.Optional[RequestOptions] verbatim from access-controls, including the `"*"` wildcard for `owner` — callers that need to render the full permission list should expand the wildcard themselves (see - `ee.src.services.converters._expand_permissions`). + `ee.src.services.db_manager_ee._expand_permissions`). Parameters ---------- @@ -305,7 +305,7 @@ def sso_callback_redirect(self, organization_slug: str, provider_slug: str, *, r raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def verify_permissions(self, *, action: typing.Optional[str] = None, scope_type: typing.Optional[str] = None, scope_id: typing.Optional[str] = None, resource_type: typing.Optional[str] = None, resource_id: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[typing.Any]: + def check_permissions(self, *, action: typing.Optional[str] = None, scope_type: typing.Optional[str] = None, scope_id: typing.Optional[str] = None, resource_type: typing.Optional[str] = None, resource_id: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[typing.Any]: """ Parameters ---------- @@ -328,7 +328,7 @@ def verify_permissions(self, *, action: typing.Optional[str] = None, scope_type: Successful Response """ _response = self._client_wrapper.httpx_client.request( - "permissions/verify",method="GET", + "access/permissions/check",method="GET", params={"action": action, "scope_type": scope_type, "scope_id": scope_id, "resource_type": resource_type, "resource_id": resource_id, } , request_options=request_options,) @@ -405,7 +405,7 @@ async def fetch_access_roles(self, *, request_options: typing.Optional[RequestOp verbatim from access-controls, including the `"*"` wildcard for `owner` — callers that need to render the full permission list should expand the wildcard themselves (see - `ee.src.services.converters._expand_permissions`). + `ee.src.services.db_manager_ee._expand_permissions`). Parameters ---------- @@ -646,7 +646,7 @@ async def sso_callback_redirect(self, organization_slug: str, provider_slug: str raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def verify_permissions(self, *, action: typing.Optional[str] = None, scope_type: typing.Optional[str] = None, scope_id: typing.Optional[str] = None, resource_type: typing.Optional[str] = None, resource_id: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[typing.Any]: + async def check_permissions(self, *, action: typing.Optional[str] = None, scope_type: typing.Optional[str] = None, scope_id: typing.Optional[str] = None, resource_type: typing.Optional[str] = None, resource_id: typing.Optional[str] = None, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[typing.Any]: """ Parameters ---------- @@ -669,7 +669,7 @@ async def verify_permissions(self, *, action: typing.Optional[str] = None, scope Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "permissions/verify",method="GET", + "access/permissions/check",method="GET", params={"action": action, "scope_type": scope_type, "scope_id": scope_id, "resource_type": resource_type, "resource_id": resource_id, } , request_options=request_options,) diff --git a/clients/python/agenta_client/client.py b/clients/python/agenta_client/client.py index 6b07f0438b..a451eb5fc8 100644 --- a/clients/python/agenta_client/client.py +++ b/clients/python/agenta_client/client.py @@ -80,13 +80,13 @@ def __init__(self, *, base_url: typing.Optional[str] = None, environment: Agenta , timeout=_defaulted_timeout) self._access: typing.Optional[AccessClient] = None self._billing: typing.Optional[BillingClient] = None + self._events: typing.Optional[EventsClient] = None self._organizations: typing.Optional[OrganizationsClient] = None self._workspaces: typing.Optional[WorkspacesClient] = None self._secrets: typing.Optional[SecretsClient] = None self._webhooks: typing.Optional[WebhooksClient] = None self._legacy: typing.Optional[LegacyClient] = None self._traces: typing.Optional[TracesClient] = None - self._events: typing.Optional[EventsClient] = None self._invocations: typing.Optional[InvocationsClient] = None self._annotations: typing.Optional[AnnotationsClient] = None self._testcases: typing.Optional[TestcasesClient] = None @@ -118,6 +118,13 @@ def billing(self): self._billing = BillingClient(client_wrapper=self._client_wrapper) return self._billing + @property + def events(self): + if self._events is None: + from .events.client import EventsClient # noqa: E402 + self._events = EventsClient(client_wrapper=self._client_wrapper) + return self._events + @property def organizations(self): if self._organizations is None: @@ -160,13 +167,6 @@ def traces(self): self._traces = TracesClient(client_wrapper=self._client_wrapper) return self._traces - @property - def events(self): - if self._events is None: - from .events.client import EventsClient # noqa: E402 - self._events = EventsClient(client_wrapper=self._client_wrapper) - return self._events - @property def invocations(self): if self._invocations is None: @@ -324,13 +324,13 @@ def __init__(self, *, base_url: typing.Optional[str] = None, environment: Agenta , timeout=_defaulted_timeout) self._access: typing.Optional[AsyncAccessClient] = None self._billing: typing.Optional[AsyncBillingClient] = None + self._events: typing.Optional[AsyncEventsClient] = None self._organizations: typing.Optional[AsyncOrganizationsClient] = None self._workspaces: typing.Optional[AsyncWorkspacesClient] = None self._secrets: typing.Optional[AsyncSecretsClient] = None self._webhooks: typing.Optional[AsyncWebhooksClient] = None self._legacy: typing.Optional[AsyncLegacyClient] = None self._traces: typing.Optional[AsyncTracesClient] = None - self._events: typing.Optional[AsyncEventsClient] = None self._invocations: typing.Optional[AsyncInvocationsClient] = None self._annotations: typing.Optional[AsyncAnnotationsClient] = None self._testcases: typing.Optional[AsyncTestcasesClient] = None @@ -362,6 +362,13 @@ def billing(self): self._billing = AsyncBillingClient(client_wrapper=self._client_wrapper) return self._billing + @property + def events(self): + if self._events is None: + from .events.client import AsyncEventsClient # noqa: E402 + self._events = AsyncEventsClient(client_wrapper=self._client_wrapper) + return self._events + @property def organizations(self): if self._organizations is None: @@ -404,13 +411,6 @@ def traces(self): self._traces = AsyncTracesClient(client_wrapper=self._client_wrapper) return self._traces - @property - def events(self): - if self._events is None: - from .events.client import AsyncEventsClient # noqa: E402 - self._events = AsyncEventsClient(client_wrapper=self._client_wrapper) - return self._events - @property def invocations(self): if self._invocations is None: diff --git a/clients/python/agenta_client/evaluations/client.py b/clients/python/agenta_client/evaluations/client.py index 5c5e1be270..b22620b6ce 100644 --- a/clients/python/agenta_client/evaluations/client.py +++ b/clients/python/agenta_client/evaluations/client.py @@ -1,11 +1,11 @@ # This file was auto-generated by Fern from our API Definition. +import datetime as dt import typing from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper from ..core.request_options import RequestOptions from ..types.evaluation_metrics_create import EvaluationMetricsCreate -from ..types.evaluation_metrics_edit import EvaluationMetricsEdit from ..types.evaluation_metrics_ids_response import EvaluationMetricsIdsResponse from ..types.evaluation_metrics_query import EvaluationMetricsQuery from ..types.evaluation_metrics_refresh import EvaluationMetricsRefresh @@ -19,13 +19,13 @@ from ..types.evaluation_queue_scenarios_query import EvaluationQueueScenariosQuery from ..types.evaluation_queues_response import EvaluationQueuesResponse from ..types.evaluation_result_create import EvaluationResultCreate -from ..types.evaluation_result_edit import EvaluationResultEdit from ..types.evaluation_result_id_response import EvaluationResultIdResponse from ..types.evaluation_result_ids_response import EvaluationResultIdsResponse from ..types.evaluation_result_query import EvaluationResultQuery from ..types.evaluation_result_response import EvaluationResultResponse from ..types.evaluation_results_response import EvaluationResultsResponse from ..types.evaluation_run_create import EvaluationRunCreate +from ..types.evaluation_run_data_step import EvaluationRunDataStep from ..types.evaluation_run_edit import EvaluationRunEdit from ..types.evaluation_run_id_response import EvaluationRunIdResponse from ..types.evaluation_run_ids_response import EvaluationRunIdsResponse @@ -358,14 +358,12 @@ def close_run(self, run_id: str, *, status: typing.Optional[EvaluationStatus] = _response = self._raw_client.close_run(run_id, status=status, request_options=request_options) return _response.data - def close_run_with_status(self, run_id: str, status: typing.Optional[EvaluationStatus], *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: + def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: """ Parameters ---------- run_id : str - status : typing.Optional[EvaluationStatus] - request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -381,15 +379,14 @@ def close_run_with_status(self, run_id: str, status: typing.Optional[EvaluationS client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.close_run_with_status( + client.evaluations.open_run( run_id="run_id", - status="pending", ) """ - _response = self._raw_client.close_run_with_status(run_id, status, request_options=request_options) + _response = self._raw_client.open_run(run_id, request_options=request_options) return _response.data - def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: + def fetch_default_queue(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueueResponse: """ Parameters ---------- @@ -400,7 +397,7 @@ def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptio Returns ------- - EvaluationRunResponse + EvaluationQueueResponse Successful Response Examples @@ -410,11 +407,11 @@ def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptio client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.open_run( + client.evaluations.fetch_default_queue( run_id="run_id", ) """ - _response = self._raw_client.open_run(run_id, request_options=request_options) + _response = self._raw_client.fetch_default_queue(run_id, request_options=request_options) return _response.data def create_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: @@ -620,7 +617,7 @@ def edit_scenario(self, scenario_id: str, *, scenario: EvaluationScenarioEdit, r _response = self._raw_client.edit_scenario(scenario_id, scenario=scenario, request_options=request_options) return _response.data - def create_results(self, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: + def set_results(self, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: """ Parameters ---------- @@ -641,7 +638,7 @@ def create_results(self, *, results: typing.Sequence[EvaluationResultCreate], re client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.create_results( + client.evaluations.set_results( results=[ EvaluationResultCreate( step_key="step_key", @@ -651,7 +648,7 @@ def create_results(self, *, results: typing.Sequence[EvaluationResultCreate], re ], ) """ - _response = self._raw_client.create_results(results=results, request_options=request_options) + _response = self._raw_client.set_results(results=results, request_options=request_options) return _response.data def delete_results(self, *, result_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultIdsResponse: @@ -682,34 +679,6 @@ def delete_results(self, *, result_ids: typing.Sequence[str], request_options: t _response = self._raw_client.delete_results(result_ids=result_ids, request_options=request_options) return _response.data - def edit_results(self, *, results: typing.Sequence[EvaluationResultEdit], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: - """ - Parameters - ---------- - results : typing.Sequence[EvaluationResultEdit] - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - EvaluationResultsResponse - Successful Response - - Examples - -------- - from agenta import AgentaApi, EvaluationResultEdit - - client = AgentaApi( - api_key="YOUR_API_KEY", - ) - client.evaluations.edit_results( - results=[EvaluationResultEdit()], - ) - """ - _response = self._raw_client.edit_results(results=results, request_options=request_options) - return _response.data - def query_results(self, *, result: typing.Optional[EvaluationResultQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: """ Parameters @@ -794,37 +763,6 @@ def delete_result(self, result_id: str, *, request_options: typing.Optional[Requ _response = self._raw_client.delete_result(result_id, request_options=request_options) return _response.data - def edit_result(self, result_id: str, *, result: EvaluationResultEdit, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultResponse: - """ - Parameters - ---------- - result_id : str - - result : EvaluationResultEdit - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - EvaluationResultResponse - Successful Response - - Examples - -------- - from agenta import AgentaApi, EvaluationResultEdit - - client = AgentaApi( - api_key="YOUR_API_KEY", - ) - client.evaluations.edit_result( - result_id="result_id", - result=EvaluationResultEdit(), - ) - """ - _response = self._raw_client.edit_result(result_id, result=result, request_options=request_options) - return _response.data - def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: """ Parameters @@ -853,7 +791,7 @@ def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_options: _response = self._raw_client.refresh_metrics(metrics=metrics, request_options=request_options) return _response.data - def create_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: + def set_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: """ Parameters ---------- @@ -874,7 +812,7 @@ def create_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], r client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.create_metrics( + client.evaluations.set_metrics( metrics=[ EvaluationMetricsCreate( run_id="run_id", @@ -882,7 +820,7 @@ def create_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], r ], ) """ - _response = self._raw_client.create_metrics(metrics=metrics, request_options=request_options) + _response = self._raw_client.set_metrics(metrics=metrics, request_options=request_options) return _response.data def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsIdsResponse: @@ -913,11 +851,13 @@ def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_options: _response = self._raw_client.delete_metrics(metrics_ids=metrics_ids, request_options=request_options) return _response.data - def edit_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsEdit], request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: + def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: """ Parameters ---------- - metrics : typing.Sequence[EvaluationMetricsEdit] + metrics : typing.Optional[EvaluationMetricsQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -929,25 +869,21 @@ def edit_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsEdit], reque Examples -------- - from agenta import AgentaApi, EvaluationMetricsEdit + from agenta import AgentaApi client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.edit_metrics( - metrics=[EvaluationMetricsEdit()], - ) + client.evaluations.query_metrics() """ - _response = self._raw_client.edit_metrics(metrics=metrics, request_options=request_options) + _response = self._raw_client.query_metrics(metrics=metrics, windowing=windowing, request_options=request_options) return _response.data - def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: + def fetch_metric(self, metrics_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: """ Parameters ---------- - metrics : typing.Optional[EvaluationMetricsQuery] - - windowing : typing.Optional[Windowing] + metrics_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -964,9 +900,39 @@ def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery] = OM client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.query_metrics() + client.evaluations.fetch_metric( + metrics_id="metrics_id", + ) """ - _response = self._raw_client.query_metrics(metrics=metrics, windowing=windowing, request_options=request_options) + _response = self._raw_client.fetch_metric(metrics_id, request_options=request_options) + return _response.data + + def delete_metric(self, metrics_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsIdsResponse: + """ + Parameters + ---------- + metrics_id : str + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationMetricsIdsResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + ) + client.evaluations.delete_metric( + metrics_id="metrics_id", + ) + """ + _response = self._raw_client.delete_metric(metrics_id, request_options=request_options) return _response.data def create_queues(self, *, queues: typing.Sequence[EvaluationQueueCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueuesResponse: @@ -1234,18 +1200,20 @@ def create_simple_evaluation(self, *, evaluation: SimpleEvaluationCreate, reques _response = self._raw_client.create_simple_evaluation(evaluation=evaluation, request_options=request_options) return _response.data - def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: + def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEvaluationQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationsResponse: """ Parameters ---------- - evaluation_id : str + evaluation : typing.Optional[SimpleEvaluationQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationResponse + SimpleEvaluationsResponse Successful Response Examples @@ -1255,14 +1223,12 @@ def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: typing client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.fetch_simple_evaluation( - evaluation_id="evaluation_id", - ) + client.evaluations.query_simple_evaluations() """ - _response = self._raw_client.fetch_simple_evaluation(evaluation_id, request_options=request_options) + _response = self._raw_client.query_simple_evaluations(evaluation=evaluation, windowing=windowing, request_options=request_options) return _response.data - def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationIdResponse: + def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: """ Parameters ---------- @@ -1273,7 +1239,7 @@ def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typin Returns ------- - SimpleEvaluationIdResponse + SimpleEvaluationResponse Successful Response Examples @@ -1283,70 +1249,70 @@ def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typin client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.delete_simple_evaluation( + client.evaluations.fetch_simple_evaluation( evaluation_id="evaluation_id", ) """ - _response = self._raw_client.delete_simple_evaluation(evaluation_id, request_options=request_options) + _response = self._raw_client.fetch_simple_evaluation(evaluation_id, request_options=request_options) return _response.data - def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: SimpleEvaluationEdit, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: + def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationIdResponse: """ Parameters ---------- evaluation_id : str - evaluation : SimpleEvaluationEdit - request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationResponse + SimpleEvaluationIdResponse Successful Response Examples -------- - from agenta import AgentaApi, SimpleEvaluationEdit + from agenta import AgentaApi client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.edit_simple_evaluation( + client.evaluations.delete_simple_evaluation( evaluation_id="evaluation_id", - evaluation=SimpleEvaluationEdit(), ) """ - _response = self._raw_client.edit_simple_evaluation(evaluation_id, evaluation=evaluation, request_options=request_options) + _response = self._raw_client.delete_simple_evaluation(evaluation_id, request_options=request_options) return _response.data - def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEvaluationQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationsResponse: + def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: SimpleEvaluationEdit, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: """ Parameters ---------- - evaluation : typing.Optional[SimpleEvaluationQuery] + evaluation_id : str - windowing : typing.Optional[Windowing] + evaluation : SimpleEvaluationEdit request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationsResponse + SimpleEvaluationResponse Successful Response Examples -------- - from agenta import AgentaApi + from agenta import AgentaApi, SimpleEvaluationEdit client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.query_simple_evaluations() + client.evaluations.edit_simple_evaluation( + evaluation_id="evaluation_id", + evaluation=SimpleEvaluationEdit(), + ) """ - _response = self._raw_client.query_simple_evaluations(evaluation=evaluation, windowing=windowing, request_options=request_options) + _response = self._raw_client.edit_simple_evaluation(evaluation_id, evaluation=evaluation, request_options=request_options) return _response.data def start_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: @@ -1461,49 +1427,64 @@ def open_simple_evaluation(self, evaluation_id: str, *, request_options: typing. _response = self._raw_client.open_simple_evaluation(evaluation_id, request_options=request_options) return _response.data - def create_simple_queue(self, *, queue: SimpleQueueCreate, request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueResponse: + def populate_slice(self, evaluation_id: str, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: """ Parameters ---------- - queue : SimpleQueueCreate + evaluation_id : str + + results : typing.Sequence[EvaluationResultCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleQueueResponse + EvaluationResultsResponse Successful Response Examples -------- - from agenta import AgentaApi, SimpleQueueCreate + from agenta import AgentaApi, EvaluationResultCreate client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.create_simple_queue( - queue=SimpleQueueCreate(), + client.evaluations.populate_slice( + evaluation_id="evaluation_id", + results=[ + EvaluationResultCreate( + step_key="step_key", + scenario_id="scenario_id", + run_id="run_id", + ) + ], ) """ - _response = self._raw_client.create_simple_queue(queue=queue, request_options=request_options) + _response = self._raw_client.populate_slice(evaluation_id, results=results, request_options=request_options) return _response.data - def query_simple_queues(self, *, queue: typing.Optional[SimpleQueueQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> SimpleQueuesResponse: + def process_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, overwrite: typing.Optional[bool] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> typing.Any: """ Parameters ---------- - queue : typing.Optional[SimpleQueueQuery] + evaluation_id : str - windowing : typing.Optional[Windowing] + scenario_ids : typing.Optional[typing.Sequence[str]] + + step_keys : typing.Optional[typing.Sequence[str]] + + repeat_idxs : typing.Optional[typing.Sequence[int]] + + overwrite : typing.Optional[bool] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleQueuesResponse - Successful Response + typing.Any + Accepted. Examples -------- @@ -1512,23 +1493,31 @@ def query_simple_queues(self, *, queue: typing.Optional[SimpleQueueQuery] = OMIT client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.query_simple_queues() + client.evaluations.process_slice( + evaluation_id="evaluation_id", + ) """ - _response = self._raw_client.query_simple_queues(queue=queue, windowing=windowing, request_options=request_options) + _response = self._raw_client.process_slice(evaluation_id, scenario_ids=scenario_ids, step_keys=step_keys, repeat_idxs=repeat_idxs, overwrite=overwrite, request_options=request_options) return _response.data - def fetch_simple_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueResponse: + def probe_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: """ Parameters ---------- - queue_id : str + evaluation_id : str + + scenario_ids : typing.Optional[typing.Sequence[str]] + + step_keys : typing.Optional[typing.Sequence[str]] + + repeat_idxs : typing.Optional[typing.Sequence[int]] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleQueueResponse + EvaluationResultsResponse Successful Response Examples @@ -1538,32 +1527,31 @@ def fetch_simple_queue(self, queue_id: str, *, request_options: typing.Optional[ client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.fetch_simple_queue( - queue_id="queue_id", + client.evaluations.probe_slice( + evaluation_id="evaluation_id", ) """ - _response = self._raw_client.fetch_simple_queue(queue_id, request_options=request_options) + _response = self._raw_client.probe_slice(evaluation_id, scenario_ids=scenario_ids, step_keys=step_keys, repeat_idxs=repeat_idxs, request_options=request_options) return _response.data - def query_simple_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[SimpleQueueScenariosQuery] = OMIT, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueScenariosResponse: + def prune_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> None: """ Parameters ---------- - queue_id : str + evaluation_id : str - queue : typing.Optional[SimpleQueueScenariosQuery] + scenario_ids : typing.Optional[typing.Sequence[str]] - scenario : typing.Optional[EvaluationScenarioQuery] + step_keys : typing.Optional[typing.Sequence[str]] - windowing : typing.Optional[Windowing] + repeat_idxs : typing.Optional[typing.Sequence[int]] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleQueueScenariosResponse - Successful Response + None Examples -------- @@ -1572,27 +1560,29 @@ def query_simple_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[ client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.query_simple_queue_scenarios( - queue_id="queue_id", + client.evaluations.prune_slice( + evaluation_id="evaluation_id", ) """ - _response = self._raw_client.query_simple_queue_scenarios(queue_id, queue=queue, scenario=scenario, windowing=windowing, request_options=request_options) + _response = self._raw_client.prune_slice(evaluation_id, scenario_ids=scenario_ids, step_keys=step_keys, repeat_idxs=repeat_idxs, request_options=request_options) return _response.data - def add_simple_queue_traces(self, queue_id: str, *, trace_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueIdResponse: + def add_scenarios(self, evaluation_id: str, *, count: int, timestamp: typing.Optional[dt.datetime] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: """ Parameters ---------- - queue_id : str + evaluation_id : str - trace_ids : typing.Sequence[str] + count : int + + timestamp : typing.Optional[dt.datetime] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleQueueIdResponse + EvaluationScenariosResponse Successful Response Examples @@ -1602,29 +1592,28 @@ def add_simple_queue_traces(self, queue_id: str, *, trace_ids: typing.Sequence[s client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.add_simple_queue_traces( - queue_id="queue_id", - trace_ids=["trace_ids"], + client.evaluations.add_scenarios( + evaluation_id="evaluation_id", + count=1, ) """ - _response = self._raw_client.add_simple_queue_traces(queue_id, trace_ids=trace_ids, request_options=request_options) + _response = self._raw_client.add_scenarios(evaluation_id, count=count, timestamp=timestamp, request_options=request_options) return _response.data - def add_simple_queue_testcases(self, queue_id: str, *, testcase_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueIdResponse: + def remove_scenarios(self, evaluation_id: str, *, scenario_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> None: """ Parameters ---------- - queue_id : str + evaluation_id : str - testcase_ids : typing.Sequence[str] + scenario_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleQueueIdResponse - Successful Response + None Examples -------- @@ -1633,65 +1622,489 @@ def add_simple_queue_testcases(self, queue_id: str, *, testcase_ids: typing.Sequ client = AgentaApi( api_key="YOUR_API_KEY", ) - client.evaluations.add_simple_queue_testcases( - queue_id="queue_id", - testcase_ids=["testcase_ids"], + client.evaluations.remove_scenarios( + evaluation_id="evaluation_id", + scenario_ids=["scenario_ids"], ) """ - _response = self._raw_client.add_simple_queue_testcases(queue_id, testcase_ids=testcase_ids, request_options=request_options) + _response = self._raw_client.remove_scenarios(evaluation_id, scenario_ids=scenario_ids, request_options=request_options) return _response.data -class AsyncEvaluationsClient: - def __init__(self, *, client_wrapper: AsyncClientWrapper): - self._raw_client = AsyncRawEvaluationsClient(client_wrapper=client_wrapper) - - @property - def with_raw_response(self) -> AsyncRawEvaluationsClient: - """ - Retrieves a raw implementation of this client that returns raw responses. - - Returns - ------- - AsyncRawEvaluationsClient - """ - return self._raw_client - async def create_runs(self, *, runs: typing.Sequence[EvaluationRunCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunsResponse: + def add_steps(self, evaluation_id: str, *, steps: typing.Sequence[EvaluationRunDataStep], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: """ Parameters ---------- - runs : typing.Sequence[EvaluationRunCreate] + evaluation_id : str + + steps : typing.Sequence[EvaluationRunDataStep] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationRunsResponse + EvaluationRunResponse Successful Response Examples -------- - import asyncio - - from agenta import AsyncAgentaApi, EvaluationRunCreate + from agenta import AgentaApi, EvaluationRunDataStep, Reference - client = AsyncAgentaApi( + client = AgentaApi( api_key="YOUR_API_KEY", ) - - + client.evaluations.add_steps( + evaluation_id="evaluation_id", + steps=[ + EvaluationRunDataStep( + key="key", + type="input", + origin="custom", + references={"key": Reference()}, + ) + ], + ) + """ + _response = self._raw_client.add_steps(evaluation_id, steps=steps, request_options=request_options) + return _response.data + + def remove_steps(self, evaluation_id: str, *, step_keys: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: + """ + Parameters + ---------- + evaluation_id : str + + step_keys : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRunResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + ) + client.evaluations.remove_steps( + evaluation_id="evaluation_id", + step_keys=["step_keys"], + ) + """ + _response = self._raw_client.remove_steps(evaluation_id, step_keys=step_keys, request_options=request_options) + return _response.data + + def set_repeats(self, evaluation_id: str, *, repeats: int, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: + """ + Parameters + ---------- + evaluation_id : str + + repeats : int + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRunResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + ) + client.evaluations.set_repeats( + evaluation_id="evaluation_id", + repeats=1, + ) + """ + _response = self._raw_client.set_repeats(evaluation_id, repeats=repeats, request_options=request_options) + return _response.data + + def create_simple_queue(self, *, queue: SimpleQueueCreate, request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueResponse: + """ + Parameters + ---------- + queue : SimpleQueueCreate + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + SimpleQueueResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi, SimpleQueueCreate + + client = AgentaApi( + api_key="YOUR_API_KEY", + ) + client.evaluations.create_simple_queue( + queue=SimpleQueueCreate(), + ) + """ + _response = self._raw_client.create_simple_queue(queue=queue, request_options=request_options) + return _response.data + + def query_simple_queues(self, *, queue: typing.Optional[SimpleQueueQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> SimpleQueuesResponse: + """ + Parameters + ---------- + queue : typing.Optional[SimpleQueueQuery] + + windowing : typing.Optional[Windowing] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + SimpleQueuesResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + ) + client.evaluations.query_simple_queues() + """ + _response = self._raw_client.query_simple_queues(queue=queue, windowing=windowing, request_options=request_options) + return _response.data + + def fetch_simple_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueResponse: + """ + Parameters + ---------- + queue_id : str + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + SimpleQueueResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + ) + client.evaluations.fetch_simple_queue( + queue_id="queue_id", + ) + """ + _response = self._raw_client.fetch_simple_queue(queue_id, request_options=request_options) + return _response.data + + def query_simple_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[SimpleQueueScenariosQuery] = OMIT, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueScenariosResponse: + """ + Parameters + ---------- + queue_id : str + + queue : typing.Optional[SimpleQueueScenariosQuery] + + scenario : typing.Optional[EvaluationScenarioQuery] + + windowing : typing.Optional[Windowing] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + SimpleQueueScenariosResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + ) + client.evaluations.query_simple_queue_scenarios( + queue_id="queue_id", + ) + """ + _response = self._raw_client.query_simple_queue_scenarios(queue_id, queue=queue, scenario=scenario, windowing=windowing, request_options=request_options) + return _response.data + + def add_simple_queue_traces(self, queue_id: str, *, trace_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueIdResponse: + """ + Parameters + ---------- + queue_id : str + + trace_ids : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + SimpleQueueIdResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + ) + client.evaluations.add_simple_queue_traces( + queue_id="queue_id", + trace_ids=["trace_ids"], + ) + """ + _response = self._raw_client.add_simple_queue_traces(queue_id, trace_ids=trace_ids, request_options=request_options) + return _response.data + + def add_simple_queue_testcases(self, queue_id: str, *, testcase_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueIdResponse: + """ + Parameters + ---------- + queue_id : str + + testcase_ids : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + SimpleQueueIdResponse + Successful Response + + Examples + -------- + from agenta import AgentaApi + + client = AgentaApi( + api_key="YOUR_API_KEY", + ) + client.evaluations.add_simple_queue_testcases( + queue_id="queue_id", + testcase_ids=["testcase_ids"], + ) + """ + _response = self._raw_client.add_simple_queue_testcases(queue_id, testcase_ids=testcase_ids, request_options=request_options) + return _response.data +class AsyncEvaluationsClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._raw_client = AsyncRawEvaluationsClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> AsyncRawEvaluationsClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + AsyncRawEvaluationsClient + """ + return self._raw_client + + async def create_runs(self, *, runs: typing.Sequence[EvaluationRunCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunsResponse: + """ + Parameters + ---------- + runs : typing.Sequence[EvaluationRunCreate] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRunsResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi, EvaluationRunCreate + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + ) + + + async def main() -> None: + await client.evaluations.create_runs( + runs=[EvaluationRunCreate()], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.create_runs(runs=runs, request_options=request_options) + return _response.data + + async def delete_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunIdsResponse: + """ + Parameters + ---------- + run_ids : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRunIdsResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + ) + + + async def main() -> None: + await client.evaluations.delete_runs( + run_ids=["run_ids"], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.delete_runs(run_ids=run_ids, request_options=request_options) + return _response.data + + async def edit_runs(self, *, runs: typing.Sequence[EvaluationRunEdit], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunsResponse: + """ + Parameters + ---------- + runs : typing.Sequence[EvaluationRunEdit] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRunsResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi, EvaluationRunEdit + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + ) + + + async def main() -> None: + await client.evaluations.edit_runs( + runs=[EvaluationRunEdit()], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.edit_runs(runs=runs, request_options=request_options) + return _response.data + + async def query_runs(self, *, run: typing.Optional[EvaluationRunQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunsResponse: + """ + Parameters + ---------- + run : typing.Optional[EvaluationRunQuery] + + windowing : typing.Optional[Windowing] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRunsResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + ) + + + async def main() -> None: + await client.evaluations.query_runs() + + + asyncio.run(main()) + """ + _response = await self._raw_client.query_runs(run=run, windowing=windowing, request_options=request_options) + return _response.data + + async def close_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunsResponse: + """ + Parameters + ---------- + run_ids : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRunsResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + ) + + async def main() -> None: - await client.evaluations.create_runs( - runs=[EvaluationRunCreate()], + await client.evaluations.close_runs( + run_ids=["run_ids"], ) asyncio.run(main()) """ - _response = await self._raw_client.create_runs(runs=runs, request_options=request_options) + _response = await self._raw_client.close_runs(run_ids=run_ids, request_options=request_options) return _response.data - async def delete_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunIdsResponse: + async def open_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunsResponse: """ Parameters ---------- @@ -1702,7 +2115,7 @@ async def delete_runs(self, *, run_ids: typing.Sequence[str], request_options: t Returns ------- - EvaluationRunIdsResponse + EvaluationRunsResponse Successful Response Examples @@ -1717,35 +2130,35 @@ async def delete_runs(self, *, run_ids: typing.Sequence[str], request_options: t async def main() -> None: - await client.evaluations.delete_runs( + await client.evaluations.open_runs( run_ids=["run_ids"], ) asyncio.run(main()) """ - _response = await self._raw_client.delete_runs(run_ids=run_ids, request_options=request_options) + _response = await self._raw_client.open_runs(run_ids=run_ids, request_options=request_options) return _response.data - async def edit_runs(self, *, runs: typing.Sequence[EvaluationRunEdit], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunsResponse: + async def fetch_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: """ Parameters ---------- - runs : typing.Sequence[EvaluationRunEdit] + run_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationRunsResponse + EvaluationRunResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationRunEdit + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -1753,30 +2166,28 @@ async def edit_runs(self, *, runs: typing.Sequence[EvaluationRunEdit], request_o async def main() -> None: - await client.evaluations.edit_runs( - runs=[EvaluationRunEdit()], + await client.evaluations.fetch_run( + run_id="run_id", ) asyncio.run(main()) """ - _response = await self._raw_client.edit_runs(runs=runs, request_options=request_options) + _response = await self._raw_client.fetch_run(run_id, request_options=request_options) return _response.data - async def query_runs(self, *, run: typing.Optional[EvaluationRunQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunsResponse: + async def delete_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunIdResponse: """ Parameters ---------- - run : typing.Optional[EvaluationRunQuery] - - windowing : typing.Optional[Windowing] + run_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationRunsResponse + EvaluationRunIdResponse Successful Response Examples @@ -1791,33 +2202,37 @@ async def query_runs(self, *, run: typing.Optional[EvaluationRunQuery] = OMIT, w async def main() -> None: - await client.evaluations.query_runs() + await client.evaluations.delete_run( + run_id="run_id", + ) asyncio.run(main()) """ - _response = await self._raw_client.query_runs(run=run, windowing=windowing, request_options=request_options) + _response = await self._raw_client.delete_run(run_id, request_options=request_options) return _response.data - async def close_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunsResponse: + async def edit_run(self, run_id: str, *, run: EvaluationRunEdit, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: """ Parameters ---------- - run_ids : typing.Sequence[str] + run_id : str + + run : EvaluationRunEdit request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationRunsResponse + EvaluationRunResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, EvaluationRunEdit client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -1825,28 +2240,31 @@ async def close_runs(self, *, run_ids: typing.Sequence[str], request_options: ty async def main() -> None: - await client.evaluations.close_runs( - run_ids=["run_ids"], + await client.evaluations.edit_run( + run_id="run_id", + run=EvaluationRunEdit(), ) asyncio.run(main()) """ - _response = await self._raw_client.close_runs(run_ids=run_ids, request_options=request_options) + _response = await self._raw_client.edit_run(run_id, run=run, request_options=request_options) return _response.data - async def open_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunsResponse: + async def close_run(self, run_id: str, *, status: typing.Optional[EvaluationStatus] = None, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: """ Parameters ---------- - run_ids : typing.Sequence[str] + run_id : str + + status : typing.Optional[EvaluationStatus] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationRunsResponse + EvaluationRunResponse Successful Response Examples @@ -1861,17 +2279,17 @@ async def open_runs(self, *, run_ids: typing.Sequence[str], request_options: typ async def main() -> None: - await client.evaluations.open_runs( - run_ids=["run_ids"], + await client.evaluations.close_run( + run_id="run_id", ) asyncio.run(main()) """ - _response = await self._raw_client.open_runs(run_ids=run_ids, request_options=request_options) + _response = await self._raw_client.close_run(run_id, status=status, request_options=request_options) return _response.data - async def fetch_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: + async def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: """ Parameters ---------- @@ -1897,17 +2315,17 @@ async def fetch_run(self, run_id: str, *, request_options: typing.Optional[Reque async def main() -> None: - await client.evaluations.fetch_run( + await client.evaluations.open_run( run_id="run_id", ) asyncio.run(main()) """ - _response = await self._raw_client.fetch_run(run_id, request_options=request_options) + _response = await self._raw_client.open_run(run_id, request_options=request_options) return _response.data - async def delete_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunIdResponse: + async def fetch_default_queue(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueueResponse: """ Parameters ---------- @@ -1918,7 +2336,7 @@ async def delete_run(self, run_id: str, *, request_options: typing.Optional[Requ Returns ------- - EvaluationRunIdResponse + EvaluationQueueResponse Successful Response Examples @@ -1933,37 +2351,75 @@ async def delete_run(self, run_id: str, *, request_options: typing.Optional[Requ async def main() -> None: - await client.evaluations.delete_run( + await client.evaluations.fetch_default_queue( run_id="run_id", ) asyncio.run(main()) """ - _response = await self._raw_client.delete_run(run_id, request_options=request_options) + _response = await self._raw_client.fetch_default_queue(run_id, request_options=request_options) return _response.data - async def edit_run(self, run_id: str, *, run: EvaluationRunEdit, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: + async def create_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: """ Parameters ---------- - run_id : str + scenarios : typing.Sequence[EvaluationScenarioCreate] - run : EvaluationRunEdit + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationScenariosResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi, EvaluationScenarioCreate + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + ) + + + async def main() -> None: + await client.evaluations.create_scenarios( + scenarios=[ + EvaluationScenarioCreate( + run_id="run_id", + ) + ], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.create_scenarios(scenarios=scenarios, request_options=request_options) + return _response.data + + async def delete_scenarios(self, *, scenario_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenarioIdsResponse: + """ + Parameters + ---------- + scenario_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationRunResponse + EvaluationScenarioIdsResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationRunEdit + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -1971,31 +2427,66 @@ async def edit_run(self, run_id: str, *, run: EvaluationRunEdit, request_options async def main() -> None: - await client.evaluations.edit_run( - run_id="run_id", - run=EvaluationRunEdit(), + await client.evaluations.delete_scenarios( + scenario_ids=["scenario_ids"], ) asyncio.run(main()) """ - _response = await self._raw_client.edit_run(run_id, run=run, request_options=request_options) + _response = await self._raw_client.delete_scenarios(scenario_ids=scenario_ids, request_options=request_options) return _response.data - async def close_run(self, run_id: str, *, status: typing.Optional[EvaluationStatus] = None, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: + async def edit_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioEdit], request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: """ Parameters ---------- - run_id : str + scenarios : typing.Sequence[EvaluationScenarioEdit] - status : typing.Optional[EvaluationStatus] + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationScenariosResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi, EvaluationScenarioEdit + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + ) + + + async def main() -> None: + await client.evaluations.edit_scenarios( + scenarios=[EvaluationScenarioEdit()], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.edit_scenarios(scenarios=scenarios, request_options=request_options) + return _response.data + + async def query_scenarios(self, *, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: + """ + Parameters + ---------- + scenario : typing.Optional[EvaluationScenarioQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationRunResponse + EvaluationScenariosResponse Successful Response Examples @@ -2010,37 +2501,107 @@ async def close_run(self, run_id: str, *, status: typing.Optional[EvaluationStat async def main() -> None: - await client.evaluations.close_run( - run_id="run_id", + await client.evaluations.query_scenarios() + + + asyncio.run(main()) + """ + _response = await self._raw_client.query_scenarios(scenario=scenario, windowing=windowing, request_options=request_options) + return _response.data + + async def fetch_scenario(self, scenario_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenarioResponse: + """ + Parameters + ---------- + scenario_id : str + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationScenarioResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + ) + + + async def main() -> None: + await client.evaluations.fetch_scenario( + scenario_id="scenario_id", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.fetch_scenario(scenario_id, request_options=request_options) + return _response.data + + async def delete_scenario(self, scenario_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenarioIdResponse: + """ + Parameters + ---------- + scenario_id : str + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationScenarioIdResponse + Successful Response + + Examples + -------- + import asyncio + + from agenta import AsyncAgentaApi + + client = AsyncAgentaApi( + api_key="YOUR_API_KEY", + ) + + + async def main() -> None: + await client.evaluations.delete_scenario( + scenario_id="scenario_id", ) asyncio.run(main()) """ - _response = await self._raw_client.close_run(run_id, status=status, request_options=request_options) + _response = await self._raw_client.delete_scenario(scenario_id, request_options=request_options) return _response.data - async def close_run_with_status(self, run_id: str, status: typing.Optional[EvaluationStatus], *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: + async def edit_scenario(self, scenario_id: str, *, scenario: EvaluationScenarioEdit, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenarioResponse: """ Parameters ---------- - run_id : str + scenario_id : str - status : typing.Optional[EvaluationStatus] + scenario : EvaluationScenarioEdit request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationRunResponse + EvaluationScenarioResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, EvaluationScenarioEdit client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2048,36 +2609,36 @@ async def close_run_with_status(self, run_id: str, status: typing.Optional[Evalu async def main() -> None: - await client.evaluations.close_run_with_status( - run_id="run_id", - status="pending", + await client.evaluations.edit_scenario( + scenario_id="scenario_id", + scenario=EvaluationScenarioEdit(), ) asyncio.run(main()) """ - _response = await self._raw_client.close_run_with_status(run_id, status, request_options=request_options) + _response = await self._raw_client.edit_scenario(scenario_id, scenario=scenario, request_options=request_options) return _response.data - async def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: + async def set_results(self, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: """ Parameters ---------- - run_id : str + results : typing.Sequence[EvaluationResultCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationRunResponse + EvaluationResultsResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, EvaluationResultCreate client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2085,35 +2646,41 @@ async def open_run(self, run_id: str, *, request_options: typing.Optional[Reques async def main() -> None: - await client.evaluations.open_run( - run_id="run_id", + await client.evaluations.set_results( + results=[ + EvaluationResultCreate( + step_key="step_key", + scenario_id="scenario_id", + run_id="run_id", + ) + ], ) asyncio.run(main()) """ - _response = await self._raw_client.open_run(run_id, request_options=request_options) + _response = await self._raw_client.set_results(results=results, request_options=request_options) return _response.data - async def create_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: + async def delete_results(self, *, result_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultIdsResponse: """ Parameters ---------- - scenarios : typing.Sequence[EvaluationScenarioCreate] + result_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationScenariosResponse + EvaluationResultIdsResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationScenarioCreate + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2121,32 +2688,30 @@ async def create_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenari async def main() -> None: - await client.evaluations.create_scenarios( - scenarios=[ - EvaluationScenarioCreate( - run_id="run_id", - ) - ], + await client.evaluations.delete_results( + result_ids=["result_ids"], ) asyncio.run(main()) """ - _response = await self._raw_client.create_scenarios(scenarios=scenarios, request_options=request_options) + _response = await self._raw_client.delete_results(result_ids=result_ids, request_options=request_options) return _response.data - async def delete_scenarios(self, *, scenario_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenarioIdsResponse: + async def query_results(self, *, result: typing.Optional[EvaluationResultQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: """ Parameters ---------- - scenario_ids : typing.Sequence[str] + result : typing.Optional[EvaluationResultQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationScenarioIdsResponse + EvaluationResultsResponse Successful Response Examples @@ -2161,35 +2726,33 @@ async def delete_scenarios(self, *, scenario_ids: typing.Sequence[str], request_ async def main() -> None: - await client.evaluations.delete_scenarios( - scenario_ids=["scenario_ids"], - ) + await client.evaluations.query_results() asyncio.run(main()) """ - _response = await self._raw_client.delete_scenarios(scenario_ids=scenario_ids, request_options=request_options) + _response = await self._raw_client.query_results(result=result, windowing=windowing, request_options=request_options) return _response.data - async def edit_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioEdit], request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: + async def fetch_result(self, result_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultResponse: """ Parameters ---------- - scenarios : typing.Sequence[EvaluationScenarioEdit] + result_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationScenariosResponse + EvaluationResultResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationScenarioEdit + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2197,30 +2760,28 @@ async def edit_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioE async def main() -> None: - await client.evaluations.edit_scenarios( - scenarios=[EvaluationScenarioEdit()], + await client.evaluations.fetch_result( + result_id="result_id", ) asyncio.run(main()) """ - _response = await self._raw_client.edit_scenarios(scenarios=scenarios, request_options=request_options) + _response = await self._raw_client.fetch_result(result_id, request_options=request_options) return _response.data - async def query_scenarios(self, *, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: + async def delete_result(self, result_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultIdResponse: """ Parameters ---------- - scenario : typing.Optional[EvaluationScenarioQuery] - - windowing : typing.Optional[Windowing] + result_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationScenariosResponse + EvaluationResultIdResponse Successful Response Examples @@ -2235,33 +2796,35 @@ async def query_scenarios(self, *, scenario: typing.Optional[EvaluationScenarioQ async def main() -> None: - await client.evaluations.query_scenarios() + await client.evaluations.delete_result( + result_id="result_id", + ) asyncio.run(main()) """ - _response = await self._raw_client.query_scenarios(scenario=scenario, windowing=windowing, request_options=request_options) + _response = await self._raw_client.delete_result(result_id, request_options=request_options) return _response.data - async def fetch_scenario(self, scenario_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenarioResponse: + async def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: """ Parameters ---------- - scenario_id : str + metrics : EvaluationMetricsRefresh request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationScenarioResponse + EvaluationMetricsResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, EvaluationMetricsRefresh client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2269,35 +2832,35 @@ async def fetch_scenario(self, scenario_id: str, *, request_options: typing.Opti async def main() -> None: - await client.evaluations.fetch_scenario( - scenario_id="scenario_id", + await client.evaluations.refresh_metrics( + metrics=EvaluationMetricsRefresh(), ) asyncio.run(main()) """ - _response = await self._raw_client.fetch_scenario(scenario_id, request_options=request_options) + _response = await self._raw_client.refresh_metrics(metrics=metrics, request_options=request_options) return _response.data - async def delete_scenario(self, scenario_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenarioIdResponse: + async def set_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: """ Parameters ---------- - scenario_id : str + metrics : typing.Sequence[EvaluationMetricsCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationScenarioIdResponse + EvaluationMetricsResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, EvaluationMetricsCreate client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2305,37 +2868,39 @@ async def delete_scenario(self, scenario_id: str, *, request_options: typing.Opt async def main() -> None: - await client.evaluations.delete_scenario( - scenario_id="scenario_id", + await client.evaluations.set_metrics( + metrics=[ + EvaluationMetricsCreate( + run_id="run_id", + ) + ], ) asyncio.run(main()) """ - _response = await self._raw_client.delete_scenario(scenario_id, request_options=request_options) + _response = await self._raw_client.set_metrics(metrics=metrics, request_options=request_options) return _response.data - async def edit_scenario(self, scenario_id: str, *, scenario: EvaluationScenarioEdit, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenarioResponse: + async def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsIdsResponse: """ Parameters ---------- - scenario_id : str - - scenario : EvaluationScenarioEdit + metrics_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationScenarioResponse + EvaluationMetricsIdsResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationScenarioEdit + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2343,36 +2908,37 @@ async def edit_scenario(self, scenario_id: str, *, scenario: EvaluationScenarioE async def main() -> None: - await client.evaluations.edit_scenario( - scenario_id="scenario_id", - scenario=EvaluationScenarioEdit(), + await client.evaluations.delete_metrics( + metrics_ids=["metrics_ids"], ) asyncio.run(main()) """ - _response = await self._raw_client.edit_scenario(scenario_id, scenario=scenario, request_options=request_options) + _response = await self._raw_client.delete_metrics(metrics_ids=metrics_ids, request_options=request_options) return _response.data - async def create_results(self, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: + async def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: """ Parameters ---------- - results : typing.Sequence[EvaluationResultCreate] + metrics : typing.Optional[EvaluationMetricsQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationResultsResponse + EvaluationMetricsResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationResultCreate + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2380,34 +2946,26 @@ async def create_results(self, *, results: typing.Sequence[EvaluationResultCreat async def main() -> None: - await client.evaluations.create_results( - results=[ - EvaluationResultCreate( - step_key="step_key", - scenario_id="scenario_id", - run_id="run_id", - ) - ], - ) + await client.evaluations.query_metrics() asyncio.run(main()) """ - _response = await self._raw_client.create_results(results=results, request_options=request_options) + _response = await self._raw_client.query_metrics(metrics=metrics, windowing=windowing, request_options=request_options) return _response.data - async def delete_results(self, *, result_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultIdsResponse: + async def fetch_metric(self, metrics_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: """ Parameters ---------- - result_ids : typing.Sequence[str] + metrics_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationResultIdsResponse + EvaluationMetricsResponse Successful Response Examples @@ -2422,35 +2980,35 @@ async def delete_results(self, *, result_ids: typing.Sequence[str], request_opti async def main() -> None: - await client.evaluations.delete_results( - result_ids=["result_ids"], + await client.evaluations.fetch_metric( + metrics_id="metrics_id", ) asyncio.run(main()) """ - _response = await self._raw_client.delete_results(result_ids=result_ids, request_options=request_options) + _response = await self._raw_client.fetch_metric(metrics_id, request_options=request_options) return _response.data - async def edit_results(self, *, results: typing.Sequence[EvaluationResultEdit], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: + async def delete_metric(self, metrics_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsIdsResponse: """ Parameters ---------- - results : typing.Sequence[EvaluationResultEdit] + metrics_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationResultsResponse + EvaluationMetricsIdsResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationResultEdit + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2458,37 +3016,35 @@ async def edit_results(self, *, results: typing.Sequence[EvaluationResultEdit], async def main() -> None: - await client.evaluations.edit_results( - results=[EvaluationResultEdit()], + await client.evaluations.delete_metric( + metrics_id="metrics_id", ) asyncio.run(main()) """ - _response = await self._raw_client.edit_results(results=results, request_options=request_options) + _response = await self._raw_client.delete_metric(metrics_id, request_options=request_options) return _response.data - async def query_results(self, *, result: typing.Optional[EvaluationResultQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: + async def create_queues(self, *, queues: typing.Sequence[EvaluationQueueCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueuesResponse: """ Parameters ---------- - result : typing.Optional[EvaluationResultQuery] - - windowing : typing.Optional[Windowing] + queues : typing.Sequence[EvaluationQueueCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationResultsResponse + EvaluationQueuesResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, EvaluationQueueCreate client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2496,26 +3052,32 @@ async def query_results(self, *, result: typing.Optional[EvaluationResultQuery] async def main() -> None: - await client.evaluations.query_results() + await client.evaluations.create_queues( + queues=[ + EvaluationQueueCreate( + run_id="run_id", + ) + ], + ) asyncio.run(main()) """ - _response = await self._raw_client.query_results(result=result, windowing=windowing, request_options=request_options) + _response = await self._raw_client.create_queues(queues=queues, request_options=request_options) return _response.data - async def fetch_result(self, result_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultResponse: + async def delete_queues(self, *, queue_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueueIdsResponse: """ Parameters ---------- - result_id : str + queue_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationResultResponse + EvaluationQueueIdsResponse Successful Response Examples @@ -2530,35 +3092,35 @@ async def fetch_result(self, result_id: str, *, request_options: typing.Optional async def main() -> None: - await client.evaluations.fetch_result( - result_id="result_id", + await client.evaluations.delete_queues( + queue_ids=["queue_ids"], ) asyncio.run(main()) """ - _response = await self._raw_client.fetch_result(result_id, request_options=request_options) + _response = await self._raw_client.delete_queues(queue_ids=queue_ids, request_options=request_options) return _response.data - async def delete_result(self, result_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultIdResponse: + async def edit_queues(self, *, queues: typing.Sequence[EvaluationQueueEdit], request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueuesResponse: """ Parameters ---------- - result_id : str + queues : typing.Sequence[EvaluationQueueEdit] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationResultIdResponse + EvaluationQueuesResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, EvaluationQueueEdit client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2566,37 +3128,37 @@ async def delete_result(self, result_id: str, *, request_options: typing.Optiona async def main() -> None: - await client.evaluations.delete_result( - result_id="result_id", + await client.evaluations.edit_queues( + queues=[EvaluationQueueEdit()], ) asyncio.run(main()) """ - _response = await self._raw_client.delete_result(result_id, request_options=request_options) + _response = await self._raw_client.edit_queues(queues=queues, request_options=request_options) return _response.data - async def edit_result(self, result_id: str, *, result: EvaluationResultEdit, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultResponse: + async def query_queues(self, *, queue: typing.Optional[EvaluationQueueQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueuesResponse: """ Parameters ---------- - result_id : str + queue : typing.Optional[EvaluationQueueQuery] - result : EvaluationResultEdit + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationResultResponse + EvaluationQueuesResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationResultEdit + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2604,36 +3166,33 @@ async def edit_result(self, result_id: str, *, result: EvaluationResultEdit, req async def main() -> None: - await client.evaluations.edit_result( - result_id="result_id", - result=EvaluationResultEdit(), - ) + await client.evaluations.query_queues() asyncio.run(main()) """ - _response = await self._raw_client.edit_result(result_id, result=result, request_options=request_options) + _response = await self._raw_client.query_queues(queue=queue, windowing=windowing, request_options=request_options) return _response.data - async def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: + async def fetch_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueueResponse: """ Parameters ---------- - metrics : EvaluationMetricsRefresh + queue_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationMetricsResponse + EvaluationQueueResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationMetricsRefresh + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2641,35 +3200,35 @@ async def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_op async def main() -> None: - await client.evaluations.refresh_metrics( - metrics=EvaluationMetricsRefresh(), + await client.evaluations.fetch_queue( + queue_id="queue_id", ) asyncio.run(main()) """ - _response = await self._raw_client.refresh_metrics(metrics=metrics, request_options=request_options) + _response = await self._raw_client.fetch_queue(queue_id, request_options=request_options) return _response.data - async def create_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: + async def delete_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueueIdResponse: """ Parameters ---------- - metrics : typing.Sequence[EvaluationMetricsCreate] + queue_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationMetricsResponse + EvaluationQueueIdResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationMetricsCreate + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2677,39 +3236,37 @@ async def create_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCrea async def main() -> None: - await client.evaluations.create_metrics( - metrics=[ - EvaluationMetricsCreate( - run_id="run_id", - ) - ], + await client.evaluations.delete_queue( + queue_id="queue_id", ) asyncio.run(main()) """ - _response = await self._raw_client.create_metrics(metrics=metrics, request_options=request_options) + _response = await self._raw_client.delete_queue(queue_id, request_options=request_options) return _response.data - async def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsIdsResponse: + async def edit_queue(self, queue_id: str, *, queue: EvaluationQueueEdit, request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueueResponse: """ Parameters ---------- - metrics_ids : typing.Sequence[str] + queue_id : str + + queue : EvaluationQueueEdit request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationMetricsIdsResponse + EvaluationQueueResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, EvaluationQueueEdit client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2717,35 +3274,42 @@ async def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_opt async def main() -> None: - await client.evaluations.delete_metrics( - metrics_ids=["metrics_ids"], + await client.evaluations.edit_queue( + queue_id="queue_id", + queue=EvaluationQueueEdit(), ) asyncio.run(main()) """ - _response = await self._raw_client.delete_metrics(metrics_ids=metrics_ids, request_options=request_options) + _response = await self._raw_client.edit_queue(queue_id, queue=queue, request_options=request_options) return _response.data - async def edit_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsEdit], request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: + async def query_evaluation_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[EvaluationQueueScenariosQuery] = OMIT, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: """ Parameters ---------- - metrics : typing.Sequence[EvaluationMetricsEdit] + queue_id : str + + queue : typing.Optional[EvaluationQueueScenariosQuery] + + scenario : typing.Optional[EvaluationScenarioQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationMetricsResponse + EvaluationScenariosResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationMetricsEdit + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2753,37 +3317,35 @@ async def edit_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsEdit], async def main() -> None: - await client.evaluations.edit_metrics( - metrics=[EvaluationMetricsEdit()], + await client.evaluations.query_evaluation_queue_scenarios( + queue_id="queue_id", ) asyncio.run(main()) """ - _response = await self._raw_client.edit_metrics(metrics=metrics, request_options=request_options) + _response = await self._raw_client.query_evaluation_queue_scenarios(queue_id, queue=queue, scenario=scenario, windowing=windowing, request_options=request_options) return _response.data - async def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationMetricsResponse: + async def create_simple_evaluation(self, *, evaluation: SimpleEvaluationCreate, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: """ Parameters ---------- - metrics : typing.Optional[EvaluationMetricsQuery] - - windowing : typing.Optional[Windowing] + evaluation : SimpleEvaluationCreate request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationMetricsResponse + SimpleEvaluationResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, SimpleEvaluationCreate client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2791,33 +3353,37 @@ async def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery async def main() -> None: - await client.evaluations.query_metrics() + await client.evaluations.create_simple_evaluation( + evaluation=SimpleEvaluationCreate(), + ) asyncio.run(main()) """ - _response = await self._raw_client.query_metrics(metrics=metrics, windowing=windowing, request_options=request_options) + _response = await self._raw_client.create_simple_evaluation(evaluation=evaluation, request_options=request_options) return _response.data - async def create_queues(self, *, queues: typing.Sequence[EvaluationQueueCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueuesResponse: + async def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEvaluationQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationsResponse: """ Parameters ---------- - queues : typing.Sequence[EvaluationQueueCreate] + evaluation : typing.Optional[SimpleEvaluationQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationQueuesResponse + SimpleEvaluationsResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationQueueCreate + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2825,32 +3391,26 @@ async def create_queues(self, *, queues: typing.Sequence[EvaluationQueueCreate], async def main() -> None: - await client.evaluations.create_queues( - queues=[ - EvaluationQueueCreate( - run_id="run_id", - ) - ], - ) + await client.evaluations.query_simple_evaluations() asyncio.run(main()) """ - _response = await self._raw_client.create_queues(queues=queues, request_options=request_options) + _response = await self._raw_client.query_simple_evaluations(evaluation=evaluation, windowing=windowing, request_options=request_options) return _response.data - async def delete_queues(self, *, queue_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueueIdsResponse: + async def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: """ Parameters ---------- - queue_ids : typing.Sequence[str] + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationQueueIdsResponse + SimpleEvaluationResponse Successful Response Examples @@ -2865,35 +3425,35 @@ async def delete_queues(self, *, queue_ids: typing.Sequence[str], request_option async def main() -> None: - await client.evaluations.delete_queues( - queue_ids=["queue_ids"], + await client.evaluations.fetch_simple_evaluation( + evaluation_id="evaluation_id", ) asyncio.run(main()) """ - _response = await self._raw_client.delete_queues(queue_ids=queue_ids, request_options=request_options) + _response = await self._raw_client.fetch_simple_evaluation(evaluation_id, request_options=request_options) return _response.data - async def edit_queues(self, *, queues: typing.Sequence[EvaluationQueueEdit], request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueuesResponse: + async def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationIdResponse: """ Parameters ---------- - queues : typing.Sequence[EvaluationQueueEdit] + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationQueuesResponse + SimpleEvaluationIdResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationQueueEdit + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2901,37 +3461,37 @@ async def edit_queues(self, *, queues: typing.Sequence[EvaluationQueueEdit], req async def main() -> None: - await client.evaluations.edit_queues( - queues=[EvaluationQueueEdit()], + await client.evaluations.delete_simple_evaluation( + evaluation_id="evaluation_id", ) asyncio.run(main()) """ - _response = await self._raw_client.edit_queues(queues=queues, request_options=request_options) + _response = await self._raw_client.delete_simple_evaluation(evaluation_id, request_options=request_options) return _response.data - async def query_queues(self, *, queue: typing.Optional[EvaluationQueueQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueuesResponse: + async def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: SimpleEvaluationEdit, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: """ Parameters ---------- - queue : typing.Optional[EvaluationQueueQuery] + evaluation_id : str - windowing : typing.Optional[Windowing] + evaluation : SimpleEvaluationEdit request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationQueuesResponse + SimpleEvaluationResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, SimpleEvaluationEdit client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -2939,26 +3499,29 @@ async def query_queues(self, *, queue: typing.Optional[EvaluationQueueQuery] = O async def main() -> None: - await client.evaluations.query_queues() + await client.evaluations.edit_simple_evaluation( + evaluation_id="evaluation_id", + evaluation=SimpleEvaluationEdit(), + ) asyncio.run(main()) """ - _response = await self._raw_client.query_queues(queue=queue, windowing=windowing, request_options=request_options) + _response = await self._raw_client.edit_simple_evaluation(evaluation_id, evaluation=evaluation, request_options=request_options) return _response.data - async def fetch_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueueResponse: + async def start_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: """ Parameters ---------- - queue_id : str + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationQueueResponse + SimpleEvaluationResponse Successful Response Examples @@ -2973,28 +3536,28 @@ async def fetch_queue(self, queue_id: str, *, request_options: typing.Optional[R async def main() -> None: - await client.evaluations.fetch_queue( - queue_id="queue_id", + await client.evaluations.start_simple_evaluation( + evaluation_id="evaluation_id", ) asyncio.run(main()) """ - _response = await self._raw_client.fetch_queue(queue_id, request_options=request_options) + _response = await self._raw_client.start_simple_evaluation(evaluation_id, request_options=request_options) return _response.data - async def delete_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueueIdResponse: + async def stop_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: """ Parameters ---------- - queue_id : str + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationQueueIdResponse + SimpleEvaluationResponse Successful Response Examples @@ -3009,37 +3572,35 @@ async def delete_queue(self, queue_id: str, *, request_options: typing.Optional[ async def main() -> None: - await client.evaluations.delete_queue( - queue_id="queue_id", + await client.evaluations.stop_simple_evaluation( + evaluation_id="evaluation_id", ) asyncio.run(main()) """ - _response = await self._raw_client.delete_queue(queue_id, request_options=request_options) + _response = await self._raw_client.stop_simple_evaluation(evaluation_id, request_options=request_options) return _response.data - async def edit_queue(self, queue_id: str, *, queue: EvaluationQueueEdit, request_options: typing.Optional[RequestOptions] = None) -> EvaluationQueueResponse: + async def close_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: """ Parameters ---------- - queue_id : str - - queue : EvaluationQueueEdit + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationQueueResponse + SimpleEvaluationResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, EvaluationQueueEdit + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -3047,35 +3608,28 @@ async def edit_queue(self, queue_id: str, *, queue: EvaluationQueueEdit, request async def main() -> None: - await client.evaluations.edit_queue( - queue_id="queue_id", - queue=EvaluationQueueEdit(), + await client.evaluations.close_simple_evaluation( + evaluation_id="evaluation_id", ) asyncio.run(main()) """ - _response = await self._raw_client.edit_queue(queue_id, queue=queue, request_options=request_options) + _response = await self._raw_client.close_simple_evaluation(evaluation_id, request_options=request_options) return _response.data - async def query_evaluation_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[EvaluationQueueScenariosQuery] = OMIT, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: + async def open_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: """ Parameters ---------- - queue_id : str - - queue : typing.Optional[EvaluationQueueScenariosQuery] - - scenario : typing.Optional[EvaluationScenarioQuery] - - windowing : typing.Optional[Windowing] + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - EvaluationScenariosResponse + SimpleEvaluationResponse Successful Response Examples @@ -3090,35 +3644,37 @@ async def query_evaluation_queue_scenarios(self, queue_id: str, *, queue: typing async def main() -> None: - await client.evaluations.query_evaluation_queue_scenarios( - queue_id="queue_id", + await client.evaluations.open_simple_evaluation( + evaluation_id="evaluation_id", ) asyncio.run(main()) """ - _response = await self._raw_client.query_evaluation_queue_scenarios(queue_id, queue=queue, scenario=scenario, windowing=windowing, request_options=request_options) + _response = await self._raw_client.open_simple_evaluation(evaluation_id, request_options=request_options) return _response.data - async def create_simple_evaluation(self, *, evaluation: SimpleEvaluationCreate, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: + async def populate_slice(self, evaluation_id: str, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: """ Parameters ---------- - evaluation : SimpleEvaluationCreate + evaluation_id : str + + results : typing.Sequence[EvaluationResultCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationResponse + EvaluationResultsResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi, SimpleEvaluationCreate + from agenta import AsyncAgentaApi, EvaluationResultCreate client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -3126,29 +3682,44 @@ async def create_simple_evaluation(self, *, evaluation: SimpleEvaluationCreate, async def main() -> None: - await client.evaluations.create_simple_evaluation( - evaluation=SimpleEvaluationCreate(), + await client.evaluations.populate_slice( + evaluation_id="evaluation_id", + results=[ + EvaluationResultCreate( + step_key="step_key", + scenario_id="scenario_id", + run_id="run_id", + ) + ], ) asyncio.run(main()) """ - _response = await self._raw_client.create_simple_evaluation(evaluation=evaluation, request_options=request_options) + _response = await self._raw_client.populate_slice(evaluation_id, results=results, request_options=request_options) return _response.data - async def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: + async def process_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, overwrite: typing.Optional[bool] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> typing.Any: """ Parameters ---------- evaluation_id : str + scenario_ids : typing.Optional[typing.Sequence[str]] + + step_keys : typing.Optional[typing.Sequence[str]] + + repeat_idxs : typing.Optional[typing.Sequence[int]] + + overwrite : typing.Optional[bool] + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationResponse - Successful Response + typing.Any + Accepted. Examples -------- @@ -3162,28 +3733,34 @@ async def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: async def main() -> None: - await client.evaluations.fetch_simple_evaluation( + await client.evaluations.process_slice( evaluation_id="evaluation_id", ) asyncio.run(main()) """ - _response = await self._raw_client.fetch_simple_evaluation(evaluation_id, request_options=request_options) + _response = await self._raw_client.process_slice(evaluation_id, scenario_ids=scenario_ids, step_keys=step_keys, repeat_idxs=repeat_idxs, overwrite=overwrite, request_options=request_options) return _response.data - async def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationIdResponse: + async def probe_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationResultsResponse: """ Parameters ---------- evaluation_id : str + scenario_ids : typing.Optional[typing.Sequence[str]] + + step_keys : typing.Optional[typing.Sequence[str]] + + repeat_idxs : typing.Optional[typing.Sequence[int]] + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationIdResponse + EvaluationResultsResponse Successful Response Examples @@ -3198,37 +3775,40 @@ async def delete_simple_evaluation(self, evaluation_id: str, *, request_options: async def main() -> None: - await client.evaluations.delete_simple_evaluation( + await client.evaluations.probe_slice( evaluation_id="evaluation_id", ) asyncio.run(main()) """ - _response = await self._raw_client.delete_simple_evaluation(evaluation_id, request_options=request_options) + _response = await self._raw_client.probe_slice(evaluation_id, scenario_ids=scenario_ids, step_keys=step_keys, repeat_idxs=repeat_idxs, request_options=request_options) return _response.data - async def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: SimpleEvaluationEdit, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: + async def prune_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> None: """ Parameters ---------- evaluation_id : str - evaluation : SimpleEvaluationEdit + scenario_ids : typing.Optional[typing.Sequence[str]] + + step_keys : typing.Optional[typing.Sequence[str]] + + repeat_idxs : typing.Optional[typing.Sequence[int]] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationResponse - Successful Response + None Examples -------- import asyncio - from agenta import AsyncAgentaApi, SimpleEvaluationEdit + from agenta import AsyncAgentaApi client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -3236,31 +3816,32 @@ async def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: Simple async def main() -> None: - await client.evaluations.edit_simple_evaluation( + await client.evaluations.prune_slice( evaluation_id="evaluation_id", - evaluation=SimpleEvaluationEdit(), ) asyncio.run(main()) """ - _response = await self._raw_client.edit_simple_evaluation(evaluation_id, evaluation=evaluation, request_options=request_options) + _response = await self._raw_client.prune_slice(evaluation_id, scenario_ids=scenario_ids, step_keys=step_keys, repeat_idxs=repeat_idxs, request_options=request_options) return _response.data - async def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEvaluationQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationsResponse: + async def add_scenarios(self, evaluation_id: str, *, count: int, timestamp: typing.Optional[dt.datetime] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> EvaluationScenariosResponse: """ Parameters ---------- - evaluation : typing.Optional[SimpleEvaluationQuery] + evaluation_id : str - windowing : typing.Optional[Windowing] + count : int + + timestamp : typing.Optional[dt.datetime] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationsResponse + EvaluationScenariosResponse Successful Response Examples @@ -3275,27 +3856,31 @@ async def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEv async def main() -> None: - await client.evaluations.query_simple_evaluations() + await client.evaluations.add_scenarios( + evaluation_id="evaluation_id", + count=1, + ) asyncio.run(main()) """ - _response = await self._raw_client.query_simple_evaluations(evaluation=evaluation, windowing=windowing, request_options=request_options) + _response = await self._raw_client.add_scenarios(evaluation_id, count=count, timestamp=timestamp, request_options=request_options) return _response.data - async def start_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: + async def remove_scenarios(self, evaluation_id: str, *, scenario_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> None: """ Parameters ---------- evaluation_id : str + scenario_ids : typing.Sequence[str] + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationResponse - Successful Response + None Examples -------- @@ -3309,35 +3894,38 @@ async def start_simple_evaluation(self, evaluation_id: str, *, request_options: async def main() -> None: - await client.evaluations.start_simple_evaluation( + await client.evaluations.remove_scenarios( evaluation_id="evaluation_id", + scenario_ids=["scenario_ids"], ) asyncio.run(main()) """ - _response = await self._raw_client.start_simple_evaluation(evaluation_id, request_options=request_options) + _response = await self._raw_client.remove_scenarios(evaluation_id, scenario_ids=scenario_ids, request_options=request_options) return _response.data - async def stop_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: + async def add_steps(self, evaluation_id: str, *, steps: typing.Sequence[EvaluationRunDataStep], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: """ Parameters ---------- evaluation_id : str + steps : typing.Sequence[EvaluationRunDataStep] + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationResponse + EvaluationRunResponse Successful Response Examples -------- import asyncio - from agenta import AsyncAgentaApi + from agenta import AsyncAgentaApi, EvaluationRunDataStep, Reference client = AsyncAgentaApi( api_key="YOUR_API_KEY", @@ -3345,28 +3933,38 @@ async def stop_simple_evaluation(self, evaluation_id: str, *, request_options: t async def main() -> None: - await client.evaluations.stop_simple_evaluation( + await client.evaluations.add_steps( evaluation_id="evaluation_id", + steps=[ + EvaluationRunDataStep( + key="key", + type="input", + origin="custom", + references={"key": Reference()}, + ) + ], ) asyncio.run(main()) """ - _response = await self._raw_client.stop_simple_evaluation(evaluation_id, request_options=request_options) + _response = await self._raw_client.add_steps(evaluation_id, steps=steps, request_options=request_options) return _response.data - async def close_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: + async def remove_steps(self, evaluation_id: str, *, step_keys: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: """ Parameters ---------- evaluation_id : str + step_keys : typing.Sequence[str] + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationResponse + EvaluationRunResponse Successful Response Examples @@ -3381,28 +3979,31 @@ async def close_simple_evaluation(self, evaluation_id: str, *, request_options: async def main() -> None: - await client.evaluations.close_simple_evaluation( + await client.evaluations.remove_steps( evaluation_id="evaluation_id", + step_keys=["step_keys"], ) asyncio.run(main()) """ - _response = await self._raw_client.close_simple_evaluation(evaluation_id, request_options=request_options) + _response = await self._raw_client.remove_steps(evaluation_id, step_keys=step_keys, request_options=request_options) return _response.data - async def open_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> SimpleEvaluationResponse: + async def set_repeats(self, evaluation_id: str, *, repeats: int, request_options: typing.Optional[RequestOptions] = None) -> EvaluationRunResponse: """ Parameters ---------- evaluation_id : str + repeats : int + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - SimpleEvaluationResponse + EvaluationRunResponse Successful Response Examples @@ -3417,14 +4018,15 @@ async def open_simple_evaluation(self, evaluation_id: str, *, request_options: t async def main() -> None: - await client.evaluations.open_simple_evaluation( + await client.evaluations.set_repeats( evaluation_id="evaluation_id", + repeats=1, ) asyncio.run(main()) """ - _response = await self._raw_client.open_simple_evaluation(evaluation_id, request_options=request_options) + _response = await self._raw_client.set_repeats(evaluation_id, repeats=repeats, request_options=request_options) return _response.data async def create_simple_queue(self, *, queue: SimpleQueueCreate, request_options: typing.Optional[RequestOptions] = None) -> SimpleQueueResponse: diff --git a/clients/python/agenta_client/evaluations/raw_client.py b/clients/python/agenta_client/evaluations/raw_client.py index 47abfdda12..9302a48c6e 100644 --- a/clients/python/agenta_client/evaluations/raw_client.py +++ b/clients/python/agenta_client/evaluations/raw_client.py @@ -1,5 +1,6 @@ # This file was auto-generated by Fern from our API Definition. +import datetime as dt import typing from json.decoder import JSONDecodeError @@ -12,7 +13,6 @@ from ..core.serialization import convert_and_respect_annotation_metadata from ..errors.unprocessable_entity_error import UnprocessableEntityError from ..types.evaluation_metrics_create import EvaluationMetricsCreate -from ..types.evaluation_metrics_edit import EvaluationMetricsEdit from ..types.evaluation_metrics_ids_response import EvaluationMetricsIdsResponse from ..types.evaluation_metrics_query import EvaluationMetricsQuery from ..types.evaluation_metrics_refresh import EvaluationMetricsRefresh @@ -26,13 +26,13 @@ from ..types.evaluation_queue_scenarios_query import EvaluationQueueScenariosQuery from ..types.evaluation_queues_response import EvaluationQueuesResponse from ..types.evaluation_result_create import EvaluationResultCreate -from ..types.evaluation_result_edit import EvaluationResultEdit from ..types.evaluation_result_id_response import EvaluationResultIdResponse from ..types.evaluation_result_ids_response import EvaluationResultIdsResponse from ..types.evaluation_result_query import EvaluationResultQuery from ..types.evaluation_result_response import EvaluationResultResponse from ..types.evaluation_results_response import EvaluationResultsResponse from ..types.evaluation_run_create import EvaluationRunCreate +from ..types.evaluation_run_data_step import EvaluationRunDataStep from ..types.evaluation_run_edit import EvaluationRunEdit from ..types.evaluation_run_id_response import EvaluationRunIdResponse from ..types.evaluation_run_ids_response import EvaluationRunIdsResponse @@ -527,14 +527,12 @@ def close_run(self, run_id: str, *, status: typing.Optional[EvaluationStatus] = raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def close_run_with_status(self, run_id: str, status: typing.Optional[EvaluationStatus], *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationRunResponse]: + def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationRunResponse]: """ Parameters ---------- run_id : str - status : typing.Optional[EvaluationStatus] - request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -544,7 +542,7 @@ def close_run_with_status(self, run_id: str, status: typing.Optional[EvaluationS Successful Response """ _response = self._client_wrapper.httpx_client.request( - f"evaluations/runs/{jsonable_encoder(run_id)}/close/{jsonable_encoder(status)}",method="POST", + f"evaluations/runs/{jsonable_encoder(run_id)}/open",method="POST", request_options=request_options,) try: if 200 <= _response.status_code < 300: @@ -569,7 +567,7 @@ def close_run_with_status(self, run_id: str, status: typing.Optional[EvaluationS raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationRunResponse]: + def fetch_default_queue(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationQueueResponse]: """ Parameters ---------- @@ -580,18 +578,18 @@ def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptio Returns ------- - HttpResponse[EvaluationRunResponse] + HttpResponse[EvaluationQueueResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - f"evaluations/runs/{jsonable_encoder(run_id)}/open",method="POST", + f"evaluations/runs/{jsonable_encoder(run_id)}/queues/default",method="GET", request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunResponse, + EvaluationQueueResponse, parse_obj_as( - type_ =EvaluationRunResponse, # type: ignore + type_ =EvaluationQueueResponse, # type: ignore object_ =_response.json() ) ) @@ -929,7 +927,7 @@ def edit_scenario(self, scenario_id: str, *, scenario: EvaluationScenarioEdit, r raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def create_results(self, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationResultsResponse]: + def set_results(self, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationResultsResponse]: """ Parameters ---------- @@ -1023,53 +1021,6 @@ def delete_results(self, *, result_ids: typing.Sequence[str], request_options: t raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def edit_results(self, *, results: typing.Sequence[EvaluationResultEdit], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationResultsResponse]: - """ - Parameters - ---------- - results : typing.Sequence[EvaluationResultEdit] - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[EvaluationResultsResponse] - Successful Response - """ - _response = self._client_wrapper.httpx_client.request( - "evaluations/results/",method="PATCH", - json={ - "results": convert_and_respect_annotation_metadata(object_=results, annotation=typing.Sequence[EvaluationResultEdit], direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - EvaluationResultsResponse, - parse_obj_as( - type_ =EvaluationResultsResponse, # type: ignore - object_ =_response.json() - ) - ) - return HttpResponse(response=_response, data=_data) - if _response.status_code == 422: - raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( - HttpValidationError, - parse_obj_as( - type_ =HttpValidationError, # type: ignore - object_ =_response.json() - ) - )) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) - raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def query_results(self, *, result: typing.Optional[EvaluationResultQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationResultsResponse]: """ Parameters @@ -1200,26 +1151,24 @@ def delete_result(self, result_id: str, *, request_options: typing.Optional[Requ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def edit_result(self, result_id: str, *, result: EvaluationResultEdit, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationResultResponse]: + def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsResponse]: """ Parameters ---------- - result_id : str - - result : EvaluationResultEdit + metrics : EvaluationMetricsRefresh request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[EvaluationResultResponse] + HttpResponse[EvaluationMetricsResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - f"evaluations/results/{jsonable_encoder(result_id)}",method="PATCH", + "evaluations/metrics/refresh",method="POST", json={ - "result": convert_and_respect_annotation_metadata(object_=result, annotation=EvaluationResultEdit, direction="write"), + "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=EvaluationMetricsRefresh, direction="write"), } , headers={"content-type": "application/json", } @@ -1229,9 +1178,9 @@ def edit_result(self, result_id: str, *, result: EvaluationResultEdit, request_o try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationResultResponse, + EvaluationMetricsResponse, parse_obj_as( - type_ =EvaluationResultResponse, # type: ignore + type_ =EvaluationMetricsResponse, # type: ignore object_ =_response.json() ) ) @@ -1249,11 +1198,11 @@ def edit_result(self, result_id: str, *, result: EvaluationResultEdit, request_o raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsResponse]: + def set_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsResponse]: """ Parameters ---------- - metrics : EvaluationMetricsRefresh + metrics : typing.Sequence[EvaluationMetricsCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -1264,9 +1213,9 @@ def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_options: Successful Response """ _response = self._client_wrapper.httpx_client.request( - "evaluations/metrics/refresh",method="POST", + "evaluations/metrics/",method="POST", json={ - "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=EvaluationMetricsRefresh, direction="write"), + "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=typing.Sequence[EvaluationMetricsCreate], direction="write"), } , headers={"content-type": "application/json", } @@ -1296,24 +1245,24 @@ def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_options: raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def create_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsResponse]: + def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsIdsResponse]: """ Parameters ---------- - metrics : typing.Sequence[EvaluationMetricsCreate] + metrics_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[EvaluationMetricsResponse] + HttpResponse[EvaluationMetricsIdsResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - "evaluations/metrics/",method="POST", + "evaluations/metrics/",method="DELETE", json={ - "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=typing.Sequence[EvaluationMetricsCreate], direction="write"), + "metrics_ids": metrics_ids, } , headers={"content-type": "application/json", } @@ -1323,9 +1272,9 @@ def create_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], r try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationMetricsResponse, + EvaluationMetricsIdsResponse, parse_obj_as( - type_ =EvaluationMetricsResponse, # type: ignore + type_ =EvaluationMetricsIdsResponse, # type: ignore object_ =_response.json() ) ) @@ -1343,24 +1292,27 @@ def create_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], r raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsIdsResponse]: + def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsResponse]: """ Parameters ---------- - metrics_ids : typing.Sequence[str] + metrics : typing.Optional[EvaluationMetricsQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[EvaluationMetricsIdsResponse] + HttpResponse[EvaluationMetricsResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - "evaluations/metrics/",method="DELETE", + "evaluations/metrics/query",method="POST", json={ - "metrics_ids": metrics_ids, + "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=typing.Optional[EvaluationMetricsQuery], direction="write"), + "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), } , headers={"content-type": "application/json", } @@ -1370,9 +1322,9 @@ def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_options: try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationMetricsIdsResponse, + EvaluationMetricsResponse, parse_obj_as( - type_ =EvaluationMetricsIdsResponse, # type: ignore + type_ =EvaluationMetricsResponse, # type: ignore object_ =_response.json() ) ) @@ -1390,11 +1342,11 @@ def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_options: raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def edit_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsEdit], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsResponse]: + def fetch_metric(self, metrics_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsResponse]: """ Parameters ---------- - metrics : typing.Sequence[EvaluationMetricsEdit] + metrics_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. @@ -1405,15 +1357,8 @@ def edit_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsEdit], reque Successful Response """ _response = self._client_wrapper.httpx_client.request( - "evaluations/metrics/",method="PATCH", - json={ - "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=typing.Sequence[EvaluationMetricsEdit], direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"evaluations/metrics/{jsonable_encoder(metrics_id)}",method="GET", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( @@ -1437,39 +1382,29 @@ def edit_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsEdit], reque raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsResponse]: + def delete_metric(self, metrics_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationMetricsIdsResponse]: """ Parameters ---------- - metrics : typing.Optional[EvaluationMetricsQuery] - - windowing : typing.Optional[Windowing] + metrics_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[EvaluationMetricsResponse] + HttpResponse[EvaluationMetricsIdsResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - "evaluations/metrics/query",method="POST", - json={ - "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=typing.Optional[EvaluationMetricsQuery], direction="write"), - "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"evaluations/metrics/{jsonable_encoder(metrics_id)}",method="DELETE", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationMetricsResponse, + EvaluationMetricsIdsResponse, parse_obj_as( - type_ =EvaluationMetricsResponse, # type: ignore + type_ =EvaluationMetricsIdsResponse, # type: ignore object_ =_response.json() ) ) @@ -1909,29 +1844,39 @@ def create_simple_evaluation(self, *, evaluation: SimpleEvaluationCreate, reques raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleEvaluationResponse]: + def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEvaluationQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleEvaluationsResponse]: """ Parameters ---------- - evaluation_id : str + evaluation : typing.Optional[SimpleEvaluationQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[SimpleEvaluationResponse] + HttpResponse[SimpleEvaluationsResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="GET", - request_options=request_options,) + "simple/evaluations/query",method="POST", + json={ + "evaluation": convert_and_respect_annotation_metadata(object_=evaluation, annotation=typing.Optional[SimpleEvaluationQuery], direction="write"), + "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationResponse, + SimpleEvaluationsResponse, parse_obj_as( - type_ =SimpleEvaluationResponse, # type: ignore + type_ =SimpleEvaluationsResponse, # type: ignore object_ =_response.json() ) ) @@ -1949,7 +1894,7 @@ def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: typing raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleEvaluationIdResponse]: + def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleEvaluationResponse]: """ Parameters ---------- @@ -1960,18 +1905,18 @@ def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typin Returns ------- - HttpResponse[SimpleEvaluationIdResponse] + HttpResponse[SimpleEvaluationResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="DELETE", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="GET", request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationIdResponse, + SimpleEvaluationResponse, parse_obj_as( - type_ =SimpleEvaluationIdResponse, # type: ignore + type_ =SimpleEvaluationResponse, # type: ignore object_ =_response.json() ) ) @@ -1989,38 +1934,29 @@ def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typin raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: SimpleEvaluationEdit, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleEvaluationResponse]: + def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleEvaluationIdResponse]: """ Parameters ---------- evaluation_id : str - evaluation : SimpleEvaluationEdit - request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[SimpleEvaluationResponse] + HttpResponse[SimpleEvaluationIdResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="PATCH", - json={ - "evaluation": convert_and_respect_annotation_metadata(object_=evaluation, annotation=SimpleEvaluationEdit, direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="DELETE", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationResponse, + SimpleEvaluationIdResponse, parse_obj_as( - type_ =SimpleEvaluationResponse, # type: ignore + type_ =SimpleEvaluationIdResponse, # type: ignore object_ =_response.json() ) ) @@ -2038,27 +1974,26 @@ def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: SimpleEvalua raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEvaluationQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleEvaluationsResponse]: + def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: SimpleEvaluationEdit, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleEvaluationResponse]: """ Parameters ---------- - evaluation : typing.Optional[SimpleEvaluationQuery] + evaluation_id : str - windowing : typing.Optional[Windowing] + evaluation : SimpleEvaluationEdit request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[SimpleEvaluationsResponse] + HttpResponse[SimpleEvaluationResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - "simple/evaluations/query",method="POST", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="PATCH", json={ - "evaluation": convert_and_respect_annotation_metadata(object_=evaluation, annotation=typing.Optional[SimpleEvaluationQuery], direction="write"), - "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), + "evaluation": convert_and_respect_annotation_metadata(object_=evaluation, annotation=SimpleEvaluationEdit, direction="write"), } , headers={"content-type": "application/json", } @@ -2068,9 +2003,9 @@ def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEvaluati try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationsResponse, + SimpleEvaluationResponse, parse_obj_as( - type_ =SimpleEvaluationsResponse, # type: ignore + type_ =SimpleEvaluationResponse, # type: ignore object_ =_response.json() ) ) @@ -2248,24 +2183,26 @@ def open_simple_evaluation(self, evaluation_id: str, *, request_options: typing. raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def create_simple_queue(self, *, queue: SimpleQueueCreate, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueueResponse]: + def populate_slice(self, evaluation_id: str, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationResultsResponse]: """ Parameters ---------- - queue : SimpleQueueCreate + evaluation_id : str + + results : typing.Sequence[EvaluationResultCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[SimpleQueueResponse] + HttpResponse[EvaluationResultsResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - "simple/queues/",method="POST", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/populate",method="POST", json={ - "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=SimpleQueueCreate, direction="write"), + "results": convert_and_respect_annotation_metadata(object_=results, annotation=typing.Sequence[EvaluationResultCreate], direction="write"), } , headers={"content-type": "application/json", } @@ -2275,9 +2212,9 @@ def create_simple_queue(self, *, queue: SimpleQueueCreate, request_options: typi try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleQueueResponse, + EvaluationResultsResponse, parse_obj_as( - type_ =SimpleQueueResponse, # type: ignore + type_ =EvaluationResultsResponse, # type: ignore object_ =_response.json() ) ) @@ -2295,27 +2232,35 @@ def create_simple_queue(self, *, queue: SimpleQueueCreate, request_options: typi raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def query_simple_queues(self, *, queue: typing.Optional[SimpleQueueQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueuesResponse]: + def process_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, overwrite: typing.Optional[bool] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[typing.Any]: """ Parameters ---------- - queue : typing.Optional[SimpleQueueQuery] + evaluation_id : str - windowing : typing.Optional[Windowing] + scenario_ids : typing.Optional[typing.Sequence[str]] + + step_keys : typing.Optional[typing.Sequence[str]] + + repeat_idxs : typing.Optional[typing.Sequence[int]] + + overwrite : typing.Optional[bool] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[SimpleQueuesResponse] - Successful Response + HttpResponse[typing.Any] + Accepted. """ _response = self._client_wrapper.httpx_client.request( - "simple/queues/query",method="POST", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/process",method="POST", json={ - "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=typing.Optional[SimpleQueueQuery], direction="write"), - "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), + "scenario_ids": scenario_ids, + "step_keys": step_keys, + "repeat_idxs": repeat_idxs, + "overwrite": overwrite, } , headers={"content-type": "application/json", } @@ -2323,11 +2268,13 @@ def query_simple_queues(self, *, queue: typing.Optional[SimpleQueueQuery] = OMIT request_options=request_options,omit=OMIT, ) try: + if _response is None or not _response.text.strip(): + return HttpResponse(response=_response, data=None) if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleQueuesResponse, + typing.Any, parse_obj_as( - type_ =SimpleQueuesResponse, # type: ignore + type_ =typing.Any, # type: ignore object_ =_response.json() ) ) @@ -2345,29 +2292,44 @@ def query_simple_queues(self, *, queue: typing.Optional[SimpleQueueQuery] = OMIT raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def fetch_simple_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueueResponse]: + def probe_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationResultsResponse]: """ Parameters ---------- - queue_id : str + evaluation_id : str + + scenario_ids : typing.Optional[typing.Sequence[str]] + + step_keys : typing.Optional[typing.Sequence[str]] + + repeat_idxs : typing.Optional[typing.Sequence[int]] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[SimpleQueueResponse] + HttpResponse[EvaluationResultsResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - f"simple/queues/{jsonable_encoder(queue_id)}",method="GET", - request_options=request_options,) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/probe",method="POST", + json={ + "scenario_ids": scenario_ids, + "step_keys": step_keys, + "repeat_idxs": repeat_idxs, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleQueueResponse, + EvaluationResultsResponse, parse_obj_as( - type_ =SimpleQueueResponse, # type: ignore + type_ =EvaluationResultsResponse, # type: ignore object_ =_response.json() ) ) @@ -2385,32 +2347,31 @@ def fetch_simple_queue(self, queue_id: str, *, request_options: typing.Optional[ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def query_simple_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[SimpleQueueScenariosQuery] = OMIT, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueueScenariosResponse]: + def prune_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[None]: """ Parameters ---------- - queue_id : str + evaluation_id : str - queue : typing.Optional[SimpleQueueScenariosQuery] + scenario_ids : typing.Optional[typing.Sequence[str]] - scenario : typing.Optional[EvaluationScenarioQuery] + step_keys : typing.Optional[typing.Sequence[str]] - windowing : typing.Optional[Windowing] + repeat_idxs : typing.Optional[typing.Sequence[int]] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[SimpleQueueScenariosResponse] - Successful Response + HttpResponse[None] """ _response = self._client_wrapper.httpx_client.request( - f"simple/queues/{jsonable_encoder(queue_id)}/scenarios/query",method="POST", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/prune",method="POST", json={ - "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=typing.Optional[SimpleQueueScenariosQuery], direction="write"), - "scenario": convert_and_respect_annotation_metadata(object_=scenario, annotation=typing.Optional[EvaluationScenarioQuery], direction="write"), - "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), + "scenario_ids": scenario_ids, + "step_keys": step_keys, + "repeat_idxs": repeat_idxs, } , headers={"content-type": "application/json", } @@ -2419,14 +2380,7 @@ def query_simple_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[ ) try: if 200 <= _response.status_code < 300: - _data = typing.cast( - SimpleQueueScenariosResponse, - parse_obj_as( - type_ =SimpleQueueScenariosResponse, # type: ignore - object_ =_response.json() - ) - ) - return HttpResponse(response=_response, data=_data) + return HttpResponse(response=_response, data=None) if _response.status_code == 422: raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( HttpValidationError, @@ -2440,26 +2394,29 @@ def query_simple_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def add_simple_queue_traces(self, queue_id: str, *, trace_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueueIdResponse]: + def add_scenarios(self, evaluation_id: str, *, count: int, timestamp: typing.Optional[dt.datetime] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationScenariosResponse]: """ Parameters ---------- - queue_id : str + evaluation_id : str - trace_ids : typing.Sequence[str] + count : int + + timestamp : typing.Optional[dt.datetime] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[SimpleQueueIdResponse] + HttpResponse[EvaluationScenariosResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - f"simple/queues/{jsonable_encoder(queue_id)}/traces/",method="POST", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/scenarios/add",method="POST", json={ - "trace_ids": trace_ids, + "count": count, + "timestamp": timestamp, } , headers={"content-type": "application/json", } @@ -2469,9 +2426,9 @@ def add_simple_queue_traces(self, queue_id: str, *, trace_ids: typing.Sequence[s try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleQueueIdResponse, + EvaluationScenariosResponse, parse_obj_as( - type_ =SimpleQueueIdResponse, # type: ignore + type_ =EvaluationScenariosResponse, # type: ignore object_ =_response.json() ) ) @@ -2489,26 +2446,67 @@ def add_simple_queue_traces(self, queue_id: str, *, trace_ids: typing.Sequence[s raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - def add_simple_queue_testcases(self, queue_id: str, *, testcase_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueueIdResponse]: + def remove_scenarios(self, evaluation_id: str, *, scenario_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[None]: """ Parameters ---------- - queue_id : str + evaluation_id : str - testcase_ids : typing.Sequence[str] + scenario_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - HttpResponse[SimpleQueueIdResponse] + HttpResponse[None] + """ + _response = self._client_wrapper.httpx_client.request( + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/scenarios/remove",method="POST", + json={ + "scenario_ids": scenario_ids, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + return HttpResponse(response=_response, data=None) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + def add_steps(self, evaluation_id: str, *, steps: typing.Sequence[EvaluationRunDataStep], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationRunResponse]: + """ + Parameters + ---------- + evaluation_id : str + + steps : typing.Sequence[EvaluationRunDataStep] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[EvaluationRunResponse] Successful Response """ _response = self._client_wrapper.httpx_client.request( - f"simple/queues/{jsonable_encoder(queue_id)}/testcases/",method="POST", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/steps/add",method="POST", json={ - "testcase_ids": testcase_ids, + "steps": convert_and_respect_annotation_metadata(object_=steps, annotation=typing.Sequence[EvaluationRunDataStep], direction="write"), } , headers={"content-type": "application/json", } @@ -2518,9 +2516,9 @@ def add_simple_queue_testcases(self, queue_id: str, *, testcase_ids: typing.Sequ try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleQueueIdResponse, + EvaluationRunResponse, parse_obj_as( - type_ =SimpleQueueIdResponse, # type: ignore + type_ =EvaluationRunResponse, # type: ignore object_ =_response.json() ) ) @@ -2537,28 +2535,27 @@ def add_simple_queue_testcases(self, queue_id: str, *, testcase_ids: typing.Sequ except JSONDecodeError: raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) -class AsyncRawEvaluationsClient: - def __init__(self, *, client_wrapper: AsyncClientWrapper): - self._client_wrapper = client_wrapper - async def create_runs(self, *, runs: typing.Sequence[EvaluationRunCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunsResponse]: + def remove_steps(self, evaluation_id: str, *, step_keys: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationRunResponse]: """ Parameters ---------- - runs : typing.Sequence[EvaluationRunCreate] + evaluation_id : str + + step_keys : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunsResponse] + HttpResponse[EvaluationRunResponse] Successful Response """ - _response = await self._client_wrapper.httpx_client.request( - "evaluations/runs/",method="POST", + _response = self._client_wrapper.httpx_client.request( + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/steps/remove",method="POST", json={ - "runs": convert_and_respect_annotation_metadata(object_=runs, annotation=typing.Sequence[EvaluationRunCreate], direction="write"), + "step_keys": step_keys, } , headers={"content-type": "application/json", } @@ -2568,13 +2565,13 @@ async def create_runs(self, *, runs: typing.Sequence[EvaluationRunCreate], reque try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunsResponse, + EvaluationRunResponse, parse_obj_as( - type_ =EvaluationRunsResponse, # type: ignore + type_ =EvaluationRunResponse, # type: ignore object_ =_response.json() ) ) - return AsyncHttpResponse(response=_response, data=_data) + return HttpResponse(response=_response, data=_data) if _response.status_code == 422: raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( HttpValidationError, @@ -2588,24 +2585,26 @@ async def create_runs(self, *, runs: typing.Sequence[EvaluationRunCreate], reque raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def delete_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunIdsResponse]: + def set_repeats(self, evaluation_id: str, *, repeats: int, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[EvaluationRunResponse]: """ Parameters ---------- - run_ids : typing.Sequence[str] + evaluation_id : str + + repeats : int request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunIdsResponse] + HttpResponse[EvaluationRunResponse] Successful Response """ - _response = await self._client_wrapper.httpx_client.request( - "evaluations/runs/",method="DELETE", + _response = self._client_wrapper.httpx_client.request( + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/repeats/set",method="POST", json={ - "run_ids": run_ids, + "repeats": repeats, } , headers={"content-type": "application/json", } @@ -2615,13 +2614,13 @@ async def delete_runs(self, *, run_ids: typing.Sequence[str], request_options: t try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunIdsResponse, + EvaluationRunResponse, parse_obj_as( - type_ =EvaluationRunIdsResponse, # type: ignore + type_ =EvaluationRunResponse, # type: ignore object_ =_response.json() ) ) - return AsyncHttpResponse(response=_response, data=_data) + return HttpResponse(response=_response, data=_data) if _response.status_code == 422: raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( HttpValidationError, @@ -2635,24 +2634,24 @@ async def delete_runs(self, *, run_ids: typing.Sequence[str], request_options: t raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def edit_runs(self, *, runs: typing.Sequence[EvaluationRunEdit], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunsResponse]: + def create_simple_queue(self, *, queue: SimpleQueueCreate, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueueResponse]: """ Parameters ---------- - runs : typing.Sequence[EvaluationRunEdit] + queue : SimpleQueueCreate request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunsResponse] + HttpResponse[SimpleQueueResponse] Successful Response """ - _response = await self._client_wrapper.httpx_client.request( - "evaluations/runs/",method="PATCH", + _response = self._client_wrapper.httpx_client.request( + "simple/queues/",method="POST", json={ - "runs": convert_and_respect_annotation_metadata(object_=runs, annotation=typing.Sequence[EvaluationRunEdit], direction="write"), + "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=SimpleQueueCreate, direction="write"), } , headers={"content-type": "application/json", } @@ -2662,13 +2661,13 @@ async def edit_runs(self, *, runs: typing.Sequence[EvaluationRunEdit], request_o try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunsResponse, + SimpleQueueResponse, parse_obj_as( - type_ =EvaluationRunsResponse, # type: ignore + type_ =SimpleQueueResponse, # type: ignore object_ =_response.json() ) ) - return AsyncHttpResponse(response=_response, data=_data) + return HttpResponse(response=_response, data=_data) if _response.status_code == 422: raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( HttpValidationError, @@ -2682,11 +2681,11 @@ async def edit_runs(self, *, runs: typing.Sequence[EvaluationRunEdit], request_o raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def query_runs(self, *, run: typing.Optional[EvaluationRunQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunsResponse]: + def query_simple_queues(self, *, queue: typing.Optional[SimpleQueueQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueuesResponse]: """ Parameters ---------- - run : typing.Optional[EvaluationRunQuery] + queue : typing.Optional[SimpleQueueQuery] windowing : typing.Optional[Windowing] @@ -2695,13 +2694,13 @@ async def query_runs(self, *, run: typing.Optional[EvaluationRunQuery] = OMIT, w Returns ------- - AsyncHttpResponse[EvaluationRunsResponse] + HttpResponse[SimpleQueuesResponse] Successful Response """ - _response = await self._client_wrapper.httpx_client.request( - "evaluations/runs/query",method="POST", + _response = self._client_wrapper.httpx_client.request( + "simple/queues/query",method="POST", json={ - "run": convert_and_respect_annotation_metadata(object_=run, annotation=typing.Optional[EvaluationRunQuery], direction="write"), + "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=typing.Optional[SimpleQueueQuery], direction="write"), "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), } , @@ -2712,13 +2711,13 @@ async def query_runs(self, *, run: typing.Optional[EvaluationRunQuery] = OMIT, w try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunsResponse, + SimpleQueuesResponse, parse_obj_as( - type_ =EvaluationRunsResponse, # type: ignore + type_ =SimpleQueuesResponse, # type: ignore object_ =_response.json() ) ) - return AsyncHttpResponse(response=_response, data=_data) + return HttpResponse(response=_response, data=_data) if _response.status_code == 422: raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( HttpValidationError, @@ -2732,24 +2731,72 @@ async def query_runs(self, *, run: typing.Optional[EvaluationRunQuery] = OMIT, w raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def close_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunsResponse]: + def fetch_simple_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueueResponse]: """ Parameters ---------- - run_ids : typing.Sequence[str] + queue_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunsResponse] + HttpResponse[SimpleQueueResponse] Successful Response """ - _response = await self._client_wrapper.httpx_client.request( - "evaluations/runs/close",method="POST", + _response = self._client_wrapper.httpx_client.request( + f"simple/queues/{jsonable_encoder(queue_id)}",method="GET", + request_options=request_options,) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + SimpleQueueResponse, + parse_obj_as( + type_ =SimpleQueueResponse, # type: ignore + object_ =_response.json() + ) + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + def query_simple_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[SimpleQueueScenariosQuery] = OMIT, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueueScenariosResponse]: + """ + Parameters + ---------- + queue_id : str + + queue : typing.Optional[SimpleQueueScenariosQuery] + + scenario : typing.Optional[EvaluationScenarioQuery] + + windowing : typing.Optional[Windowing] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[SimpleQueueScenariosResponse] + Successful Response + """ + _response = self._client_wrapper.httpx_client.request( + f"simple/queues/{jsonable_encoder(queue_id)}/scenarios/query",method="POST", json={ - "run_ids": run_ids, + "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=typing.Optional[SimpleQueueScenariosQuery], direction="write"), + "scenario": convert_and_respect_annotation_metadata(object_=scenario, annotation=typing.Optional[EvaluationScenarioQuery], direction="write"), + "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), } , headers={"content-type": "application/json", } @@ -2759,13 +2806,13 @@ async def close_runs(self, *, run_ids: typing.Sequence[str], request_options: ty try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunsResponse, + SimpleQueueScenariosResponse, parse_obj_as( - type_ =EvaluationRunsResponse, # type: ignore + type_ =SimpleQueueScenariosResponse, # type: ignore object_ =_response.json() ) ) - return AsyncHttpResponse(response=_response, data=_data) + return HttpResponse(response=_response, data=_data) if _response.status_code == 422: raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( HttpValidationError, @@ -2779,24 +2826,26 @@ async def close_runs(self, *, run_ids: typing.Sequence[str], request_options: ty raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def open_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunsResponse]: + def add_simple_queue_traces(self, queue_id: str, *, trace_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueueIdResponse]: """ Parameters ---------- - run_ids : typing.Sequence[str] + queue_id : str + + trace_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunsResponse] + HttpResponse[SimpleQueueIdResponse] Successful Response """ - _response = await self._client_wrapper.httpx_client.request( - "evaluations/runs/open",method="POST", + _response = self._client_wrapper.httpx_client.request( + f"simple/queues/{jsonable_encoder(queue_id)}/traces/",method="POST", json={ - "run_ids": run_ids, + "trace_ids": trace_ids, } , headers={"content-type": "application/json", } @@ -2806,13 +2855,13 @@ async def open_runs(self, *, run_ids: typing.Sequence[str], request_options: typ try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunsResponse, + SimpleQueueIdResponse, parse_obj_as( - type_ =EvaluationRunsResponse, # type: ignore + type_ =SimpleQueueIdResponse, # type: ignore object_ =_response.json() ) ) - return AsyncHttpResponse(response=_response, data=_data) + return HttpResponse(response=_response, data=_data) if _response.status_code == 422: raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( HttpValidationError, @@ -2826,29 +2875,720 @@ async def open_runs(self, *, run_ids: typing.Sequence[str], request_options: typ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def fetch_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: + def add_simple_queue_testcases(self, queue_id: str, *, testcase_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> HttpResponse[SimpleQueueIdResponse]: """ Parameters ---------- - run_id : str + queue_id : str + + testcase_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunResponse] + HttpResponse[SimpleQueueIdResponse] Successful Response """ - _response = await self._client_wrapper.httpx_client.request( - f"evaluations/runs/{jsonable_encoder(run_id)}",method="GET", - request_options=request_options,) + _response = self._client_wrapper.httpx_client.request( + f"simple/queues/{jsonable_encoder(queue_id)}/testcases/",method="POST", + json={ + "testcase_ids": testcase_ids, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + SimpleQueueIdResponse, + parse_obj_as( + type_ =SimpleQueueIdResponse, # type: ignore + object_ =_response.json() + ) + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) +class AsyncRawEvaluationsClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._client_wrapper = client_wrapper + + async def create_runs(self, *, runs: typing.Sequence[EvaluationRunCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunsResponse]: + """ + Parameters + ---------- + runs : typing.Sequence[EvaluationRunCreate] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunsResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + "evaluations/runs/",method="POST", + json={ + "runs": convert_and_respect_annotation_metadata(object_=runs, annotation=typing.Sequence[EvaluationRunCreate], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunsResponse, + parse_obj_as( + type_ =EvaluationRunsResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def delete_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunIdsResponse]: + """ + Parameters + ---------- + run_ids : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunIdsResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + "evaluations/runs/",method="DELETE", + json={ + "run_ids": run_ids, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunIdsResponse, + parse_obj_as( + type_ =EvaluationRunIdsResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def edit_runs(self, *, runs: typing.Sequence[EvaluationRunEdit], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunsResponse]: + """ + Parameters + ---------- + runs : typing.Sequence[EvaluationRunEdit] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunsResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + "evaluations/runs/",method="PATCH", + json={ + "runs": convert_and_respect_annotation_metadata(object_=runs, annotation=typing.Sequence[EvaluationRunEdit], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunsResponse, + parse_obj_as( + type_ =EvaluationRunsResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def query_runs(self, *, run: typing.Optional[EvaluationRunQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunsResponse]: + """ + Parameters + ---------- + run : typing.Optional[EvaluationRunQuery] + + windowing : typing.Optional[Windowing] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunsResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + "evaluations/runs/query",method="POST", + json={ + "run": convert_and_respect_annotation_metadata(object_=run, annotation=typing.Optional[EvaluationRunQuery], direction="write"), + "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunsResponse, + parse_obj_as( + type_ =EvaluationRunsResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def close_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunsResponse]: + """ + Parameters + ---------- + run_ids : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunsResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + "evaluations/runs/close",method="POST", + json={ + "run_ids": run_ids, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunsResponse, + parse_obj_as( + type_ =EvaluationRunsResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def open_runs(self, *, run_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunsResponse]: + """ + Parameters + ---------- + run_ids : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunsResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + "evaluations/runs/open",method="POST", + json={ + "run_ids": run_ids, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunsResponse, + parse_obj_as( + type_ =EvaluationRunsResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def fetch_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: + """ + Parameters + ---------- + run_id : str + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + f"evaluations/runs/{jsonable_encoder(run_id)}",method="GET", + request_options=request_options,) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunResponse, + parse_obj_as( + type_ =EvaluationRunResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def delete_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunIdResponse]: + """ + Parameters + ---------- + run_id : str + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunIdResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + f"evaluations/runs/{jsonable_encoder(run_id)}",method="DELETE", + request_options=request_options,) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunIdResponse, + parse_obj_as( + type_ =EvaluationRunIdResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def edit_run(self, run_id: str, *, run: EvaluationRunEdit, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: + """ + Parameters + ---------- + run_id : str + + run : EvaluationRunEdit + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + f"evaluations/runs/{jsonable_encoder(run_id)}",method="PATCH", + json={ + "run": convert_and_respect_annotation_metadata(object_=run, annotation=EvaluationRunEdit, direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunResponse, + parse_obj_as( + type_ =EvaluationRunResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def close_run(self, run_id: str, *, status: typing.Optional[EvaluationStatus] = None, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: + """ + Parameters + ---------- + run_id : str + + status : typing.Optional[EvaluationStatus] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + f"evaluations/runs/{jsonable_encoder(run_id)}/close",method="POST", + params={"status": status, } + , + request_options=request_options,) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunResponse, + parse_obj_as( + type_ =EvaluationRunResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: + """ + Parameters + ---------- + run_id : str + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRunResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + f"evaluations/runs/{jsonable_encoder(run_id)}/open",method="POST", + request_options=request_options,) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRunResponse, + parse_obj_as( + type_ =EvaluationRunResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def fetch_default_queue(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueueResponse]: + """ + Parameters + ---------- + run_id : str + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationQueueResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + f"evaluations/runs/{jsonable_encoder(run_id)}/queues/default",method="GET", + request_options=request_options,) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationQueueResponse, + parse_obj_as( + type_ =EvaluationQueueResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def create_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenariosResponse]: + """ + Parameters + ---------- + scenarios : typing.Sequence[EvaluationScenarioCreate] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationScenariosResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + "evaluations/scenarios/",method="POST", + json={ + "scenarios": convert_and_respect_annotation_metadata(object_=scenarios, annotation=typing.Sequence[EvaluationScenarioCreate], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationScenariosResponse, + parse_obj_as( + type_ =EvaluationScenariosResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def delete_scenarios(self, *, scenario_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenarioIdsResponse]: + """ + Parameters + ---------- + scenario_ids : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationScenarioIdsResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + "evaluations/scenarios/",method="DELETE", + json={ + "scenario_ids": scenario_ids, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationScenarioIdsResponse, + parse_obj_as( + type_ =EvaluationScenarioIdsResponse, # type: ignore + object_ =_response.json() + ) + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 422: + raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( + HttpValidationError, + parse_obj_as( + type_ =HttpValidationError, # type: ignore + object_ =_response.json() + ) + )) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def edit_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioEdit], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenariosResponse]: + """ + Parameters + ---------- + scenarios : typing.Sequence[EvaluationScenarioEdit] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationScenariosResponse] + Successful Response + """ + _response = await self._client_wrapper.httpx_client.request( + "evaluations/scenarios/",method="PATCH", + json={ + "scenarios": convert_and_respect_annotation_metadata(object_=scenarios, annotation=typing.Sequence[EvaluationScenarioEdit], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunResponse, + EvaluationScenariosResponse, parse_obj_as( - type_ =EvaluationRunResponse, # type: ignore + type_ =EvaluationScenariosResponse, # type: ignore object_ =_response.json() ) ) @@ -2866,29 +3606,39 @@ async def fetch_run(self, run_id: str, *, request_options: typing.Optional[Reque raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def delete_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunIdResponse]: + async def query_scenarios(self, *, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenariosResponse]: """ Parameters ---------- - run_id : str + scenario : typing.Optional[EvaluationScenarioQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunIdResponse] + AsyncHttpResponse[EvaluationScenariosResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/runs/{jsonable_encoder(run_id)}",method="DELETE", - request_options=request_options,) + "evaluations/scenarios/query",method="POST", + json={ + "scenario": convert_and_respect_annotation_metadata(object_=scenario, annotation=typing.Optional[EvaluationScenarioQuery], direction="write"), + "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunIdResponse, + EvaluationScenariosResponse, parse_obj_as( - type_ =EvaluationRunIdResponse, # type: ignore + type_ =EvaluationScenariosResponse, # type: ignore object_ =_response.json() ) ) @@ -2906,38 +3656,29 @@ async def delete_run(self, run_id: str, *, request_options: typing.Optional[Requ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def edit_run(self, run_id: str, *, run: EvaluationRunEdit, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: + async def fetch_scenario(self, scenario_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenarioResponse]: """ Parameters ---------- - run_id : str - - run : EvaluationRunEdit + scenario_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunResponse] + AsyncHttpResponse[EvaluationScenarioResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/runs/{jsonable_encoder(run_id)}",method="PATCH", - json={ - "run": convert_and_respect_annotation_metadata(object_=run, annotation=EvaluationRunEdit, direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"evaluations/scenarios/{jsonable_encoder(scenario_id)}",method="GET", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunResponse, + EvaluationScenarioResponse, parse_obj_as( - type_ =EvaluationRunResponse, # type: ignore + type_ =EvaluationScenarioResponse, # type: ignore object_ =_response.json() ) ) @@ -2955,33 +3696,29 @@ async def edit_run(self, run_id: str, *, run: EvaluationRunEdit, request_options raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def close_run(self, run_id: str, *, status: typing.Optional[EvaluationStatus] = None, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: + async def delete_scenario(self, scenario_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenarioIdResponse]: """ Parameters ---------- - run_id : str - - status : typing.Optional[EvaluationStatus] + scenario_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunResponse] + AsyncHttpResponse[EvaluationScenarioIdResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/runs/{jsonable_encoder(run_id)}/close",method="POST", - params={"status": status, } - , + f"evaluations/scenarios/{jsonable_encoder(scenario_id)}",method="DELETE", request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunResponse, + EvaluationScenarioIdResponse, parse_obj_as( - type_ =EvaluationRunResponse, # type: ignore + type_ =EvaluationScenarioIdResponse, # type: ignore object_ =_response.json() ) ) @@ -2999,31 +3736,38 @@ async def close_run(self, run_id: str, *, status: typing.Optional[EvaluationStat raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def close_run_with_status(self, run_id: str, status: typing.Optional[EvaluationStatus], *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: + async def edit_scenario(self, scenario_id: str, *, scenario: EvaluationScenarioEdit, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenarioResponse]: """ Parameters ---------- - run_id : str + scenario_id : str - status : typing.Optional[EvaluationStatus] + scenario : EvaluationScenarioEdit request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunResponse] + AsyncHttpResponse[EvaluationScenarioResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/runs/{jsonable_encoder(run_id)}/close/{jsonable_encoder(status)}",method="POST", - request_options=request_options,) + f"evaluations/scenarios/{jsonable_encoder(scenario_id)}",method="PATCH", + json={ + "scenario": convert_and_respect_annotation_metadata(object_=scenario, annotation=EvaluationScenarioEdit, direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunResponse, + EvaluationScenarioResponse, parse_obj_as( - type_ =EvaluationRunResponse, # type: ignore + type_ =EvaluationScenarioResponse, # type: ignore object_ =_response.json() ) ) @@ -3041,29 +3785,36 @@ async def close_run_with_status(self, run_id: str, status: typing.Optional[Evalu raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def open_run(self, run_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: + async def set_results(self, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultsResponse]: """ Parameters ---------- - run_id : str + results : typing.Sequence[EvaluationResultCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationRunResponse] + AsyncHttpResponse[EvaluationResultsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/runs/{jsonable_encoder(run_id)}/open",method="POST", - request_options=request_options,) + "evaluations/results/",method="POST", + json={ + "results": convert_and_respect_annotation_metadata(object_=results, annotation=typing.Sequence[EvaluationResultCreate], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationRunResponse, + EvaluationResultsResponse, parse_obj_as( - type_ =EvaluationRunResponse, # type: ignore + type_ =EvaluationResultsResponse, # type: ignore object_ =_response.json() ) ) @@ -3081,24 +3832,24 @@ async def open_run(self, run_id: str, *, request_options: typing.Optional[Reques raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def create_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenariosResponse]: + async def delete_results(self, *, result_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultIdsResponse]: """ Parameters ---------- - scenarios : typing.Sequence[EvaluationScenarioCreate] + result_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationScenariosResponse] + AsyncHttpResponse[EvaluationResultIdsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/scenarios/",method="POST", + "evaluations/results/",method="DELETE", json={ - "scenarios": convert_and_respect_annotation_metadata(object_=scenarios, annotation=typing.Sequence[EvaluationScenarioCreate], direction="write"), + "result_ids": result_ids, } , headers={"content-type": "application/json", } @@ -3108,9 +3859,9 @@ async def create_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenari try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationScenariosResponse, + EvaluationResultIdsResponse, parse_obj_as( - type_ =EvaluationScenariosResponse, # type: ignore + type_ =EvaluationResultIdsResponse, # type: ignore object_ =_response.json() ) ) @@ -3128,24 +3879,27 @@ async def create_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenari raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def delete_scenarios(self, *, scenario_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenarioIdsResponse]: + async def query_results(self, *, result: typing.Optional[EvaluationResultQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultsResponse]: """ Parameters ---------- - scenario_ids : typing.Sequence[str] + result : typing.Optional[EvaluationResultQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationScenarioIdsResponse] + AsyncHttpResponse[EvaluationResultsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/scenarios/",method="DELETE", + "evaluations/results/query",method="POST", json={ - "scenario_ids": scenario_ids, + "result": convert_and_respect_annotation_metadata(object_=result, annotation=typing.Optional[EvaluationResultQuery], direction="write"), + "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), } , headers={"content-type": "application/json", } @@ -3155,9 +3909,9 @@ async def delete_scenarios(self, *, scenario_ids: typing.Sequence[str], request_ try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationScenarioIdsResponse, + EvaluationResultsResponse, parse_obj_as( - type_ =EvaluationScenarioIdsResponse, # type: ignore + type_ =EvaluationResultsResponse, # type: ignore object_ =_response.json() ) ) @@ -3175,36 +3929,29 @@ async def delete_scenarios(self, *, scenario_ids: typing.Sequence[str], request_ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def edit_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioEdit], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenariosResponse]: + async def fetch_result(self, result_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultResponse]: """ Parameters ---------- - scenarios : typing.Sequence[EvaluationScenarioEdit] + result_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationScenariosResponse] + AsyncHttpResponse[EvaluationResultResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/scenarios/",method="PATCH", - json={ - "scenarios": convert_and_respect_annotation_metadata(object_=scenarios, annotation=typing.Sequence[EvaluationScenarioEdit], direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"evaluations/results/{jsonable_encoder(result_id)}",method="GET", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationScenariosResponse, + EvaluationResultResponse, parse_obj_as( - type_ =EvaluationScenariosResponse, # type: ignore + type_ =EvaluationResultResponse, # type: ignore object_ =_response.json() ) ) @@ -3222,39 +3969,29 @@ async def edit_scenarios(self, *, scenarios: typing.Sequence[EvaluationScenarioE raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def query_scenarios(self, *, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenariosResponse]: + async def delete_result(self, result_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultIdResponse]: """ Parameters ---------- - scenario : typing.Optional[EvaluationScenarioQuery] - - windowing : typing.Optional[Windowing] + result_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationScenariosResponse] + AsyncHttpResponse[EvaluationResultIdResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/scenarios/query",method="POST", - json={ - "scenario": convert_and_respect_annotation_metadata(object_=scenario, annotation=typing.Optional[EvaluationScenarioQuery], direction="write"), - "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"evaluations/results/{jsonable_encoder(result_id)}",method="DELETE", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationScenariosResponse, + EvaluationResultIdResponse, parse_obj_as( - type_ =EvaluationScenariosResponse, # type: ignore + type_ =EvaluationResultIdResponse, # type: ignore object_ =_response.json() ) ) @@ -3272,29 +4009,36 @@ async def query_scenarios(self, *, scenario: typing.Optional[EvaluationScenarioQ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def fetch_scenario(self, scenario_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenarioResponse]: + async def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsResponse]: """ Parameters ---------- - scenario_id : str + metrics : EvaluationMetricsRefresh request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationScenarioResponse] + AsyncHttpResponse[EvaluationMetricsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/scenarios/{jsonable_encoder(scenario_id)}",method="GET", - request_options=request_options,) + "evaluations/metrics/refresh",method="POST", + json={ + "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=EvaluationMetricsRefresh, direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationScenarioResponse, + EvaluationMetricsResponse, parse_obj_as( - type_ =EvaluationScenarioResponse, # type: ignore + type_ =EvaluationMetricsResponse, # type: ignore object_ =_response.json() ) ) @@ -3312,29 +4056,36 @@ async def fetch_scenario(self, scenario_id: str, *, request_options: typing.Opti raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def delete_scenario(self, scenario_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenarioIdResponse]: + async def set_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsResponse]: """ Parameters ---------- - scenario_id : str + metrics : typing.Sequence[EvaluationMetricsCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationScenarioIdResponse] + AsyncHttpResponse[EvaluationMetricsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/scenarios/{jsonable_encoder(scenario_id)}",method="DELETE", - request_options=request_options,) + "evaluations/metrics/",method="POST", + json={ + "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=typing.Sequence[EvaluationMetricsCreate], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationScenarioIdResponse, + EvaluationMetricsResponse, parse_obj_as( - type_ =EvaluationScenarioIdResponse, # type: ignore + type_ =EvaluationMetricsResponse, # type: ignore object_ =_response.json() ) ) @@ -3352,26 +4103,24 @@ async def delete_scenario(self, scenario_id: str, *, request_options: typing.Opt raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def edit_scenario(self, scenario_id: str, *, scenario: EvaluationScenarioEdit, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenarioResponse]: + async def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsIdsResponse]: """ Parameters ---------- - scenario_id : str - - scenario : EvaluationScenarioEdit + metrics_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationScenarioResponse] + AsyncHttpResponse[EvaluationMetricsIdsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/scenarios/{jsonable_encoder(scenario_id)}",method="PATCH", + "evaluations/metrics/",method="DELETE", json={ - "scenario": convert_and_respect_annotation_metadata(object_=scenario, annotation=EvaluationScenarioEdit, direction="write"), + "metrics_ids": metrics_ids, } , headers={"content-type": "application/json", } @@ -3381,9 +4130,9 @@ async def edit_scenario(self, scenario_id: str, *, scenario: EvaluationScenarioE try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationScenarioResponse, + EvaluationMetricsIdsResponse, parse_obj_as( - type_ =EvaluationScenarioResponse, # type: ignore + type_ =EvaluationMetricsIdsResponse, # type: ignore object_ =_response.json() ) ) @@ -3401,24 +4150,27 @@ async def edit_scenario(self, scenario_id: str, *, scenario: EvaluationScenarioE raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def create_results(self, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultsResponse]: + async def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsResponse]: """ Parameters ---------- - results : typing.Sequence[EvaluationResultCreate] + metrics : typing.Optional[EvaluationMetricsQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationResultsResponse] + AsyncHttpResponse[EvaluationMetricsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/results/",method="POST", + "evaluations/metrics/query",method="POST", json={ - "results": convert_and_respect_annotation_metadata(object_=results, annotation=typing.Sequence[EvaluationResultCreate], direction="write"), + "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=typing.Optional[EvaluationMetricsQuery], direction="write"), + "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), } , headers={"content-type": "application/json", } @@ -3428,9 +4180,9 @@ async def create_results(self, *, results: typing.Sequence[EvaluationResultCreat try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationResultsResponse, + EvaluationMetricsResponse, parse_obj_as( - type_ =EvaluationResultsResponse, # type: ignore + type_ =EvaluationMetricsResponse, # type: ignore object_ =_response.json() ) ) @@ -3448,36 +4200,29 @@ async def create_results(self, *, results: typing.Sequence[EvaluationResultCreat raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def delete_results(self, *, result_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultIdsResponse]: + async def fetch_metric(self, metrics_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsResponse]: """ Parameters ---------- - result_ids : typing.Sequence[str] + metrics_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationResultIdsResponse] + AsyncHttpResponse[EvaluationMetricsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/results/",method="DELETE", - json={ - "result_ids": result_ids, - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"evaluations/metrics/{jsonable_encoder(metrics_id)}",method="GET", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationResultIdsResponse, + EvaluationMetricsResponse, parse_obj_as( - type_ =EvaluationResultIdsResponse, # type: ignore + type_ =EvaluationMetricsResponse, # type: ignore object_ =_response.json() ) ) @@ -3495,36 +4240,29 @@ async def delete_results(self, *, result_ids: typing.Sequence[str], request_opti raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def edit_results(self, *, results: typing.Sequence[EvaluationResultEdit], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultsResponse]: + async def delete_metric(self, metrics_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsIdsResponse]: """ Parameters ---------- - results : typing.Sequence[EvaluationResultEdit] + metrics_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationResultsResponse] + AsyncHttpResponse[EvaluationMetricsIdsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/results/",method="PATCH", - json={ - "results": convert_and_respect_annotation_metadata(object_=results, annotation=typing.Sequence[EvaluationResultEdit], direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"evaluations/metrics/{jsonable_encoder(metrics_id)}",method="DELETE", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationResultsResponse, + EvaluationMetricsIdsResponse, parse_obj_as( - type_ =EvaluationResultsResponse, # type: ignore + type_ =EvaluationMetricsIdsResponse, # type: ignore object_ =_response.json() ) ) @@ -3542,27 +4280,24 @@ async def edit_results(self, *, results: typing.Sequence[EvaluationResultEdit], raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def query_results(self, *, result: typing.Optional[EvaluationResultQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultsResponse]: + async def create_queues(self, *, queues: typing.Sequence[EvaluationQueueCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueuesResponse]: """ Parameters ---------- - result : typing.Optional[EvaluationResultQuery] - - windowing : typing.Optional[Windowing] + queues : typing.Sequence[EvaluationQueueCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationResultsResponse] + AsyncHttpResponse[EvaluationQueuesResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/results/query",method="POST", + "evaluations/queues/",method="POST", json={ - "result": convert_and_respect_annotation_metadata(object_=result, annotation=typing.Optional[EvaluationResultQuery], direction="write"), - "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), + "queues": convert_and_respect_annotation_metadata(object_=queues, annotation=typing.Sequence[EvaluationQueueCreate], direction="write"), } , headers={"content-type": "application/json", } @@ -3572,9 +4307,9 @@ async def query_results(self, *, result: typing.Optional[EvaluationResultQuery] try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationResultsResponse, + EvaluationQueuesResponse, parse_obj_as( - type_ =EvaluationResultsResponse, # type: ignore + type_ =EvaluationQueuesResponse, # type: ignore object_ =_response.json() ) ) @@ -3592,29 +4327,36 @@ async def query_results(self, *, result: typing.Optional[EvaluationResultQuery] raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def fetch_result(self, result_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultResponse]: + async def delete_queues(self, *, queue_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueueIdsResponse]: """ Parameters ---------- - result_id : str + queue_ids : typing.Sequence[str] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationResultResponse] + AsyncHttpResponse[EvaluationQueueIdsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/results/{jsonable_encoder(result_id)}",method="GET", - request_options=request_options,) + "evaluations/queues/",method="DELETE", + json={ + "queue_ids": queue_ids, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationResultResponse, + EvaluationQueueIdsResponse, parse_obj_as( - type_ =EvaluationResultResponse, # type: ignore + type_ =EvaluationQueueIdsResponse, # type: ignore object_ =_response.json() ) ) @@ -3632,29 +4374,36 @@ async def fetch_result(self, result_id: str, *, request_options: typing.Optional raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def delete_result(self, result_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultIdResponse]: + async def edit_queues(self, *, queues: typing.Sequence[EvaluationQueueEdit], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueuesResponse]: """ Parameters ---------- - result_id : str + queues : typing.Sequence[EvaluationQueueEdit] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationResultIdResponse] + AsyncHttpResponse[EvaluationQueuesResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/results/{jsonable_encoder(result_id)}",method="DELETE", - request_options=request_options,) + "evaluations/queues/",method="PATCH", + json={ + "queues": convert_and_respect_annotation_metadata(object_=queues, annotation=typing.Sequence[EvaluationQueueEdit], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationResultIdResponse, + EvaluationQueuesResponse, parse_obj_as( - type_ =EvaluationResultIdResponse, # type: ignore + type_ =EvaluationQueuesResponse, # type: ignore object_ =_response.json() ) ) @@ -3672,26 +4421,27 @@ async def delete_result(self, result_id: str, *, request_options: typing.Optiona raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def edit_result(self, result_id: str, *, result: EvaluationResultEdit, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultResponse]: + async def query_queues(self, *, queue: typing.Optional[EvaluationQueueQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueuesResponse]: """ Parameters ---------- - result_id : str + queue : typing.Optional[EvaluationQueueQuery] - result : EvaluationResultEdit + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationResultResponse] + AsyncHttpResponse[EvaluationQueuesResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/results/{jsonable_encoder(result_id)}",method="PATCH", + "evaluations/queues/query",method="POST", json={ - "result": convert_and_respect_annotation_metadata(object_=result, annotation=EvaluationResultEdit, direction="write"), + "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=typing.Optional[EvaluationQueueQuery], direction="write"), + "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), } , headers={"content-type": "application/json", } @@ -3701,9 +4451,9 @@ async def edit_result(self, result_id: str, *, result: EvaluationResultEdit, req try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationResultResponse, + EvaluationQueuesResponse, parse_obj_as( - type_ =EvaluationResultResponse, # type: ignore + type_ =EvaluationQueuesResponse, # type: ignore object_ =_response.json() ) ) @@ -3721,36 +4471,29 @@ async def edit_result(self, result_id: str, *, result: EvaluationResultEdit, req raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsResponse]: + async def fetch_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueueResponse]: """ Parameters ---------- - metrics : EvaluationMetricsRefresh + queue_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationMetricsResponse] + AsyncHttpResponse[EvaluationQueueResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/metrics/refresh",method="POST", - json={ - "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=EvaluationMetricsRefresh, direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"evaluations/queues/{jsonable_encoder(queue_id)}",method="GET", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationMetricsResponse, + EvaluationQueueResponse, parse_obj_as( - type_ =EvaluationMetricsResponse, # type: ignore + type_ =EvaluationQueueResponse, # type: ignore object_ =_response.json() ) ) @@ -3768,36 +4511,29 @@ async def refresh_metrics(self, *, metrics: EvaluationMetricsRefresh, request_op raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def create_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsResponse]: + async def delete_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueueIdResponse]: """ Parameters ---------- - metrics : typing.Sequence[EvaluationMetricsCreate] + queue_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationMetricsResponse] + AsyncHttpResponse[EvaluationQueueIdResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/metrics/",method="POST", - json={ - "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=typing.Sequence[EvaluationMetricsCreate], direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"evaluations/queues/{jsonable_encoder(queue_id)}",method="DELETE", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationMetricsResponse, + EvaluationQueueIdResponse, parse_obj_as( - type_ =EvaluationMetricsResponse, # type: ignore + type_ =EvaluationQueueIdResponse, # type: ignore object_ =_response.json() ) ) @@ -3815,24 +4551,26 @@ async def create_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsCrea raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsIdsResponse]: + async def edit_queue(self, queue_id: str, *, queue: EvaluationQueueEdit, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueueResponse]: """ Parameters ---------- - metrics_ids : typing.Sequence[str] + queue_id : str + + queue : EvaluationQueueEdit request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationMetricsIdsResponse] + AsyncHttpResponse[EvaluationQueueResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/metrics/",method="DELETE", + f"evaluations/queues/{jsonable_encoder(queue_id)}",method="PATCH", json={ - "metrics_ids": metrics_ids, + "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=EvaluationQueueEdit, direction="write"), } , headers={"content-type": "application/json", } @@ -3842,9 +4580,9 @@ async def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_opt try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationMetricsIdsResponse, + EvaluationQueueResponse, parse_obj_as( - type_ =EvaluationMetricsIdsResponse, # type: ignore + type_ =EvaluationQueueResponse, # type: ignore object_ =_response.json() ) ) @@ -3862,24 +4600,32 @@ async def delete_metrics(self, *, metrics_ids: typing.Sequence[str], request_opt raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def edit_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsEdit], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsResponse]: + async def query_evaluation_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[EvaluationQueueScenariosQuery] = OMIT, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenariosResponse]: """ Parameters ---------- - metrics : typing.Sequence[EvaluationMetricsEdit] + queue_id : str + + queue : typing.Optional[EvaluationQueueScenariosQuery] + + scenario : typing.Optional[EvaluationScenarioQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationMetricsResponse] + AsyncHttpResponse[EvaluationScenariosResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/metrics/",method="PATCH", + f"evaluations/queues/{jsonable_encoder(queue_id)}/scenarios/query",method="POST", json={ - "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=typing.Sequence[EvaluationMetricsEdit], direction="write"), + "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=typing.Optional[EvaluationQueueScenariosQuery], direction="write"), + "scenario": convert_and_respect_annotation_metadata(object_=scenario, annotation=typing.Optional[EvaluationScenarioQuery], direction="write"), + "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), } , headers={"content-type": "application/json", } @@ -3889,9 +4635,9 @@ async def edit_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsEdit], try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationMetricsResponse, + EvaluationScenariosResponse, parse_obj_as( - type_ =EvaluationMetricsResponse, # type: ignore + type_ =EvaluationScenariosResponse, # type: ignore object_ =_response.json() ) ) @@ -3909,27 +4655,24 @@ async def edit_metrics(self, *, metrics: typing.Sequence[EvaluationMetricsEdit], raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationMetricsResponse]: + async def create_simple_evaluation(self, *, evaluation: SimpleEvaluationCreate, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: """ Parameters ---------- - metrics : typing.Optional[EvaluationMetricsQuery] - - windowing : typing.Optional[Windowing] + evaluation : SimpleEvaluationCreate request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationMetricsResponse] + AsyncHttpResponse[SimpleEvaluationResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/metrics/query",method="POST", + "simple/evaluations/",method="POST", json={ - "metrics": convert_and_respect_annotation_metadata(object_=metrics, annotation=typing.Optional[EvaluationMetricsQuery], direction="write"), - "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), + "evaluation": convert_and_respect_annotation_metadata(object_=evaluation, annotation=SimpleEvaluationCreate, direction="write"), } , headers={"content-type": "application/json", } @@ -3939,9 +4682,9 @@ async def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationMetricsResponse, + SimpleEvaluationResponse, parse_obj_as( - type_ =EvaluationMetricsResponse, # type: ignore + type_ =SimpleEvaluationResponse, # type: ignore object_ =_response.json() ) ) @@ -3959,24 +4702,27 @@ async def query_metrics(self, *, metrics: typing.Optional[EvaluationMetricsQuery raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def create_queues(self, *, queues: typing.Sequence[EvaluationQueueCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueuesResponse]: + async def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEvaluationQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationsResponse]: """ Parameters ---------- - queues : typing.Sequence[EvaluationQueueCreate] + evaluation : typing.Optional[SimpleEvaluationQuery] + + windowing : typing.Optional[Windowing] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationQueuesResponse] + AsyncHttpResponse[SimpleEvaluationsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/queues/",method="POST", + "simple/evaluations/query",method="POST", json={ - "queues": convert_and_respect_annotation_metadata(object_=queues, annotation=typing.Sequence[EvaluationQueueCreate], direction="write"), + "evaluation": convert_and_respect_annotation_metadata(object_=evaluation, annotation=typing.Optional[SimpleEvaluationQuery], direction="write"), + "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), } , headers={"content-type": "application/json", } @@ -3986,9 +4732,9 @@ async def create_queues(self, *, queues: typing.Sequence[EvaluationQueueCreate], try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationQueuesResponse, + SimpleEvaluationsResponse, parse_obj_as( - type_ =EvaluationQueuesResponse, # type: ignore + type_ =SimpleEvaluationsResponse, # type: ignore object_ =_response.json() ) ) @@ -4006,36 +4752,29 @@ async def create_queues(self, *, queues: typing.Sequence[EvaluationQueueCreate], raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def delete_queues(self, *, queue_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueueIdsResponse]: + async def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: """ Parameters ---------- - queue_ids : typing.Sequence[str] + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationQueueIdsResponse] + AsyncHttpResponse[SimpleEvaluationResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/queues/",method="DELETE", - json={ - "queue_ids": queue_ids, - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="GET", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationQueueIdsResponse, + SimpleEvaluationResponse, parse_obj_as( - type_ =EvaluationQueueIdsResponse, # type: ignore + type_ =SimpleEvaluationResponse, # type: ignore object_ =_response.json() ) ) @@ -4053,36 +4792,29 @@ async def delete_queues(self, *, queue_ids: typing.Sequence[str], request_option raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def edit_queues(self, *, queues: typing.Sequence[EvaluationQueueEdit], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueuesResponse]: + async def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationIdResponse]: """ Parameters ---------- - queues : typing.Sequence[EvaluationQueueEdit] + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationQueuesResponse] + AsyncHttpResponse[SimpleEvaluationIdResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/queues/",method="PATCH", - json={ - "queues": convert_and_respect_annotation_metadata(object_=queues, annotation=typing.Sequence[EvaluationQueueEdit], direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="DELETE", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationQueuesResponse, + SimpleEvaluationIdResponse, parse_obj_as( - type_ =EvaluationQueuesResponse, # type: ignore + type_ =SimpleEvaluationIdResponse, # type: ignore object_ =_response.json() ) ) @@ -4100,27 +4832,26 @@ async def edit_queues(self, *, queues: typing.Sequence[EvaluationQueueEdit], req raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def query_queues(self, *, queue: typing.Optional[EvaluationQueueQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueuesResponse]: + async def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: SimpleEvaluationEdit, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: """ Parameters ---------- - queue : typing.Optional[EvaluationQueueQuery] + evaluation_id : str - windowing : typing.Optional[Windowing] + evaluation : SimpleEvaluationEdit request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationQueuesResponse] + AsyncHttpResponse[SimpleEvaluationResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "evaluations/queues/query",method="POST", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="PATCH", json={ - "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=typing.Optional[EvaluationQueueQuery], direction="write"), - "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), + "evaluation": convert_and_respect_annotation_metadata(object_=evaluation, annotation=SimpleEvaluationEdit, direction="write"), } , headers={"content-type": "application/json", } @@ -4130,9 +4861,9 @@ async def query_queues(self, *, queue: typing.Optional[EvaluationQueueQuery] = O try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationQueuesResponse, + SimpleEvaluationResponse, parse_obj_as( - type_ =EvaluationQueuesResponse, # type: ignore + type_ =SimpleEvaluationResponse, # type: ignore object_ =_response.json() ) ) @@ -4150,29 +4881,29 @@ async def query_queues(self, *, queue: typing.Optional[EvaluationQueueQuery] = O raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def fetch_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueueResponse]: + async def start_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: """ Parameters ---------- - queue_id : str + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationQueueResponse] + AsyncHttpResponse[SimpleEvaluationResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/queues/{jsonable_encoder(queue_id)}",method="GET", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/start",method="POST", request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationQueueResponse, + SimpleEvaluationResponse, parse_obj_as( - type_ =EvaluationQueueResponse, # type: ignore + type_ =SimpleEvaluationResponse, # type: ignore object_ =_response.json() ) ) @@ -4190,29 +4921,29 @@ async def fetch_queue(self, queue_id: str, *, request_options: typing.Optional[R raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def delete_queue(self, queue_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueueIdResponse]: + async def stop_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: """ Parameters ---------- - queue_id : str + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationQueueIdResponse] + AsyncHttpResponse[SimpleEvaluationResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/queues/{jsonable_encoder(queue_id)}",method="DELETE", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/stop",method="POST", request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationQueueIdResponse, + SimpleEvaluationResponse, parse_obj_as( - type_ =EvaluationQueueIdResponse, # type: ignore + type_ =SimpleEvaluationResponse, # type: ignore object_ =_response.json() ) ) @@ -4230,38 +4961,29 @@ async def delete_queue(self, queue_id: str, *, request_options: typing.Optional[ raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def edit_queue(self, queue_id: str, *, queue: EvaluationQueueEdit, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationQueueResponse]: + async def close_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: """ Parameters ---------- - queue_id : str - - queue : EvaluationQueueEdit + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationQueueResponse] + AsyncHttpResponse[SimpleEvaluationResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/queues/{jsonable_encoder(queue_id)}",method="PATCH", - json={ - "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=EvaluationQueueEdit, direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/close",method="POST", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationQueueResponse, + SimpleEvaluationResponse, parse_obj_as( - type_ =EvaluationQueueResponse, # type: ignore + type_ =SimpleEvaluationResponse, # type: ignore object_ =_response.json() ) ) @@ -4279,44 +5001,29 @@ async def edit_queue(self, queue_id: str, *, queue: EvaluationQueueEdit, request raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def query_evaluation_queue_scenarios(self, queue_id: str, *, queue: typing.Optional[EvaluationQueueScenariosQuery] = OMIT, scenario: typing.Optional[EvaluationScenarioQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenariosResponse]: + async def open_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: """ Parameters ---------- - queue_id : str - - queue : typing.Optional[EvaluationQueueScenariosQuery] - - scenario : typing.Optional[EvaluationScenarioQuery] - - windowing : typing.Optional[Windowing] + evaluation_id : str request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[EvaluationScenariosResponse] + AsyncHttpResponse[SimpleEvaluationResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"evaluations/queues/{jsonable_encoder(queue_id)}/scenarios/query",method="POST", - json={ - "queue": convert_and_respect_annotation_metadata(object_=queue, annotation=typing.Optional[EvaluationQueueScenariosQuery], direction="write"), - "scenario": convert_and_respect_annotation_metadata(object_=scenario, annotation=typing.Optional[EvaluationScenarioQuery], direction="write"), - "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), - } - , - headers={"content-type": "application/json", } - , - request_options=request_options,omit=OMIT, - ) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/open",method="POST", + request_options=request_options,) try: if 200 <= _response.status_code < 300: _data = typing.cast( - EvaluationScenariosResponse, + SimpleEvaluationResponse, parse_obj_as( - type_ =EvaluationScenariosResponse, # type: ignore + type_ =SimpleEvaluationResponse, # type: ignore object_ =_response.json() ) ) @@ -4334,24 +5041,26 @@ async def query_evaluation_queue_scenarios(self, queue_id: str, *, queue: typing raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def create_simple_evaluation(self, *, evaluation: SimpleEvaluationCreate, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: + async def populate_slice(self, evaluation_id: str, *, results: typing.Sequence[EvaluationResultCreate], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultsResponse]: """ Parameters ---------- - evaluation : SimpleEvaluationCreate + evaluation_id : str + + results : typing.Sequence[EvaluationResultCreate] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[SimpleEvaluationResponse] + AsyncHttpResponse[EvaluationResultsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "simple/evaluations/",method="POST", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/populate",method="POST", json={ - "evaluation": convert_and_respect_annotation_metadata(object_=evaluation, annotation=SimpleEvaluationCreate, direction="write"), + "results": convert_and_respect_annotation_metadata(object_=results, annotation=typing.Sequence[EvaluationResultCreate], direction="write"), } , headers={"content-type": "application/json", } @@ -4361,9 +5070,9 @@ async def create_simple_evaluation(self, *, evaluation: SimpleEvaluationCreate, try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationResponse, + EvaluationResultsResponse, parse_obj_as( - type_ =SimpleEvaluationResponse, # type: ignore + type_ =EvaluationResultsResponse, # type: ignore object_ =_response.json() ) ) @@ -4381,29 +5090,49 @@ async def create_simple_evaluation(self, *, evaluation: SimpleEvaluationCreate, raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: + async def process_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, overwrite: typing.Optional[bool] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[typing.Any]: """ Parameters ---------- evaluation_id : str + scenario_ids : typing.Optional[typing.Sequence[str]] + + step_keys : typing.Optional[typing.Sequence[str]] + + repeat_idxs : typing.Optional[typing.Sequence[int]] + + overwrite : typing.Optional[bool] + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[SimpleEvaluationResponse] - Successful Response + AsyncHttpResponse[typing.Any] + Accepted. """ _response = await self._client_wrapper.httpx_client.request( - f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="GET", - request_options=request_options,) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/process",method="POST", + json={ + "scenario_ids": scenario_ids, + "step_keys": step_keys, + "repeat_idxs": repeat_idxs, + "overwrite": overwrite, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: + if _response is None or not _response.text.strip(): + return AsyncHttpResponse(response=_response, data=None) if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationResponse, + typing.Any, parse_obj_as( - type_ =SimpleEvaluationResponse, # type: ignore + type_ =typing.Any, # type: ignore object_ =_response.json() ) ) @@ -4421,29 +5150,44 @@ async def fetch_simple_evaluation(self, evaluation_id: str, *, request_options: raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def delete_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationIdResponse]: + async def probe_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationResultsResponse]: """ Parameters ---------- evaluation_id : str + scenario_ids : typing.Optional[typing.Sequence[str]] + + step_keys : typing.Optional[typing.Sequence[str]] + + repeat_idxs : typing.Optional[typing.Sequence[int]] + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[SimpleEvaluationIdResponse] + AsyncHttpResponse[EvaluationResultsResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="DELETE", - request_options=request_options,) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/probe",method="POST", + json={ + "scenario_ids": scenario_ids, + "step_keys": step_keys, + "repeat_idxs": repeat_idxs, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationIdResponse, + EvaluationResultsResponse, parse_obj_as( - type_ =SimpleEvaluationIdResponse, # type: ignore + type_ =EvaluationResultsResponse, # type: ignore object_ =_response.json() ) ) @@ -4461,26 +5205,31 @@ async def delete_simple_evaluation(self, evaluation_id: str, *, request_options: raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: SimpleEvaluationEdit, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: + async def prune_slice(self, evaluation_id: str, *, scenario_ids: typing.Optional[typing.Sequence[str]] = OMIT, step_keys: typing.Optional[typing.Sequence[str]] = OMIT, repeat_idxs: typing.Optional[typing.Sequence[int]] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[None]: """ Parameters ---------- evaluation_id : str - evaluation : SimpleEvaluationEdit + scenario_ids : typing.Optional[typing.Sequence[str]] + + step_keys : typing.Optional[typing.Sequence[str]] + + repeat_idxs : typing.Optional[typing.Sequence[int]] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[SimpleEvaluationResponse] - Successful Response + AsyncHttpResponse[None] """ _response = await self._client_wrapper.httpx_client.request( - f"simple/evaluations/{jsonable_encoder(evaluation_id)}",method="PATCH", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/prune",method="POST", json={ - "evaluation": convert_and_respect_annotation_metadata(object_=evaluation, annotation=SimpleEvaluationEdit, direction="write"), + "scenario_ids": scenario_ids, + "step_keys": step_keys, + "repeat_idxs": repeat_idxs, } , headers={"content-type": "application/json", } @@ -4489,14 +5238,7 @@ async def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: Simple ) try: if 200 <= _response.status_code < 300: - _data = typing.cast( - SimpleEvaluationResponse, - parse_obj_as( - type_ =SimpleEvaluationResponse, # type: ignore - object_ =_response.json() - ) - ) - return AsyncHttpResponse(response=_response, data=_data) + return AsyncHttpResponse(response=_response, data=None) if _response.status_code == 422: raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( HttpValidationError, @@ -4510,27 +5252,29 @@ async def edit_simple_evaluation(self, evaluation_id: str, *, evaluation: Simple raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEvaluationQuery] = OMIT, windowing: typing.Optional[Windowing] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationsResponse]: + async def add_scenarios(self, evaluation_id: str, *, count: int, timestamp: typing.Optional[dt.datetime] = OMIT, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationScenariosResponse]: """ Parameters ---------- - evaluation : typing.Optional[SimpleEvaluationQuery] + evaluation_id : str - windowing : typing.Optional[Windowing] + count : int + + timestamp : typing.Optional[dt.datetime] request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[SimpleEvaluationsResponse] + AsyncHttpResponse[EvaluationScenariosResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - "simple/evaluations/query",method="POST", + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/scenarios/add",method="POST", json={ - "evaluation": convert_and_respect_annotation_metadata(object_=evaluation, annotation=typing.Optional[SimpleEvaluationQuery], direction="write"), - "windowing": convert_and_respect_annotation_metadata(object_=windowing, annotation=typing.Optional[Windowing], direction="write"), + "count": count, + "timestamp": timestamp, } , headers={"content-type": "application/json", } @@ -4540,9 +5284,9 @@ async def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEv try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationsResponse, + EvaluationScenariosResponse, parse_obj_as( - type_ =SimpleEvaluationsResponse, # type: ignore + type_ =EvaluationScenariosResponse, # type: ignore object_ =_response.json() ) ) @@ -4560,33 +5304,34 @@ async def query_simple_evaluations(self, *, evaluation: typing.Optional[SimpleEv raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def start_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: + async def remove_scenarios(self, evaluation_id: str, *, scenario_ids: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[None]: """ Parameters ---------- evaluation_id : str + scenario_ids : typing.Sequence[str] + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[SimpleEvaluationResponse] - Successful Response + AsyncHttpResponse[None] """ _response = await self._client_wrapper.httpx_client.request( - f"simple/evaluations/{jsonable_encoder(evaluation_id)}/start",method="POST", - request_options=request_options,) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/scenarios/remove",method="POST", + json={ + "scenario_ids": scenario_ids, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: - _data = typing.cast( - SimpleEvaluationResponse, - parse_obj_as( - type_ =SimpleEvaluationResponse, # type: ignore - object_ =_response.json() - ) - ) - return AsyncHttpResponse(response=_response, data=_data) + return AsyncHttpResponse(response=_response, data=None) if _response.status_code == 422: raise UnprocessableEntityError(headers=dict(_response.headers), body=typing.cast( HttpValidationError, @@ -4600,29 +5345,38 @@ async def start_simple_evaluation(self, evaluation_id: str, *, request_options: raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def stop_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: + async def add_steps(self, evaluation_id: str, *, steps: typing.Sequence[EvaluationRunDataStep], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: """ Parameters ---------- evaluation_id : str + steps : typing.Sequence[EvaluationRunDataStep] + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[SimpleEvaluationResponse] + AsyncHttpResponse[EvaluationRunResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"simple/evaluations/{jsonable_encoder(evaluation_id)}/stop",method="POST", - request_options=request_options,) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/steps/add",method="POST", + json={ + "steps": convert_and_respect_annotation_metadata(object_=steps, annotation=typing.Sequence[EvaluationRunDataStep], direction="write"), + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationResponse, + EvaluationRunResponse, parse_obj_as( - type_ =SimpleEvaluationResponse, # type: ignore + type_ =EvaluationRunResponse, # type: ignore object_ =_response.json() ) ) @@ -4640,29 +5394,38 @@ async def stop_simple_evaluation(self, evaluation_id: str, *, request_options: t raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def close_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: + async def remove_steps(self, evaluation_id: str, *, step_keys: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: """ Parameters ---------- evaluation_id : str + step_keys : typing.Sequence[str] + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[SimpleEvaluationResponse] + AsyncHttpResponse[EvaluationRunResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"simple/evaluations/{jsonable_encoder(evaluation_id)}/close",method="POST", - request_options=request_options,) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/steps/remove",method="POST", + json={ + "step_keys": step_keys, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationResponse, + EvaluationRunResponse, parse_obj_as( - type_ =SimpleEvaluationResponse, # type: ignore + type_ =EvaluationRunResponse, # type: ignore object_ =_response.json() ) ) @@ -4680,29 +5443,38 @@ async def close_simple_evaluation(self, evaluation_id: str, *, request_options: raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) - async def open_simple_evaluation(self, evaluation_id: str, *, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[SimpleEvaluationResponse]: + async def set_repeats(self, evaluation_id: str, *, repeats: int, request_options: typing.Optional[RequestOptions] = None) -> AsyncHttpResponse[EvaluationRunResponse]: """ Parameters ---------- evaluation_id : str + repeats : int + request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- - AsyncHttpResponse[SimpleEvaluationResponse] + AsyncHttpResponse[EvaluationRunResponse] Successful Response """ _response = await self._client_wrapper.httpx_client.request( - f"simple/evaluations/{jsonable_encoder(evaluation_id)}/open",method="POST", - request_options=request_options,) + f"simple/evaluations/{jsonable_encoder(evaluation_id)}/repeats/set",method="POST", + json={ + "repeats": repeats, + } + , + headers={"content-type": "application/json", } + , + request_options=request_options,omit=OMIT, + ) try: if 200 <= _response.status_code < 300: _data = typing.cast( - SimpleEvaluationResponse, + EvaluationRunResponse, parse_obj_as( - type_ =SimpleEvaluationResponse, # type: ignore + type_ =EvaluationRunResponse, # type: ignore object_ =_response.json() ) ) diff --git a/clients/python/agenta_client/types/__init__.py b/clients/python/agenta_client/types/__init__.py index 553461ec47..274aabb152 100644 --- a/clients/python/agenta_client/types/__init__.py +++ b/clients/python/agenta_client/types/__init__.py @@ -157,7 +157,6 @@ from .error_policy import ErrorPolicy from .evaluation_metrics import EvaluationMetrics from .evaluation_metrics_create import EvaluationMetricsCreate - from .evaluation_metrics_edit import EvaluationMetricsEdit from .evaluation_metrics_ids_response import EvaluationMetricsIdsResponse from .evaluation_metrics_query import EvaluationMetricsQuery from .evaluation_metrics_query_scenario_ids import EvaluationMetricsQueryScenarioIds @@ -178,7 +177,6 @@ from .evaluation_queues_response import EvaluationQueuesResponse from .evaluation_result import EvaluationResult from .evaluation_result_create import EvaluationResultCreate - from .evaluation_result_edit import EvaluationResultEdit from .evaluation_result_id_response import EvaluationResultIdResponse from .evaluation_result_ids_response import EvaluationResultIdsResponse from .evaluation_result_query import EvaluationResultQuery @@ -187,6 +185,7 @@ from .evaluation_run import EvaluationRun from .evaluation_run_create import EvaluationRunCreate from .evaluation_run_data import EvaluationRunData + from .evaluation_run_data_concurrency import EvaluationRunDataConcurrency from .evaluation_run_data_mapping import EvaluationRunDataMapping from .evaluation_run_data_mapping_column import EvaluationRunDataMappingColumn from .evaluation_run_data_mapping_step import EvaluationRunDataMappingStep @@ -644,7 +643,7 @@ from .workspace_member_response import WorkspaceMemberResponse from .workspace_permission import WorkspacePermission from .workspace_response import WorkspaceResponse -_dynamic_imports: typing.Dict[str, str] = {"AdminAccountCreateOptions": ".admin_account_create_options", "AdminAccountRead": ".admin_account_read", "AdminAccountsCreate": ".admin_accounts_create", "AdminAccountsDelete": ".admin_accounts_delete", "AdminAccountsDeleteTarget": ".admin_accounts_delete_target", "AdminAccountsResponse": ".admin_accounts_response", "AdminApiKeyCreate": ".admin_api_key_create", "AdminApiKeyResponse": ".admin_api_key_response", "AdminDeleteResponse": ".admin_delete_response", "AdminDeletedEntities": ".admin_deleted_entities", "AdminDeletedEntity": ".admin_deleted_entity", "AdminOrganizationCreate": ".admin_organization_create", "AdminOrganizationMembershipCreate": ".admin_organization_membership_create", "AdminOrganizationMembershipRead": ".admin_organization_membership_read", "AdminOrganizationRead": ".admin_organization_read", "AdminProjectCreate": ".admin_project_create", "AdminProjectMembershipCreate": ".admin_project_membership_create", "AdminProjectMembershipRead": ".admin_project_membership_read", "AdminProjectRead": ".admin_project_read", "AdminSimpleAccountCreate": ".admin_simple_account_create", "AdminSimpleAccountDeleteEntry": ".admin_simple_account_delete_entry", "AdminSimpleAccountRead": ".admin_simple_account_read", "AdminSimpleAccountsApiKeysCreate": ".admin_simple_accounts_api_keys_create", "AdminSimpleAccountsCreate": ".admin_simple_accounts_create", "AdminSimpleAccountsDelete": ".admin_simple_accounts_delete", "AdminSimpleAccountsOrganizationsCreate": ".admin_simple_accounts_organizations_create", "AdminSimpleAccountsOrganizationsMembershipsCreate": ".admin_simple_accounts_organizations_memberships_create", "AdminSimpleAccountsOrganizationsTransferOwnership": ".admin_simple_accounts_organizations_transfer_ownership", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjects": ".admin_simple_accounts_organizations_transfer_ownership_include_projects", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjectsZero": ".admin_simple_accounts_organizations_transfer_ownership_include_projects_zero", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspaces": ".admin_simple_accounts_organizations_transfer_ownership_include_workspaces", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspacesZero": ".admin_simple_accounts_organizations_transfer_ownership_include_workspaces_zero", "AdminSimpleAccountsOrganizationsTransferOwnershipResponse": ".admin_simple_accounts_organizations_transfer_ownership_response", "AdminSimpleAccountsProjectsCreate": ".admin_simple_accounts_projects_create", "AdminSimpleAccountsProjectsMembershipsCreate": ".admin_simple_accounts_projects_memberships_create", "AdminSimpleAccountsResponse": ".admin_simple_accounts_response", "AdminSimpleAccountsUsersCreate": ".admin_simple_accounts_users_create", "AdminSimpleAccountsUsersIdentitiesCreate": ".admin_simple_accounts_users_identities_create", "AdminSimpleAccountsUsersResetPassword": ".admin_simple_accounts_users_reset_password", "AdminSimpleAccountsWorkspacesCreate": ".admin_simple_accounts_workspaces_create", "AdminSimpleAccountsWorkspacesMembershipsCreate": ".admin_simple_accounts_workspaces_memberships_create", "AdminStructuredError": ".admin_structured_error", "AdminSubscriptionCreate": ".admin_subscription_create", "AdminSubscriptionRead": ".admin_subscription_read", "AdminUserCreate": ".admin_user_create", "AdminUserIdentityCreate": ".admin_user_identity_create", "AdminUserIdentityRead": ".admin_user_identity_read", "AdminUserIdentityReadStatus": ".admin_user_identity_read_status", "AdminUserRead": ".admin_user_read", "AdminWorkspaceCreate": ".admin_workspace_create", "AdminWorkspaceMembershipCreate": ".admin_workspace_membership_create", "AdminWorkspaceMembershipRead": ".admin_workspace_membership_read", "AdminWorkspaceRead": ".admin_workspace_read", "Analytics": ".analytics", "AnalyticsResponse": ".analytics_response", "Annotation": ".annotation", "AnnotationCreate": ".annotation_create", "AnnotationCreateLinks": ".annotation_create_links", "AnnotationEdit": ".annotation_edit", "AnnotationEditLinks": ".annotation_edit_links", "AnnotationLinkResponse": ".annotation_link_response", "AnnotationLinks": ".annotation_links", "AnnotationQuery": ".annotation_query", "AnnotationQueryLinks": ".annotation_query_links", "AnnotationResponse": ".annotation_response", "AnnotationsResponse": ".annotations_response", "Application": ".application", "ApplicationArtifactFlags": ".application_artifact_flags", "ApplicationArtifactQueryFlags": ".application_artifact_query_flags", "ApplicationCatalogPreset": ".application_catalog_preset", "ApplicationCatalogPresetResponse": ".application_catalog_preset_response", "ApplicationCatalogPresetsResponse": ".application_catalog_presets_response", "ApplicationCatalogTemplate": ".application_catalog_template", "ApplicationCatalogTemplateResponse": ".application_catalog_template_response", "ApplicationCatalogTemplatesResponse": ".application_catalog_templates_response", "ApplicationCatalogType": ".application_catalog_type", "ApplicationCatalogTypesResponse": ".application_catalog_types_response", "ApplicationCreate": ".application_create", "ApplicationEdit": ".application_edit", "ApplicationFlags": ".application_flags", "ApplicationFork": ".application_fork", "ApplicationQuery": ".application_query", "ApplicationResponse": ".application_response", "ApplicationRevision": ".application_revision", "ApplicationRevisionCommit": ".application_revision_commit", "ApplicationRevisionCreate": ".application_revision_create", "ApplicationRevisionDataInput": ".application_revision_data_input", "ApplicationRevisionDataInputHeadersValue": ".application_revision_data_input_headers_value", "ApplicationRevisionDataInputRuntime": ".application_revision_data_input_runtime", "ApplicationRevisionDataOutput": ".application_revision_data_output", "ApplicationRevisionDataOutputHeadersValue": ".application_revision_data_output_headers_value", "ApplicationRevisionDataOutputRuntime": ".application_revision_data_output_runtime", "ApplicationRevisionEdit": ".application_revision_edit", "ApplicationRevisionFlags": ".application_revision_flags", "ApplicationRevisionFork": ".application_revision_fork", "ApplicationRevisionQuery": ".application_revision_query", "ApplicationRevisionQueryFlags": ".application_revision_query_flags", "ApplicationRevisionResolveResponse": ".application_revision_resolve_response", "ApplicationRevisionResponse": ".application_revision_response", "ApplicationRevisionsLog": ".application_revisions_log", "ApplicationRevisionsResponse": ".application_revisions_response", "ApplicationVariant": ".application_variant", "ApplicationVariantCreate": ".application_variant_create", "ApplicationVariantEdit": ".application_variant_edit", "ApplicationVariantFlags": ".application_variant_flags", "ApplicationVariantFork": ".application_variant_fork", "ApplicationVariantResponse": ".application_variant_response", "ApplicationVariantsResponse": ".application_variants_response", "ApplicationsResponse": ".applications_response", "BodyConfigsFetchVariantsConfigsFetchPost": ".body_configs_fetch_variants_configs_fetch_post", "Bucket": ".bucket", "CollectStatusResponse": ".collect_status_response", "ComparisonOperator": ".comparison_operator", "Condition": ".condition", "ConditionOperator": ".condition_operator", "ConditionOptions": ".condition_options", "ConditionValue": ".condition_value", "ConfigResponseModel": ".config_response_model", "CustomModelSettingsDto": ".custom_model_settings_dto", "CustomProviderDto": ".custom_provider_dto", "CustomProviderKind": ".custom_provider_kind", "CustomProviderSettingsDto": ".custom_provider_settings_dto", "DictOperator": ".dict_operator", "DiscoverResponse": ".discover_response", "DiscoverResponseMethodsValue": ".discover_response_methods_value", "EeSrcModelsApiOrganizationModelsOrganization": ".ee_src_models_api_organization_models_organization", "EntityRef": ".entity_ref", "Environment": ".environment", "EnvironmentCreate": ".environment_create", "EnvironmentEdit": ".environment_edit", "EnvironmentFlags": ".environment_flags", "EnvironmentQueryFlags": ".environment_query_flags", "EnvironmentResponse": ".environment_response", "EnvironmentRevision": ".environment_revision", "EnvironmentRevisionCommit": ".environment_revision_commit", "EnvironmentRevisionCreate": ".environment_revision_create", "EnvironmentRevisionData": ".environment_revision_data", "EnvironmentRevisionDelta": ".environment_revision_delta", "EnvironmentRevisionEdit": ".environment_revision_edit", "EnvironmentRevisionResolveResponse": ".environment_revision_resolve_response", "EnvironmentRevisionResponse": ".environment_revision_response", "EnvironmentRevisionsLog": ".environment_revisions_log", "EnvironmentRevisionsResponse": ".environment_revisions_response", "EnvironmentVariant": ".environment_variant", "EnvironmentVariantCreate": ".environment_variant_create", "EnvironmentVariantEdit": ".environment_variant_edit", "EnvironmentVariantResponse": ".environment_variant_response", "EnvironmentVariantsResponse": ".environment_variants_response", "EnvironmentsResponse": ".environments_response", "ErrorPolicy": ".error_policy", "EvaluationMetrics": ".evaluation_metrics", "EvaluationMetricsCreate": ".evaluation_metrics_create", "EvaluationMetricsEdit": ".evaluation_metrics_edit", "EvaluationMetricsIdsResponse": ".evaluation_metrics_ids_response", "EvaluationMetricsQuery": ".evaluation_metrics_query", "EvaluationMetricsQueryScenarioIds": ".evaluation_metrics_query_scenario_ids", "EvaluationMetricsQueryTimestamps": ".evaluation_metrics_query_timestamps", "EvaluationMetricsRefresh": ".evaluation_metrics_refresh", "EvaluationMetricsResponse": ".evaluation_metrics_response", "EvaluationQueue": ".evaluation_queue", "EvaluationQueueCreate": ".evaluation_queue_create", "EvaluationQueueData": ".evaluation_queue_data", "EvaluationQueueEdit": ".evaluation_queue_edit", "EvaluationQueueFlags": ".evaluation_queue_flags", "EvaluationQueueIdResponse": ".evaluation_queue_id_response", "EvaluationQueueIdsResponse": ".evaluation_queue_ids_response", "EvaluationQueueQuery": ".evaluation_queue_query", "EvaluationQueueQueryFlags": ".evaluation_queue_query_flags", "EvaluationQueueResponse": ".evaluation_queue_response", "EvaluationQueueScenariosQuery": ".evaluation_queue_scenarios_query", "EvaluationQueuesResponse": ".evaluation_queues_response", "EvaluationResult": ".evaluation_result", "EvaluationResultCreate": ".evaluation_result_create", "EvaluationResultEdit": ".evaluation_result_edit", "EvaluationResultIdResponse": ".evaluation_result_id_response", "EvaluationResultIdsResponse": ".evaluation_result_ids_response", "EvaluationResultQuery": ".evaluation_result_query", "EvaluationResultResponse": ".evaluation_result_response", "EvaluationResultsResponse": ".evaluation_results_response", "EvaluationRun": ".evaluation_run", "EvaluationRunCreate": ".evaluation_run_create", "EvaluationRunData": ".evaluation_run_data", "EvaluationRunDataMapping": ".evaluation_run_data_mapping", "EvaluationRunDataMappingColumn": ".evaluation_run_data_mapping_column", "EvaluationRunDataMappingStep": ".evaluation_run_data_mapping_step", "EvaluationRunDataStep": ".evaluation_run_data_step", "EvaluationRunDataStepInput": ".evaluation_run_data_step_input", "EvaluationRunDataStepOrigin": ".evaluation_run_data_step_origin", "EvaluationRunDataStepType": ".evaluation_run_data_step_type", "EvaluationRunEdit": ".evaluation_run_edit", "EvaluationRunFlags": ".evaluation_run_flags", "EvaluationRunIdResponse": ".evaluation_run_id_response", "EvaluationRunIdsRequest": ".evaluation_run_ids_request", "EvaluationRunIdsResponse": ".evaluation_run_ids_response", "EvaluationRunQuery": ".evaluation_run_query", "EvaluationRunQueryFlags": ".evaluation_run_query_flags", "EvaluationRunResponse": ".evaluation_run_response", "EvaluationRunsResponse": ".evaluation_runs_response", "EvaluationScenario": ".evaluation_scenario", "EvaluationScenarioCreate": ".evaluation_scenario_create", "EvaluationScenarioEdit": ".evaluation_scenario_edit", "EvaluationScenarioIdResponse": ".evaluation_scenario_id_response", "EvaluationScenarioIdsResponse": ".evaluation_scenario_ids_response", "EvaluationScenarioQuery": ".evaluation_scenario_query", "EvaluationScenarioResponse": ".evaluation_scenario_response", "EvaluationScenariosResponse": ".evaluation_scenarios_response", "EvaluationStatus": ".evaluation_status", "Evaluator": ".evaluator", "EvaluatorArtifactFlags": ".evaluator_artifact_flags", "EvaluatorArtifactQueryFlags": ".evaluator_artifact_query_flags", "EvaluatorCatalogPreset": ".evaluator_catalog_preset", "EvaluatorCatalogPresetResponse": ".evaluator_catalog_preset_response", "EvaluatorCatalogPresetsResponse": ".evaluator_catalog_presets_response", "EvaluatorCatalogTemplate": ".evaluator_catalog_template", "EvaluatorCatalogTemplateResponse": ".evaluator_catalog_template_response", "EvaluatorCatalogTemplatesResponse": ".evaluator_catalog_templates_response", "EvaluatorCatalogType": ".evaluator_catalog_type", "EvaluatorCatalogTypesResponse": ".evaluator_catalog_types_response", "EvaluatorCreate": ".evaluator_create", "EvaluatorEdit": ".evaluator_edit", "EvaluatorFlags": ".evaluator_flags", "EvaluatorFork": ".evaluator_fork", "EvaluatorQuery": ".evaluator_query", "EvaluatorResponse": ".evaluator_response", "EvaluatorRevision": ".evaluator_revision", "EvaluatorRevisionCommit": ".evaluator_revision_commit", "EvaluatorRevisionCreate": ".evaluator_revision_create", "EvaluatorRevisionDataInput": ".evaluator_revision_data_input", "EvaluatorRevisionDataInputHeadersValue": ".evaluator_revision_data_input_headers_value", "EvaluatorRevisionDataInputRuntime": ".evaluator_revision_data_input_runtime", "EvaluatorRevisionDataOutput": ".evaluator_revision_data_output", "EvaluatorRevisionDataOutputHeadersValue": ".evaluator_revision_data_output_headers_value", "EvaluatorRevisionDataOutputRuntime": ".evaluator_revision_data_output_runtime", "EvaluatorRevisionEdit": ".evaluator_revision_edit", "EvaluatorRevisionFlags": ".evaluator_revision_flags", "EvaluatorRevisionFork": ".evaluator_revision_fork", "EvaluatorRevisionQuery": ".evaluator_revision_query", "EvaluatorRevisionQueryFlags": ".evaluator_revision_query_flags", "EvaluatorRevisionResolveResponse": ".evaluator_revision_resolve_response", "EvaluatorRevisionResponse": ".evaluator_revision_response", "EvaluatorRevisionsLog": ".evaluator_revisions_log", "EvaluatorRevisionsResponse": ".evaluator_revisions_response", "EvaluatorTemplate": ".evaluator_template", "EvaluatorTemplatesResponse": ".evaluator_templates_response", "EvaluatorVariant": ".evaluator_variant", "EvaluatorVariantCreate": ".evaluator_variant_create", "EvaluatorVariantEdit": ".evaluator_variant_edit", "EvaluatorVariantFlags": ".evaluator_variant_flags", "EvaluatorVariantFork": ".evaluator_variant_fork", "EvaluatorVariantResponse": ".evaluator_variant_response", "EvaluatorVariantsResponse": ".evaluator_variants_response", "EvaluatorsResponse": ".evaluators_response", "Event": ".event", "EventQuery": ".event_query", "EventType": ".event_type", "EventsQueryResponse": ".events_query_response", "ExistenceOperator": ".existence_operator", "FilteringInput": ".filtering_input", "FilteringInputConditionsItem": ".filtering_input_conditions_item", "FilteringOutput": ".filtering_output", "FilteringOutputConditionsItem": ".filtering_output_conditions_item", "Focus": ".focus", "Folder": ".folder", "FolderCreate": ".folder_create", "FolderEdit": ".folder_edit", "FolderIdResponse": ".folder_id_response", "FolderKind": ".folder_kind", "FolderQuery": ".folder_query", "FolderQueryKinds": ".folder_query_kinds", "FolderResponse": ".folder_response", "FoldersResponse": ".folders_response", "Format": ".format", "Formatting": ".formatting", typing.Any: ".full_json_input", typing.Any: ".full_json_output", "Header": ".header", "HttpValidationError": ".http_validation_error", "InviteRequest": ".invite_request", "Invocation": ".invocation", "InvocationCreate": ".invocation_create", "InvocationCreateLinks": ".invocation_create_links", "InvocationEdit": ".invocation_edit", "InvocationEditLinks": ".invocation_edit_links", "InvocationLinkResponse": ".invocation_link_response", "InvocationLinks": ".invocation_links", "InvocationQuery": ".invocation_query", "InvocationQueryLinks": ".invocation_query_links", "InvocationResponse": ".invocation_response", "InvocationsResponse": ".invocations_response", "JsonSchemasInput": ".json_schemas_input", "JsonSchemasOutput": ".json_schemas_output", typing.Any: ".label_json_input", typing.Any: ".label_json_output", "LegacyLifecycleDto": ".legacy_lifecycle_dto", "ListApiKeysResponse": ".list_api_keys_response", "ListOperator": ".list_operator", "ListOptions": ".list_options", "LogicalOperator": ".logical_operator", "MetricSpec": ".metric_spec", "MetricType": ".metric_type", "MetricsBucket": ".metrics_bucket", "NumericOperator": ".numeric_operator", "OTelEventInput": ".o_tel_event_input", "OTelEventInputTimestamp": ".o_tel_event_input_timestamp", "OTelEventOutput": ".o_tel_event_output", "OTelEventOutputTimestamp": ".o_tel_event_output_timestamp", "OTelHashInput": ".o_tel_hash_input", "OTelHashOutput": ".o_tel_hash_output", "OTelLinkInput": ".o_tel_link_input", "OTelLinkOutput": ".o_tel_link_output", "OTelLinksResponse": ".o_tel_links_response", "OTelReferenceInput": ".o_tel_reference_input", "OTelReferenceOutput": ".o_tel_reference_output", "OTelSpanKind": ".o_tel_span_kind", "OTelStatusCode": ".o_tel_status_code", "OTelTracingRequest": ".o_tel_tracing_request", "OTelTracingResponse": ".o_tel_tracing_response", "OldAnalyticsResponse": ".old_analytics_response", "OrganizationDetails": ".organization_details", "OrganizationDomainResponse": ".organization_domain_response", "OrganizationProviderResponse": ".organization_provider_response", "OrganizationUpdate": ".organization_update", "OssSrcModelsApiOrganizationModelsOrganization": ".oss_src_models_api_organization_models_organization", "Permission": ".permission", "ProjectsResponse": ".projects_response", "QueriesResponse": ".queries_response", "Query": ".query", "QueryCreate": ".query_create", "QueryEdit": ".query_edit", "QueryFlags": ".query_flags", "QueryQueryFlags": ".query_query_flags", "QueryResponse": ".query_response", "QueryRevision": ".query_revision", "QueryRevisionCommit": ".query_revision_commit", "QueryRevisionCreate": ".query_revision_create", "QueryRevisionDataInput": ".query_revision_data_input", "QueryRevisionDataOutput": ".query_revision_data_output", "QueryRevisionEdit": ".query_revision_edit", "QueryRevisionQuery": ".query_revision_query", "QueryRevisionResponse": ".query_revision_response", "QueryRevisionsLog": ".query_revisions_log", "QueryRevisionsResponse": ".query_revisions_response", "QueryVariant": ".query_variant", "QueryVariantCreate": ".query_variant_create", "QueryVariantEdit": ".query_variant_edit", "QueryVariantQuery": ".query_variant_query", "QueryVariantResponse": ".query_variant_response", "QueryVariantsResponse": ".query_variants_response", "Reference": ".reference", "ReferenceRequestModelInput": ".reference_request_model_input", "ReferenceRequestModelOutput": ".reference_request_model_output", "RequestType": ".request_type", "ResolutionInfo": ".resolution_info", "RetrievalInfo": ".retrieval_info", "RevisionFork": ".revision_fork", "SecretDto": ".secret_dto", "SecretDtoData": ".secret_dto_data", "SecretKind": ".secret_kind", "SecretResponseDto": ".secret_response_dto", "SecretResponseDtoData": ".secret_response_dto_data", "SessionIdsResponse": ".session_ids_response", "SimpleApplication": ".simple_application", "SimpleApplicationCreate": ".simple_application_create", "SimpleApplicationDataInput": ".simple_application_data_input", "SimpleApplicationDataInputHeadersValue": ".simple_application_data_input_headers_value", "SimpleApplicationDataInputRuntime": ".simple_application_data_input_runtime", "SimpleApplicationDataOutput": ".simple_application_data_output", "SimpleApplicationDataOutputHeadersValue": ".simple_application_data_output_headers_value", "SimpleApplicationDataOutputRuntime": ".simple_application_data_output_runtime", "SimpleApplicationEdit": ".simple_application_edit", "SimpleApplicationFlags": ".simple_application_flags", "SimpleApplicationQuery": ".simple_application_query", "SimpleApplicationQueryFlags": ".simple_application_query_flags", "SimpleApplicationResponse": ".simple_application_response", "SimpleApplicationsResponse": ".simple_applications_response", "SimpleEnvironment": ".simple_environment", "SimpleEnvironmentCreate": ".simple_environment_create", "SimpleEnvironmentEdit": ".simple_environment_edit", "SimpleEnvironmentQuery": ".simple_environment_query", "SimpleEnvironmentResponse": ".simple_environment_response", "SimpleEnvironmentsResponse": ".simple_environments_response", "SimpleEvaluation": ".simple_evaluation", "SimpleEvaluationCreate": ".simple_evaluation_create", "SimpleEvaluationData": ".simple_evaluation_data", "SimpleEvaluationDataApplicationSteps": ".simple_evaluation_data_application_steps", "SimpleEvaluationDataApplicationStepsOneValue": ".simple_evaluation_data_application_steps_one_value", "SimpleEvaluationDataEvaluatorSteps": ".simple_evaluation_data_evaluator_steps", "SimpleEvaluationDataEvaluatorStepsOneValue": ".simple_evaluation_data_evaluator_steps_one_value", "SimpleEvaluationDataQuerySteps": ".simple_evaluation_data_query_steps", "SimpleEvaluationDataQueryStepsOneValue": ".simple_evaluation_data_query_steps_one_value", "SimpleEvaluationDataTestsetSteps": ".simple_evaluation_data_testset_steps", "SimpleEvaluationDataTestsetStepsOneValue": ".simple_evaluation_data_testset_steps_one_value", "SimpleEvaluationEdit": ".simple_evaluation_edit", "SimpleEvaluationIdResponse": ".simple_evaluation_id_response", "SimpleEvaluationQuery": ".simple_evaluation_query", "SimpleEvaluationResponse": ".simple_evaluation_response", "SimpleEvaluationsResponse": ".simple_evaluations_response", "SimpleEvaluator": ".simple_evaluator", "SimpleEvaluatorCreate": ".simple_evaluator_create", "SimpleEvaluatorDataInput": ".simple_evaluator_data_input", "SimpleEvaluatorDataInputHeadersValue": ".simple_evaluator_data_input_headers_value", "SimpleEvaluatorDataInputRuntime": ".simple_evaluator_data_input_runtime", "SimpleEvaluatorDataOutput": ".simple_evaluator_data_output", "SimpleEvaluatorDataOutputHeadersValue": ".simple_evaluator_data_output_headers_value", "SimpleEvaluatorDataOutputRuntime": ".simple_evaluator_data_output_runtime", "SimpleEvaluatorEdit": ".simple_evaluator_edit", "SimpleEvaluatorFlags": ".simple_evaluator_flags", "SimpleEvaluatorQuery": ".simple_evaluator_query", "SimpleEvaluatorQueryFlags": ".simple_evaluator_query_flags", "SimpleEvaluatorResponse": ".simple_evaluator_response", "SimpleEvaluatorsResponse": ".simple_evaluators_response", "SimpleQueriesResponse": ".simple_queries_response", "SimpleQuery": ".simple_query", "SimpleQueryCreate": ".simple_query_create", "SimpleQueryEdit": ".simple_query_edit", "SimpleQueryQuery": ".simple_query_query", "SimpleQueryResponse": ".simple_query_response", "SimpleQueue": ".simple_queue", "SimpleQueueCreate": ".simple_queue_create", "SimpleQueueData": ".simple_queue_data", "SimpleQueueDataEvaluators": ".simple_queue_data_evaluators", "SimpleQueueDataEvaluatorsOneValue": ".simple_queue_data_evaluators_one_value", "SimpleQueueIdResponse": ".simple_queue_id_response", "SimpleQueueKind": ".simple_queue_kind", "SimpleQueueQuery": ".simple_queue_query", "SimpleQueueResponse": ".simple_queue_response", "SimpleQueueScenariosQuery": ".simple_queue_scenarios_query", "SimpleQueueScenariosResponse": ".simple_queue_scenarios_response", "SimpleQueueSettings": ".simple_queue_settings", "SimpleQueuesResponse": ".simple_queues_response", "SimpleTestset": ".simple_testset", "SimpleTestsetCreate": ".simple_testset_create", "SimpleTestsetEdit": ".simple_testset_edit", "SimpleTestsetQuery": ".simple_testset_query", "SimpleTestsetResponse": ".simple_testset_response", "SimpleTestsetsResponse": ".simple_testsets_response", "SimpleTrace": ".simple_trace", "SimpleTraceChannel": ".simple_trace_channel", "SimpleTraceCreate": ".simple_trace_create", "SimpleTraceCreateLinks": ".simple_trace_create_links", "SimpleTraceEdit": ".simple_trace_edit", "SimpleTraceEditLinks": ".simple_trace_edit_links", "SimpleTraceKind": ".simple_trace_kind", "SimpleTraceLinkResponse": ".simple_trace_link_response", "SimpleTraceLinks": ".simple_trace_links", "SimpleTraceOrigin": ".simple_trace_origin", "SimpleTraceQuery": ".simple_trace_query", "SimpleTraceQueryLinks": ".simple_trace_query_links", "SimpleTraceReferences": ".simple_trace_references", "SimpleTraceResponse": ".simple_trace_response", "SimpleTracesResponse": ".simple_traces_response", "SimpleWorkflow": ".simple_workflow", "SimpleWorkflowCreate": ".simple_workflow_create", "SimpleWorkflowDataInput": ".simple_workflow_data_input", "SimpleWorkflowDataInputHeadersValue": ".simple_workflow_data_input_headers_value", "SimpleWorkflowDataInputRuntime": ".simple_workflow_data_input_runtime", "SimpleWorkflowDataOutput": ".simple_workflow_data_output", "SimpleWorkflowDataOutputHeadersValue": ".simple_workflow_data_output_headers_value", "SimpleWorkflowDataOutputRuntime": ".simple_workflow_data_output_runtime", "SimpleWorkflowEdit": ".simple_workflow_edit", "SimpleWorkflowFlags": ".simple_workflow_flags", "SimpleWorkflowQuery": ".simple_workflow_query", "SimpleWorkflowQueryFlags": ".simple_workflow_query_flags", "SimpleWorkflowResponse": ".simple_workflow_response", "SimpleWorkflowsResponse": ".simple_workflows_response", "SpanInput": ".span_input", "SpanInputEndTime": ".span_input_end_time", "SpanInputStartTime": ".span_input_start_time", "SpanOutput": ".span_output", "SpanOutputEndTime": ".span_output_end_time", "SpanOutputStartTime": ".span_output_start_time", "SpanResponse": ".span_response", "SpanType": ".span_type", "SpansNodeInput": ".spans_node_input", "SpansNodeInputEndTime": ".spans_node_input_end_time", "SpansNodeInputSpansValue": ".spans_node_input_spans_value", "SpansNodeInputStartTime": ".spans_node_input_start_time", "SpansNodeOutput": ".spans_node_output", "SpansNodeOutputEndTime": ".spans_node_output_end_time", "SpansNodeOutputSpansValue": ".spans_node_output_spans_value", "SpansNodeOutputStartTime": ".spans_node_output_start_time", "SpansResponse": ".spans_response", "SpansTreeInput": ".spans_tree_input", "SpansTreeInputSpansValue": ".spans_tree_input_spans_value", "SpansTreeOutput": ".spans_tree_output", "SpansTreeOutputSpansValue": ".spans_tree_output_spans_value", "SsoProviderDto": ".sso_provider_dto", "SsoProviderInfo": ".sso_provider_info", "SsoProviderSettingsDto": ".sso_provider_settings_dto", "SsoProviders": ".sso_providers", "StandardProviderDto": ".standard_provider_dto", "StandardProviderKind": ".standard_provider_kind", "StandardProviderSettingsDto": ".standard_provider_settings_dto", "Status": ".status", "StringOperator": ".string_operator", "TestcaseInput": ".testcase_input", "TestcaseOutput": ".testcase_output", "TestcaseResponse": ".testcase_response", "TestcasesResponse": ".testcases_response", "Testset": ".testset", "TestsetCreate": ".testset_create", "TestsetEdit": ".testset_edit", "TestsetFlags": ".testset_flags", "TestsetQuery": ".testset_query", "TestsetResponse": ".testset_response", "TestsetRevision": ".testset_revision", "TestsetRevisionCommit": ".testset_revision_commit", "TestsetRevisionCreate": ".testset_revision_create", "TestsetRevisionDataInput": ".testset_revision_data_input", "TestsetRevisionDataOutput": ".testset_revision_data_output", "TestsetRevisionDelta": ".testset_revision_delta", "TestsetRevisionDeltaColumns": ".testset_revision_delta_columns", "TestsetRevisionDeltaRows": ".testset_revision_delta_rows", "TestsetRevisionEdit": ".testset_revision_edit", "TestsetRevisionQuery": ".testset_revision_query", "TestsetRevisionResponse": ".testset_revision_response", "TestsetRevisionsLog": ".testset_revisions_log", "TestsetRevisionsResponse": ".testset_revisions_response", "TestsetVariant": ".testset_variant", "TestsetVariantCreate": ".testset_variant_create", "TestsetVariantEdit": ".testset_variant_edit", "TestsetVariantQuery": ".testset_variant_query", "TestsetVariantResponse": ".testset_variant_response", "TestsetVariantsResponse": ".testset_variants_response", "TestsetsResponse": ".testsets_response", "TextOptions": ".text_options", "ToolAuthScheme": ".tool_auth_scheme", "ToolCallData": ".tool_call_data", "ToolCallFunction": ".tool_call_function", "ToolCallResponse": ".tool_call_response", "ToolCatalogAction": ".tool_catalog_action", "ToolCatalogActionDetails": ".tool_catalog_action_details", "ToolCatalogActionResponse": ".tool_catalog_action_response", "ToolCatalogActionResponseAction": ".tool_catalog_action_response_action", "ToolCatalogActionsResponse": ".tool_catalog_actions_response", "ToolCatalogActionsResponseActionsItem": ".tool_catalog_actions_response_actions_item", "ToolCatalogIntegration": ".tool_catalog_integration", "ToolCatalogIntegrationDetails": ".tool_catalog_integration_details", "ToolCatalogIntegrationResponse": ".tool_catalog_integration_response", "ToolCatalogIntegrationResponseIntegration": ".tool_catalog_integration_response_integration", "ToolCatalogIntegrationsResponse": ".tool_catalog_integrations_response", "ToolCatalogIntegrationsResponseIntegrationsItem": ".tool_catalog_integrations_response_integrations_item", "ToolCatalogProvider": ".tool_catalog_provider", "ToolCatalogProviderDetails": ".tool_catalog_provider_details", "ToolCatalogProviderResponse": ".tool_catalog_provider_response", "ToolCatalogProviderResponseProvider": ".tool_catalog_provider_response_provider", "ToolCatalogProvidersResponse": ".tool_catalog_providers_response", "ToolCatalogProvidersResponseProvidersItem": ".tool_catalog_providers_response_providers_item", "ToolConnection": ".tool_connection", "ToolConnectionCreate": ".tool_connection_create", "ToolConnectionCreateData": ".tool_connection_create_data", "ToolConnectionResponse": ".tool_connection_response", "ToolConnectionStatus": ".tool_connection_status", "ToolConnectionsResponse": ".tool_connections_response", "ToolProviderKind": ".tool_provider_kind", "ToolResult": ".tool_result", "ToolResultData": ".tool_result_data", "TraceIdResponse": ".trace_id_response", "TraceIdsResponse": ".trace_ids_response", "TraceInput": ".trace_input", "TraceInputSpansValue": ".trace_input_spans_value", "TraceOutput": ".trace_output", "TraceOutputSpansValue": ".trace_output_spans_value", "TraceRequest": ".trace_request", "TraceResponse": ".trace_response", "TraceType": ".trace_type", "TracesRequest": ".traces_request", "TracesResponse": ".traces_response", "TracingQuery": ".tracing_query", "UserIdsResponse": ".user_ids_response", "ValidationError": ".validation_error", "ValidationErrorLocItem": ".validation_error_loc_item", "VariantFork": ".variant_fork", "WebhookDeliveriesResponse": ".webhook_deliveries_response", "WebhookDelivery": ".webhook_delivery", "WebhookDeliveryCreate": ".webhook_delivery_create", "WebhookDeliveryData": ".webhook_delivery_data", "WebhookDeliveryQuery": ".webhook_delivery_query", "WebhookDeliveryResponse": ".webhook_delivery_response", "WebhookDeliveryResponseInfo": ".webhook_delivery_response_info", "WebhookEventType": ".webhook_event_type", "WebhookProviderDto": ".webhook_provider_dto", "WebhookProviderSettingsDto": ".webhook_provider_settings_dto", "WebhookSubscription": ".webhook_subscription", "WebhookSubscriptionCreate": ".webhook_subscription_create", "WebhookSubscriptionData": ".webhook_subscription_data", "WebhookSubscriptionDataAuthMode": ".webhook_subscription_data_auth_mode", "WebhookSubscriptionEdit": ".webhook_subscription_edit", "WebhookSubscriptionQuery": ".webhook_subscription_query", "WebhookSubscriptionResponse": ".webhook_subscription_response", "WebhookSubscriptionsResponse": ".webhook_subscriptions_response", "Windowing": ".windowing", "WindowingOrder": ".windowing_order", "Workflow": ".workflow", "WorkflowArtifactFlags": ".workflow_artifact_flags", "WorkflowCatalogFlags": ".workflow_catalog_flags", "WorkflowCatalogPreset": ".workflow_catalog_preset", "WorkflowCatalogPresetResponse": ".workflow_catalog_preset_response", "WorkflowCatalogPresetsResponse": ".workflow_catalog_presets_response", "WorkflowCatalogTemplate": ".workflow_catalog_template", "WorkflowCatalogTemplateResponse": ".workflow_catalog_template_response", "WorkflowCatalogTemplatesResponse": ".workflow_catalog_templates_response", "WorkflowCatalogType": ".workflow_catalog_type", "WorkflowCatalogTypeResponse": ".workflow_catalog_type_response", "WorkflowCatalogTypesResponse": ".workflow_catalog_types_response", "WorkflowCreate": ".workflow_create", "WorkflowEdit": ".workflow_edit", "WorkflowFlags": ".workflow_flags", "WorkflowFork": ".workflow_fork", "WorkflowResponse": ".workflow_response", "WorkflowRevisionCommit": ".workflow_revision_commit", "WorkflowRevisionCreate": ".workflow_revision_create", "WorkflowRevisionDataInput": ".workflow_revision_data_input", "WorkflowRevisionDataInputHeadersValue": ".workflow_revision_data_input_headers_value", "WorkflowRevisionDataInputRuntime": ".workflow_revision_data_input_runtime", "WorkflowRevisionDataOutput": ".workflow_revision_data_output", "WorkflowRevisionDataOutputHeadersValue": ".workflow_revision_data_output_headers_value", "WorkflowRevisionDataOutputRuntime": ".workflow_revision_data_output_runtime", "WorkflowRevisionEdit": ".workflow_revision_edit", "WorkflowRevisionFlags": ".workflow_revision_flags", "WorkflowRevisionFork": ".workflow_revision_fork", "WorkflowRevisionInput": ".workflow_revision_input", "WorkflowRevisionOutput": ".workflow_revision_output", "WorkflowRevisionResolveResponse": ".workflow_revision_resolve_response", "WorkflowRevisionResponse": ".workflow_revision_response", "WorkflowRevisionsLog": ".workflow_revisions_log", "WorkflowRevisionsResponse": ".workflow_revisions_response", "WorkflowVariant": ".workflow_variant", "WorkflowVariantCreate": ".workflow_variant_create", "WorkflowVariantEdit": ".workflow_variant_edit", "WorkflowVariantFlags": ".workflow_variant_flags", "WorkflowVariantFork": ".workflow_variant_fork", "WorkflowVariantResponse": ".workflow_variant_response", "WorkflowVariantsResponse": ".workflow_variants_response", "WorkflowsResponse": ".workflows_response", "Workspace": ".workspace", "WorkspaceMemberResponse": ".workspace_member_response", "WorkspacePermission": ".workspace_permission", "WorkspaceResponse": ".workspace_response"} +_dynamic_imports: typing.Dict[str, str] = {"AdminAccountCreateOptions": ".admin_account_create_options", "AdminAccountRead": ".admin_account_read", "AdminAccountsCreate": ".admin_accounts_create", "AdminAccountsDelete": ".admin_accounts_delete", "AdminAccountsDeleteTarget": ".admin_accounts_delete_target", "AdminAccountsResponse": ".admin_accounts_response", "AdminApiKeyCreate": ".admin_api_key_create", "AdminApiKeyResponse": ".admin_api_key_response", "AdminDeleteResponse": ".admin_delete_response", "AdminDeletedEntities": ".admin_deleted_entities", "AdminDeletedEntity": ".admin_deleted_entity", "AdminOrganizationCreate": ".admin_organization_create", "AdminOrganizationMembershipCreate": ".admin_organization_membership_create", "AdminOrganizationMembershipRead": ".admin_organization_membership_read", "AdminOrganizationRead": ".admin_organization_read", "AdminProjectCreate": ".admin_project_create", "AdminProjectMembershipCreate": ".admin_project_membership_create", "AdminProjectMembershipRead": ".admin_project_membership_read", "AdminProjectRead": ".admin_project_read", "AdminSimpleAccountCreate": ".admin_simple_account_create", "AdminSimpleAccountDeleteEntry": ".admin_simple_account_delete_entry", "AdminSimpleAccountRead": ".admin_simple_account_read", "AdminSimpleAccountsApiKeysCreate": ".admin_simple_accounts_api_keys_create", "AdminSimpleAccountsCreate": ".admin_simple_accounts_create", "AdminSimpleAccountsDelete": ".admin_simple_accounts_delete", "AdminSimpleAccountsOrganizationsCreate": ".admin_simple_accounts_organizations_create", "AdminSimpleAccountsOrganizationsMembershipsCreate": ".admin_simple_accounts_organizations_memberships_create", "AdminSimpleAccountsOrganizationsTransferOwnership": ".admin_simple_accounts_organizations_transfer_ownership", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjects": ".admin_simple_accounts_organizations_transfer_ownership_include_projects", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjectsZero": ".admin_simple_accounts_organizations_transfer_ownership_include_projects_zero", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspaces": ".admin_simple_accounts_organizations_transfer_ownership_include_workspaces", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspacesZero": ".admin_simple_accounts_organizations_transfer_ownership_include_workspaces_zero", "AdminSimpleAccountsOrganizationsTransferOwnershipResponse": ".admin_simple_accounts_organizations_transfer_ownership_response", "AdminSimpleAccountsProjectsCreate": ".admin_simple_accounts_projects_create", "AdminSimpleAccountsProjectsMembershipsCreate": ".admin_simple_accounts_projects_memberships_create", "AdminSimpleAccountsResponse": ".admin_simple_accounts_response", "AdminSimpleAccountsUsersCreate": ".admin_simple_accounts_users_create", "AdminSimpleAccountsUsersIdentitiesCreate": ".admin_simple_accounts_users_identities_create", "AdminSimpleAccountsUsersResetPassword": ".admin_simple_accounts_users_reset_password", "AdminSimpleAccountsWorkspacesCreate": ".admin_simple_accounts_workspaces_create", "AdminSimpleAccountsWorkspacesMembershipsCreate": ".admin_simple_accounts_workspaces_memberships_create", "AdminStructuredError": ".admin_structured_error", "AdminSubscriptionCreate": ".admin_subscription_create", "AdminSubscriptionRead": ".admin_subscription_read", "AdminUserCreate": ".admin_user_create", "AdminUserIdentityCreate": ".admin_user_identity_create", "AdminUserIdentityRead": ".admin_user_identity_read", "AdminUserIdentityReadStatus": ".admin_user_identity_read_status", "AdminUserRead": ".admin_user_read", "AdminWorkspaceCreate": ".admin_workspace_create", "AdminWorkspaceMembershipCreate": ".admin_workspace_membership_create", "AdminWorkspaceMembershipRead": ".admin_workspace_membership_read", "AdminWorkspaceRead": ".admin_workspace_read", "Analytics": ".analytics", "AnalyticsResponse": ".analytics_response", "Annotation": ".annotation", "AnnotationCreate": ".annotation_create", "AnnotationCreateLinks": ".annotation_create_links", "AnnotationEdit": ".annotation_edit", "AnnotationEditLinks": ".annotation_edit_links", "AnnotationLinkResponse": ".annotation_link_response", "AnnotationLinks": ".annotation_links", "AnnotationQuery": ".annotation_query", "AnnotationQueryLinks": ".annotation_query_links", "AnnotationResponse": ".annotation_response", "AnnotationsResponse": ".annotations_response", "Application": ".application", "ApplicationArtifactFlags": ".application_artifact_flags", "ApplicationArtifactQueryFlags": ".application_artifact_query_flags", "ApplicationCatalogPreset": ".application_catalog_preset", "ApplicationCatalogPresetResponse": ".application_catalog_preset_response", "ApplicationCatalogPresetsResponse": ".application_catalog_presets_response", "ApplicationCatalogTemplate": ".application_catalog_template", "ApplicationCatalogTemplateResponse": ".application_catalog_template_response", "ApplicationCatalogTemplatesResponse": ".application_catalog_templates_response", "ApplicationCatalogType": ".application_catalog_type", "ApplicationCatalogTypesResponse": ".application_catalog_types_response", "ApplicationCreate": ".application_create", "ApplicationEdit": ".application_edit", "ApplicationFlags": ".application_flags", "ApplicationFork": ".application_fork", "ApplicationQuery": ".application_query", "ApplicationResponse": ".application_response", "ApplicationRevision": ".application_revision", "ApplicationRevisionCommit": ".application_revision_commit", "ApplicationRevisionCreate": ".application_revision_create", "ApplicationRevisionDataInput": ".application_revision_data_input", "ApplicationRevisionDataInputHeadersValue": ".application_revision_data_input_headers_value", "ApplicationRevisionDataInputRuntime": ".application_revision_data_input_runtime", "ApplicationRevisionDataOutput": ".application_revision_data_output", "ApplicationRevisionDataOutputHeadersValue": ".application_revision_data_output_headers_value", "ApplicationRevisionDataOutputRuntime": ".application_revision_data_output_runtime", "ApplicationRevisionEdit": ".application_revision_edit", "ApplicationRevisionFlags": ".application_revision_flags", "ApplicationRevisionFork": ".application_revision_fork", "ApplicationRevisionQuery": ".application_revision_query", "ApplicationRevisionQueryFlags": ".application_revision_query_flags", "ApplicationRevisionResolveResponse": ".application_revision_resolve_response", "ApplicationRevisionResponse": ".application_revision_response", "ApplicationRevisionsLog": ".application_revisions_log", "ApplicationRevisionsResponse": ".application_revisions_response", "ApplicationVariant": ".application_variant", "ApplicationVariantCreate": ".application_variant_create", "ApplicationVariantEdit": ".application_variant_edit", "ApplicationVariantFlags": ".application_variant_flags", "ApplicationVariantFork": ".application_variant_fork", "ApplicationVariantResponse": ".application_variant_response", "ApplicationVariantsResponse": ".application_variants_response", "ApplicationsResponse": ".applications_response", "BodyConfigsFetchVariantsConfigsFetchPost": ".body_configs_fetch_variants_configs_fetch_post", "Bucket": ".bucket", "CollectStatusResponse": ".collect_status_response", "ComparisonOperator": ".comparison_operator", "Condition": ".condition", "ConditionOperator": ".condition_operator", "ConditionOptions": ".condition_options", "ConditionValue": ".condition_value", "ConfigResponseModel": ".config_response_model", "CustomModelSettingsDto": ".custom_model_settings_dto", "CustomProviderDto": ".custom_provider_dto", "CustomProviderKind": ".custom_provider_kind", "CustomProviderSettingsDto": ".custom_provider_settings_dto", "DictOperator": ".dict_operator", "DiscoverResponse": ".discover_response", "DiscoverResponseMethodsValue": ".discover_response_methods_value", "EeSrcModelsApiOrganizationModelsOrganization": ".ee_src_models_api_organization_models_organization", "EntityRef": ".entity_ref", "Environment": ".environment", "EnvironmentCreate": ".environment_create", "EnvironmentEdit": ".environment_edit", "EnvironmentFlags": ".environment_flags", "EnvironmentQueryFlags": ".environment_query_flags", "EnvironmentResponse": ".environment_response", "EnvironmentRevision": ".environment_revision", "EnvironmentRevisionCommit": ".environment_revision_commit", "EnvironmentRevisionCreate": ".environment_revision_create", "EnvironmentRevisionData": ".environment_revision_data", "EnvironmentRevisionDelta": ".environment_revision_delta", "EnvironmentRevisionEdit": ".environment_revision_edit", "EnvironmentRevisionResolveResponse": ".environment_revision_resolve_response", "EnvironmentRevisionResponse": ".environment_revision_response", "EnvironmentRevisionsLog": ".environment_revisions_log", "EnvironmentRevisionsResponse": ".environment_revisions_response", "EnvironmentVariant": ".environment_variant", "EnvironmentVariantCreate": ".environment_variant_create", "EnvironmentVariantEdit": ".environment_variant_edit", "EnvironmentVariantResponse": ".environment_variant_response", "EnvironmentVariantsResponse": ".environment_variants_response", "EnvironmentsResponse": ".environments_response", "ErrorPolicy": ".error_policy", "EvaluationMetrics": ".evaluation_metrics", "EvaluationMetricsCreate": ".evaluation_metrics_create", "EvaluationMetricsIdsResponse": ".evaluation_metrics_ids_response", "EvaluationMetricsQuery": ".evaluation_metrics_query", "EvaluationMetricsQueryScenarioIds": ".evaluation_metrics_query_scenario_ids", "EvaluationMetricsQueryTimestamps": ".evaluation_metrics_query_timestamps", "EvaluationMetricsRefresh": ".evaluation_metrics_refresh", "EvaluationMetricsResponse": ".evaluation_metrics_response", "EvaluationQueue": ".evaluation_queue", "EvaluationQueueCreate": ".evaluation_queue_create", "EvaluationQueueData": ".evaluation_queue_data", "EvaluationQueueEdit": ".evaluation_queue_edit", "EvaluationQueueFlags": ".evaluation_queue_flags", "EvaluationQueueIdResponse": ".evaluation_queue_id_response", "EvaluationQueueIdsResponse": ".evaluation_queue_ids_response", "EvaluationQueueQuery": ".evaluation_queue_query", "EvaluationQueueQueryFlags": ".evaluation_queue_query_flags", "EvaluationQueueResponse": ".evaluation_queue_response", "EvaluationQueueScenariosQuery": ".evaluation_queue_scenarios_query", "EvaluationQueuesResponse": ".evaluation_queues_response", "EvaluationResult": ".evaluation_result", "EvaluationResultCreate": ".evaluation_result_create", "EvaluationResultIdResponse": ".evaluation_result_id_response", "EvaluationResultIdsResponse": ".evaluation_result_ids_response", "EvaluationResultQuery": ".evaluation_result_query", "EvaluationResultResponse": ".evaluation_result_response", "EvaluationResultsResponse": ".evaluation_results_response", "EvaluationRun": ".evaluation_run", "EvaluationRunCreate": ".evaluation_run_create", "EvaluationRunData": ".evaluation_run_data", "EvaluationRunDataConcurrency": ".evaluation_run_data_concurrency", "EvaluationRunDataMapping": ".evaluation_run_data_mapping", "EvaluationRunDataMappingColumn": ".evaluation_run_data_mapping_column", "EvaluationRunDataMappingStep": ".evaluation_run_data_mapping_step", "EvaluationRunDataStep": ".evaluation_run_data_step", "EvaluationRunDataStepInput": ".evaluation_run_data_step_input", "EvaluationRunDataStepOrigin": ".evaluation_run_data_step_origin", "EvaluationRunDataStepType": ".evaluation_run_data_step_type", "EvaluationRunEdit": ".evaluation_run_edit", "EvaluationRunFlags": ".evaluation_run_flags", "EvaluationRunIdResponse": ".evaluation_run_id_response", "EvaluationRunIdsRequest": ".evaluation_run_ids_request", "EvaluationRunIdsResponse": ".evaluation_run_ids_response", "EvaluationRunQuery": ".evaluation_run_query", "EvaluationRunQueryFlags": ".evaluation_run_query_flags", "EvaluationRunResponse": ".evaluation_run_response", "EvaluationRunsResponse": ".evaluation_runs_response", "EvaluationScenario": ".evaluation_scenario", "EvaluationScenarioCreate": ".evaluation_scenario_create", "EvaluationScenarioEdit": ".evaluation_scenario_edit", "EvaluationScenarioIdResponse": ".evaluation_scenario_id_response", "EvaluationScenarioIdsResponse": ".evaluation_scenario_ids_response", "EvaluationScenarioQuery": ".evaluation_scenario_query", "EvaluationScenarioResponse": ".evaluation_scenario_response", "EvaluationScenariosResponse": ".evaluation_scenarios_response", "EvaluationStatus": ".evaluation_status", "Evaluator": ".evaluator", "EvaluatorArtifactFlags": ".evaluator_artifact_flags", "EvaluatorArtifactQueryFlags": ".evaluator_artifact_query_flags", "EvaluatorCatalogPreset": ".evaluator_catalog_preset", "EvaluatorCatalogPresetResponse": ".evaluator_catalog_preset_response", "EvaluatorCatalogPresetsResponse": ".evaluator_catalog_presets_response", "EvaluatorCatalogTemplate": ".evaluator_catalog_template", "EvaluatorCatalogTemplateResponse": ".evaluator_catalog_template_response", "EvaluatorCatalogTemplatesResponse": ".evaluator_catalog_templates_response", "EvaluatorCatalogType": ".evaluator_catalog_type", "EvaluatorCatalogTypesResponse": ".evaluator_catalog_types_response", "EvaluatorCreate": ".evaluator_create", "EvaluatorEdit": ".evaluator_edit", "EvaluatorFlags": ".evaluator_flags", "EvaluatorFork": ".evaluator_fork", "EvaluatorQuery": ".evaluator_query", "EvaluatorResponse": ".evaluator_response", "EvaluatorRevision": ".evaluator_revision", "EvaluatorRevisionCommit": ".evaluator_revision_commit", "EvaluatorRevisionCreate": ".evaluator_revision_create", "EvaluatorRevisionDataInput": ".evaluator_revision_data_input", "EvaluatorRevisionDataInputHeadersValue": ".evaluator_revision_data_input_headers_value", "EvaluatorRevisionDataInputRuntime": ".evaluator_revision_data_input_runtime", "EvaluatorRevisionDataOutput": ".evaluator_revision_data_output", "EvaluatorRevisionDataOutputHeadersValue": ".evaluator_revision_data_output_headers_value", "EvaluatorRevisionDataOutputRuntime": ".evaluator_revision_data_output_runtime", "EvaluatorRevisionEdit": ".evaluator_revision_edit", "EvaluatorRevisionFlags": ".evaluator_revision_flags", "EvaluatorRevisionFork": ".evaluator_revision_fork", "EvaluatorRevisionQuery": ".evaluator_revision_query", "EvaluatorRevisionQueryFlags": ".evaluator_revision_query_flags", "EvaluatorRevisionResolveResponse": ".evaluator_revision_resolve_response", "EvaluatorRevisionResponse": ".evaluator_revision_response", "EvaluatorRevisionsLog": ".evaluator_revisions_log", "EvaluatorRevisionsResponse": ".evaluator_revisions_response", "EvaluatorTemplate": ".evaluator_template", "EvaluatorTemplatesResponse": ".evaluator_templates_response", "EvaluatorVariant": ".evaluator_variant", "EvaluatorVariantCreate": ".evaluator_variant_create", "EvaluatorVariantEdit": ".evaluator_variant_edit", "EvaluatorVariantFlags": ".evaluator_variant_flags", "EvaluatorVariantFork": ".evaluator_variant_fork", "EvaluatorVariantResponse": ".evaluator_variant_response", "EvaluatorVariantsResponse": ".evaluator_variants_response", "EvaluatorsResponse": ".evaluators_response", "Event": ".event", "EventQuery": ".event_query", "EventType": ".event_type", "EventsQueryResponse": ".events_query_response", "ExistenceOperator": ".existence_operator", "FilteringInput": ".filtering_input", "FilteringInputConditionsItem": ".filtering_input_conditions_item", "FilteringOutput": ".filtering_output", "FilteringOutputConditionsItem": ".filtering_output_conditions_item", "Focus": ".focus", "Folder": ".folder", "FolderCreate": ".folder_create", "FolderEdit": ".folder_edit", "FolderIdResponse": ".folder_id_response", "FolderKind": ".folder_kind", "FolderQuery": ".folder_query", "FolderQueryKinds": ".folder_query_kinds", "FolderResponse": ".folder_response", "FoldersResponse": ".folders_response", "Format": ".format", "Formatting": ".formatting", typing.Any: ".full_json_input", typing.Any: ".full_json_output", "Header": ".header", "HttpValidationError": ".http_validation_error", "InviteRequest": ".invite_request", "Invocation": ".invocation", "InvocationCreate": ".invocation_create", "InvocationCreateLinks": ".invocation_create_links", "InvocationEdit": ".invocation_edit", "InvocationEditLinks": ".invocation_edit_links", "InvocationLinkResponse": ".invocation_link_response", "InvocationLinks": ".invocation_links", "InvocationQuery": ".invocation_query", "InvocationQueryLinks": ".invocation_query_links", "InvocationResponse": ".invocation_response", "InvocationsResponse": ".invocations_response", "JsonSchemasInput": ".json_schemas_input", "JsonSchemasOutput": ".json_schemas_output", typing.Any: ".label_json_input", typing.Any: ".label_json_output", "LegacyLifecycleDto": ".legacy_lifecycle_dto", "ListApiKeysResponse": ".list_api_keys_response", "ListOperator": ".list_operator", "ListOptions": ".list_options", "LogicalOperator": ".logical_operator", "MetricSpec": ".metric_spec", "MetricType": ".metric_type", "MetricsBucket": ".metrics_bucket", "NumericOperator": ".numeric_operator", "OTelEventInput": ".o_tel_event_input", "OTelEventInputTimestamp": ".o_tel_event_input_timestamp", "OTelEventOutput": ".o_tel_event_output", "OTelEventOutputTimestamp": ".o_tel_event_output_timestamp", "OTelHashInput": ".o_tel_hash_input", "OTelHashOutput": ".o_tel_hash_output", "OTelLinkInput": ".o_tel_link_input", "OTelLinkOutput": ".o_tel_link_output", "OTelLinksResponse": ".o_tel_links_response", "OTelReferenceInput": ".o_tel_reference_input", "OTelReferenceOutput": ".o_tel_reference_output", "OTelSpanKind": ".o_tel_span_kind", "OTelStatusCode": ".o_tel_status_code", "OTelTracingRequest": ".o_tel_tracing_request", "OTelTracingResponse": ".o_tel_tracing_response", "OldAnalyticsResponse": ".old_analytics_response", "OrganizationDetails": ".organization_details", "OrganizationDomainResponse": ".organization_domain_response", "OrganizationProviderResponse": ".organization_provider_response", "OrganizationUpdate": ".organization_update", "OssSrcModelsApiOrganizationModelsOrganization": ".oss_src_models_api_organization_models_organization", "Permission": ".permission", "ProjectsResponse": ".projects_response", "QueriesResponse": ".queries_response", "Query": ".query", "QueryCreate": ".query_create", "QueryEdit": ".query_edit", "QueryFlags": ".query_flags", "QueryQueryFlags": ".query_query_flags", "QueryResponse": ".query_response", "QueryRevision": ".query_revision", "QueryRevisionCommit": ".query_revision_commit", "QueryRevisionCreate": ".query_revision_create", "QueryRevisionDataInput": ".query_revision_data_input", "QueryRevisionDataOutput": ".query_revision_data_output", "QueryRevisionEdit": ".query_revision_edit", "QueryRevisionQuery": ".query_revision_query", "QueryRevisionResponse": ".query_revision_response", "QueryRevisionsLog": ".query_revisions_log", "QueryRevisionsResponse": ".query_revisions_response", "QueryVariant": ".query_variant", "QueryVariantCreate": ".query_variant_create", "QueryVariantEdit": ".query_variant_edit", "QueryVariantQuery": ".query_variant_query", "QueryVariantResponse": ".query_variant_response", "QueryVariantsResponse": ".query_variants_response", "Reference": ".reference", "ReferenceRequestModelInput": ".reference_request_model_input", "ReferenceRequestModelOutput": ".reference_request_model_output", "RequestType": ".request_type", "ResolutionInfo": ".resolution_info", "RetrievalInfo": ".retrieval_info", "RevisionFork": ".revision_fork", "SecretDto": ".secret_dto", "SecretDtoData": ".secret_dto_data", "SecretKind": ".secret_kind", "SecretResponseDto": ".secret_response_dto", "SecretResponseDtoData": ".secret_response_dto_data", "SessionIdsResponse": ".session_ids_response", "SimpleApplication": ".simple_application", "SimpleApplicationCreate": ".simple_application_create", "SimpleApplicationDataInput": ".simple_application_data_input", "SimpleApplicationDataInputHeadersValue": ".simple_application_data_input_headers_value", "SimpleApplicationDataInputRuntime": ".simple_application_data_input_runtime", "SimpleApplicationDataOutput": ".simple_application_data_output", "SimpleApplicationDataOutputHeadersValue": ".simple_application_data_output_headers_value", "SimpleApplicationDataOutputRuntime": ".simple_application_data_output_runtime", "SimpleApplicationEdit": ".simple_application_edit", "SimpleApplicationFlags": ".simple_application_flags", "SimpleApplicationQuery": ".simple_application_query", "SimpleApplicationQueryFlags": ".simple_application_query_flags", "SimpleApplicationResponse": ".simple_application_response", "SimpleApplicationsResponse": ".simple_applications_response", "SimpleEnvironment": ".simple_environment", "SimpleEnvironmentCreate": ".simple_environment_create", "SimpleEnvironmentEdit": ".simple_environment_edit", "SimpleEnvironmentQuery": ".simple_environment_query", "SimpleEnvironmentResponse": ".simple_environment_response", "SimpleEnvironmentsResponse": ".simple_environments_response", "SimpleEvaluation": ".simple_evaluation", "SimpleEvaluationCreate": ".simple_evaluation_create", "SimpleEvaluationData": ".simple_evaluation_data", "SimpleEvaluationDataApplicationSteps": ".simple_evaluation_data_application_steps", "SimpleEvaluationDataApplicationStepsOneValue": ".simple_evaluation_data_application_steps_one_value", "SimpleEvaluationDataEvaluatorSteps": ".simple_evaluation_data_evaluator_steps", "SimpleEvaluationDataEvaluatorStepsOneValue": ".simple_evaluation_data_evaluator_steps_one_value", "SimpleEvaluationDataQuerySteps": ".simple_evaluation_data_query_steps", "SimpleEvaluationDataQueryStepsOneValue": ".simple_evaluation_data_query_steps_one_value", "SimpleEvaluationDataTestsetSteps": ".simple_evaluation_data_testset_steps", "SimpleEvaluationDataTestsetStepsOneValue": ".simple_evaluation_data_testset_steps_one_value", "SimpleEvaluationEdit": ".simple_evaluation_edit", "SimpleEvaluationIdResponse": ".simple_evaluation_id_response", "SimpleEvaluationQuery": ".simple_evaluation_query", "SimpleEvaluationResponse": ".simple_evaluation_response", "SimpleEvaluationsResponse": ".simple_evaluations_response", "SimpleEvaluator": ".simple_evaluator", "SimpleEvaluatorCreate": ".simple_evaluator_create", "SimpleEvaluatorDataInput": ".simple_evaluator_data_input", "SimpleEvaluatorDataInputHeadersValue": ".simple_evaluator_data_input_headers_value", "SimpleEvaluatorDataInputRuntime": ".simple_evaluator_data_input_runtime", "SimpleEvaluatorDataOutput": ".simple_evaluator_data_output", "SimpleEvaluatorDataOutputHeadersValue": ".simple_evaluator_data_output_headers_value", "SimpleEvaluatorDataOutputRuntime": ".simple_evaluator_data_output_runtime", "SimpleEvaluatorEdit": ".simple_evaluator_edit", "SimpleEvaluatorFlags": ".simple_evaluator_flags", "SimpleEvaluatorQuery": ".simple_evaluator_query", "SimpleEvaluatorQueryFlags": ".simple_evaluator_query_flags", "SimpleEvaluatorResponse": ".simple_evaluator_response", "SimpleEvaluatorsResponse": ".simple_evaluators_response", "SimpleQueriesResponse": ".simple_queries_response", "SimpleQuery": ".simple_query", "SimpleQueryCreate": ".simple_query_create", "SimpleQueryEdit": ".simple_query_edit", "SimpleQueryQuery": ".simple_query_query", "SimpleQueryResponse": ".simple_query_response", "SimpleQueue": ".simple_queue", "SimpleQueueCreate": ".simple_queue_create", "SimpleQueueData": ".simple_queue_data", "SimpleQueueDataEvaluators": ".simple_queue_data_evaluators", "SimpleQueueDataEvaluatorsOneValue": ".simple_queue_data_evaluators_one_value", "SimpleQueueIdResponse": ".simple_queue_id_response", "SimpleQueueKind": ".simple_queue_kind", "SimpleQueueQuery": ".simple_queue_query", "SimpleQueueResponse": ".simple_queue_response", "SimpleQueueScenariosQuery": ".simple_queue_scenarios_query", "SimpleQueueScenariosResponse": ".simple_queue_scenarios_response", "SimpleQueueSettings": ".simple_queue_settings", "SimpleQueuesResponse": ".simple_queues_response", "SimpleTestset": ".simple_testset", "SimpleTestsetCreate": ".simple_testset_create", "SimpleTestsetEdit": ".simple_testset_edit", "SimpleTestsetQuery": ".simple_testset_query", "SimpleTestsetResponse": ".simple_testset_response", "SimpleTestsetsResponse": ".simple_testsets_response", "SimpleTrace": ".simple_trace", "SimpleTraceChannel": ".simple_trace_channel", "SimpleTraceCreate": ".simple_trace_create", "SimpleTraceCreateLinks": ".simple_trace_create_links", "SimpleTraceEdit": ".simple_trace_edit", "SimpleTraceEditLinks": ".simple_trace_edit_links", "SimpleTraceKind": ".simple_trace_kind", "SimpleTraceLinkResponse": ".simple_trace_link_response", "SimpleTraceLinks": ".simple_trace_links", "SimpleTraceOrigin": ".simple_trace_origin", "SimpleTraceQuery": ".simple_trace_query", "SimpleTraceQueryLinks": ".simple_trace_query_links", "SimpleTraceReferences": ".simple_trace_references", "SimpleTraceResponse": ".simple_trace_response", "SimpleTracesResponse": ".simple_traces_response", "SimpleWorkflow": ".simple_workflow", "SimpleWorkflowCreate": ".simple_workflow_create", "SimpleWorkflowDataInput": ".simple_workflow_data_input", "SimpleWorkflowDataInputHeadersValue": ".simple_workflow_data_input_headers_value", "SimpleWorkflowDataInputRuntime": ".simple_workflow_data_input_runtime", "SimpleWorkflowDataOutput": ".simple_workflow_data_output", "SimpleWorkflowDataOutputHeadersValue": ".simple_workflow_data_output_headers_value", "SimpleWorkflowDataOutputRuntime": ".simple_workflow_data_output_runtime", "SimpleWorkflowEdit": ".simple_workflow_edit", "SimpleWorkflowFlags": ".simple_workflow_flags", "SimpleWorkflowQuery": ".simple_workflow_query", "SimpleWorkflowQueryFlags": ".simple_workflow_query_flags", "SimpleWorkflowResponse": ".simple_workflow_response", "SimpleWorkflowsResponse": ".simple_workflows_response", "SpanInput": ".span_input", "SpanInputEndTime": ".span_input_end_time", "SpanInputStartTime": ".span_input_start_time", "SpanOutput": ".span_output", "SpanOutputEndTime": ".span_output_end_time", "SpanOutputStartTime": ".span_output_start_time", "SpanResponse": ".span_response", "SpanType": ".span_type", "SpansNodeInput": ".spans_node_input", "SpansNodeInputEndTime": ".spans_node_input_end_time", "SpansNodeInputSpansValue": ".spans_node_input_spans_value", "SpansNodeInputStartTime": ".spans_node_input_start_time", "SpansNodeOutput": ".spans_node_output", "SpansNodeOutputEndTime": ".spans_node_output_end_time", "SpansNodeOutputSpansValue": ".spans_node_output_spans_value", "SpansNodeOutputStartTime": ".spans_node_output_start_time", "SpansResponse": ".spans_response", "SpansTreeInput": ".spans_tree_input", "SpansTreeInputSpansValue": ".spans_tree_input_spans_value", "SpansTreeOutput": ".spans_tree_output", "SpansTreeOutputSpansValue": ".spans_tree_output_spans_value", "SsoProviderDto": ".sso_provider_dto", "SsoProviderInfo": ".sso_provider_info", "SsoProviderSettingsDto": ".sso_provider_settings_dto", "SsoProviders": ".sso_providers", "StandardProviderDto": ".standard_provider_dto", "StandardProviderKind": ".standard_provider_kind", "StandardProviderSettingsDto": ".standard_provider_settings_dto", "Status": ".status", "StringOperator": ".string_operator", "TestcaseInput": ".testcase_input", "TestcaseOutput": ".testcase_output", "TestcaseResponse": ".testcase_response", "TestcasesResponse": ".testcases_response", "Testset": ".testset", "TestsetCreate": ".testset_create", "TestsetEdit": ".testset_edit", "TestsetFlags": ".testset_flags", "TestsetQuery": ".testset_query", "TestsetResponse": ".testset_response", "TestsetRevision": ".testset_revision", "TestsetRevisionCommit": ".testset_revision_commit", "TestsetRevisionCreate": ".testset_revision_create", "TestsetRevisionDataInput": ".testset_revision_data_input", "TestsetRevisionDataOutput": ".testset_revision_data_output", "TestsetRevisionDelta": ".testset_revision_delta", "TestsetRevisionDeltaColumns": ".testset_revision_delta_columns", "TestsetRevisionDeltaRows": ".testset_revision_delta_rows", "TestsetRevisionEdit": ".testset_revision_edit", "TestsetRevisionQuery": ".testset_revision_query", "TestsetRevisionResponse": ".testset_revision_response", "TestsetRevisionsLog": ".testset_revisions_log", "TestsetRevisionsResponse": ".testset_revisions_response", "TestsetVariant": ".testset_variant", "TestsetVariantCreate": ".testset_variant_create", "TestsetVariantEdit": ".testset_variant_edit", "TestsetVariantQuery": ".testset_variant_query", "TestsetVariantResponse": ".testset_variant_response", "TestsetVariantsResponse": ".testset_variants_response", "TestsetsResponse": ".testsets_response", "TextOptions": ".text_options", "ToolAuthScheme": ".tool_auth_scheme", "ToolCallData": ".tool_call_data", "ToolCallFunction": ".tool_call_function", "ToolCallResponse": ".tool_call_response", "ToolCatalogAction": ".tool_catalog_action", "ToolCatalogActionDetails": ".tool_catalog_action_details", "ToolCatalogActionResponse": ".tool_catalog_action_response", "ToolCatalogActionResponseAction": ".tool_catalog_action_response_action", "ToolCatalogActionsResponse": ".tool_catalog_actions_response", "ToolCatalogActionsResponseActionsItem": ".tool_catalog_actions_response_actions_item", "ToolCatalogIntegration": ".tool_catalog_integration", "ToolCatalogIntegrationDetails": ".tool_catalog_integration_details", "ToolCatalogIntegrationResponse": ".tool_catalog_integration_response", "ToolCatalogIntegrationResponseIntegration": ".tool_catalog_integration_response_integration", "ToolCatalogIntegrationsResponse": ".tool_catalog_integrations_response", "ToolCatalogIntegrationsResponseIntegrationsItem": ".tool_catalog_integrations_response_integrations_item", "ToolCatalogProvider": ".tool_catalog_provider", "ToolCatalogProviderDetails": ".tool_catalog_provider_details", "ToolCatalogProviderResponse": ".tool_catalog_provider_response", "ToolCatalogProviderResponseProvider": ".tool_catalog_provider_response_provider", "ToolCatalogProvidersResponse": ".tool_catalog_providers_response", "ToolCatalogProvidersResponseProvidersItem": ".tool_catalog_providers_response_providers_item", "ToolConnection": ".tool_connection", "ToolConnectionCreate": ".tool_connection_create", "ToolConnectionCreateData": ".tool_connection_create_data", "ToolConnectionResponse": ".tool_connection_response", "ToolConnectionStatus": ".tool_connection_status", "ToolConnectionsResponse": ".tool_connections_response", "ToolProviderKind": ".tool_provider_kind", "ToolResult": ".tool_result", "ToolResultData": ".tool_result_data", "TraceIdResponse": ".trace_id_response", "TraceIdsResponse": ".trace_ids_response", "TraceInput": ".trace_input", "TraceInputSpansValue": ".trace_input_spans_value", "TraceOutput": ".trace_output", "TraceOutputSpansValue": ".trace_output_spans_value", "TraceRequest": ".trace_request", "TraceResponse": ".trace_response", "TraceType": ".trace_type", "TracesRequest": ".traces_request", "TracesResponse": ".traces_response", "TracingQuery": ".tracing_query", "UserIdsResponse": ".user_ids_response", "ValidationError": ".validation_error", "ValidationErrorLocItem": ".validation_error_loc_item", "VariantFork": ".variant_fork", "WebhookDeliveriesResponse": ".webhook_deliveries_response", "WebhookDelivery": ".webhook_delivery", "WebhookDeliveryCreate": ".webhook_delivery_create", "WebhookDeliveryData": ".webhook_delivery_data", "WebhookDeliveryQuery": ".webhook_delivery_query", "WebhookDeliveryResponse": ".webhook_delivery_response", "WebhookDeliveryResponseInfo": ".webhook_delivery_response_info", "WebhookEventType": ".webhook_event_type", "WebhookProviderDto": ".webhook_provider_dto", "WebhookProviderSettingsDto": ".webhook_provider_settings_dto", "WebhookSubscription": ".webhook_subscription", "WebhookSubscriptionCreate": ".webhook_subscription_create", "WebhookSubscriptionData": ".webhook_subscription_data", "WebhookSubscriptionDataAuthMode": ".webhook_subscription_data_auth_mode", "WebhookSubscriptionEdit": ".webhook_subscription_edit", "WebhookSubscriptionQuery": ".webhook_subscription_query", "WebhookSubscriptionResponse": ".webhook_subscription_response", "WebhookSubscriptionsResponse": ".webhook_subscriptions_response", "Windowing": ".windowing", "WindowingOrder": ".windowing_order", "Workflow": ".workflow", "WorkflowArtifactFlags": ".workflow_artifact_flags", "WorkflowCatalogFlags": ".workflow_catalog_flags", "WorkflowCatalogPreset": ".workflow_catalog_preset", "WorkflowCatalogPresetResponse": ".workflow_catalog_preset_response", "WorkflowCatalogPresetsResponse": ".workflow_catalog_presets_response", "WorkflowCatalogTemplate": ".workflow_catalog_template", "WorkflowCatalogTemplateResponse": ".workflow_catalog_template_response", "WorkflowCatalogTemplatesResponse": ".workflow_catalog_templates_response", "WorkflowCatalogType": ".workflow_catalog_type", "WorkflowCatalogTypeResponse": ".workflow_catalog_type_response", "WorkflowCatalogTypesResponse": ".workflow_catalog_types_response", "WorkflowCreate": ".workflow_create", "WorkflowEdit": ".workflow_edit", "WorkflowFlags": ".workflow_flags", "WorkflowFork": ".workflow_fork", "WorkflowResponse": ".workflow_response", "WorkflowRevisionCommit": ".workflow_revision_commit", "WorkflowRevisionCreate": ".workflow_revision_create", "WorkflowRevisionDataInput": ".workflow_revision_data_input", "WorkflowRevisionDataInputHeadersValue": ".workflow_revision_data_input_headers_value", "WorkflowRevisionDataInputRuntime": ".workflow_revision_data_input_runtime", "WorkflowRevisionDataOutput": ".workflow_revision_data_output", "WorkflowRevisionDataOutputHeadersValue": ".workflow_revision_data_output_headers_value", "WorkflowRevisionDataOutputRuntime": ".workflow_revision_data_output_runtime", "WorkflowRevisionEdit": ".workflow_revision_edit", "WorkflowRevisionFlags": ".workflow_revision_flags", "WorkflowRevisionFork": ".workflow_revision_fork", "WorkflowRevisionInput": ".workflow_revision_input", "WorkflowRevisionOutput": ".workflow_revision_output", "WorkflowRevisionResolveResponse": ".workflow_revision_resolve_response", "WorkflowRevisionResponse": ".workflow_revision_response", "WorkflowRevisionsLog": ".workflow_revisions_log", "WorkflowRevisionsResponse": ".workflow_revisions_response", "WorkflowVariant": ".workflow_variant", "WorkflowVariantCreate": ".workflow_variant_create", "WorkflowVariantEdit": ".workflow_variant_edit", "WorkflowVariantFlags": ".workflow_variant_flags", "WorkflowVariantFork": ".workflow_variant_fork", "WorkflowVariantResponse": ".workflow_variant_response", "WorkflowVariantsResponse": ".workflow_variants_response", "WorkflowsResponse": ".workflows_response", "Workspace": ".workspace", "WorkspaceMemberResponse": ".workspace_member_response", "WorkspacePermission": ".workspace_permission", "WorkspaceResponse": ".workspace_response"} def __getattr__(attr_name: str) -> typing.Any: module_name = _dynamic_imports.get(attr_name) if module_name is None: @@ -662,4 +661,4 @@ def __getattr__(attr_name: str) -> typing.Any: def __dir__(): lazy_attrs = list(_dynamic_imports.keys()) return sorted(lazy_attrs) -__all__ = ["AdminAccountCreateOptions", "AdminAccountRead", "AdminAccountsCreate", "AdminAccountsDelete", "AdminAccountsDeleteTarget", "AdminAccountsResponse", "AdminApiKeyCreate", "AdminApiKeyResponse", "AdminDeleteResponse", "AdminDeletedEntities", "AdminDeletedEntity", "AdminOrganizationCreate", "AdminOrganizationMembershipCreate", "AdminOrganizationMembershipRead", "AdminOrganizationRead", "AdminProjectCreate", "AdminProjectMembershipCreate", "AdminProjectMembershipRead", "AdminProjectRead", "AdminSimpleAccountCreate", "AdminSimpleAccountDeleteEntry", "AdminSimpleAccountRead", "AdminSimpleAccountsApiKeysCreate", "AdminSimpleAccountsCreate", "AdminSimpleAccountsDelete", "AdminSimpleAccountsOrganizationsCreate", "AdminSimpleAccountsOrganizationsMembershipsCreate", "AdminSimpleAccountsOrganizationsTransferOwnership", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjects", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjectsZero", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspaces", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspacesZero", "AdminSimpleAccountsOrganizationsTransferOwnershipResponse", "AdminSimpleAccountsProjectsCreate", "AdminSimpleAccountsProjectsMembershipsCreate", "AdminSimpleAccountsResponse", "AdminSimpleAccountsUsersCreate", "AdminSimpleAccountsUsersIdentitiesCreate", "AdminSimpleAccountsUsersResetPassword", "AdminSimpleAccountsWorkspacesCreate", "AdminSimpleAccountsWorkspacesMembershipsCreate", "AdminStructuredError", "AdminSubscriptionCreate", "AdminSubscriptionRead", "AdminUserCreate", "AdminUserIdentityCreate", "AdminUserIdentityRead", "AdminUserIdentityReadStatus", "AdminUserRead", "AdminWorkspaceCreate", "AdminWorkspaceMembershipCreate", "AdminWorkspaceMembershipRead", "AdminWorkspaceRead", "Analytics", "AnalyticsResponse", "Annotation", "AnnotationCreate", "AnnotationCreateLinks", "AnnotationEdit", "AnnotationEditLinks", "AnnotationLinkResponse", "AnnotationLinks", "AnnotationQuery", "AnnotationQueryLinks", "AnnotationResponse", "AnnotationsResponse", "Application", "ApplicationArtifactFlags", "ApplicationArtifactQueryFlags", "ApplicationCatalogPreset", "ApplicationCatalogPresetResponse", "ApplicationCatalogPresetsResponse", "ApplicationCatalogTemplate", "ApplicationCatalogTemplateResponse", "ApplicationCatalogTemplatesResponse", "ApplicationCatalogType", "ApplicationCatalogTypesResponse", "ApplicationCreate", "ApplicationEdit", "ApplicationFlags", "ApplicationFork", "ApplicationQuery", "ApplicationResponse", "ApplicationRevision", "ApplicationRevisionCommit", "ApplicationRevisionCreate", "ApplicationRevisionDataInput", "ApplicationRevisionDataInputHeadersValue", "ApplicationRevisionDataInputRuntime", "ApplicationRevisionDataOutput", "ApplicationRevisionDataOutputHeadersValue", "ApplicationRevisionDataOutputRuntime", "ApplicationRevisionEdit", "ApplicationRevisionFlags", "ApplicationRevisionFork", "ApplicationRevisionQuery", "ApplicationRevisionQueryFlags", "ApplicationRevisionResolveResponse", "ApplicationRevisionResponse", "ApplicationRevisionsLog", "ApplicationRevisionsResponse", "ApplicationVariant", "ApplicationVariantCreate", "ApplicationVariantEdit", "ApplicationVariantFlags", "ApplicationVariantFork", "ApplicationVariantResponse", "ApplicationVariantsResponse", "ApplicationsResponse", "BodyConfigsFetchVariantsConfigsFetchPost", "Bucket", "CollectStatusResponse", "ComparisonOperator", "Condition", "ConditionOperator", "ConditionOptions", "ConditionValue", "ConfigResponseModel", "CustomModelSettingsDto", "CustomProviderDto", "CustomProviderKind", "CustomProviderSettingsDto", "DictOperator", "DiscoverResponse", "DiscoverResponseMethodsValue", "EeSrcModelsApiOrganizationModelsOrganization", "EntityRef", "Environment", "EnvironmentCreate", "EnvironmentEdit", "EnvironmentFlags", "EnvironmentQueryFlags", "EnvironmentResponse", "EnvironmentRevision", "EnvironmentRevisionCommit", "EnvironmentRevisionCreate", "EnvironmentRevisionData", "EnvironmentRevisionDelta", "EnvironmentRevisionEdit", "EnvironmentRevisionResolveResponse", "EnvironmentRevisionResponse", "EnvironmentRevisionsLog", "EnvironmentRevisionsResponse", "EnvironmentVariant", "EnvironmentVariantCreate", "EnvironmentVariantEdit", "EnvironmentVariantResponse", "EnvironmentVariantsResponse", "EnvironmentsResponse", "ErrorPolicy", "EvaluationMetrics", "EvaluationMetricsCreate", "EvaluationMetricsEdit", "EvaluationMetricsIdsResponse", "EvaluationMetricsQuery", "EvaluationMetricsQueryScenarioIds", "EvaluationMetricsQueryTimestamps", "EvaluationMetricsRefresh", "EvaluationMetricsResponse", "EvaluationQueue", "EvaluationQueueCreate", "EvaluationQueueData", "EvaluationQueueEdit", "EvaluationQueueFlags", "EvaluationQueueIdResponse", "EvaluationQueueIdsResponse", "EvaluationQueueQuery", "EvaluationQueueQueryFlags", "EvaluationQueueResponse", "EvaluationQueueScenariosQuery", "EvaluationQueuesResponse", "EvaluationResult", "EvaluationResultCreate", "EvaluationResultEdit", "EvaluationResultIdResponse", "EvaluationResultIdsResponse", "EvaluationResultQuery", "EvaluationResultResponse", "EvaluationResultsResponse", "EvaluationRun", "EvaluationRunCreate", "EvaluationRunData", "EvaluationRunDataMapping", "EvaluationRunDataMappingColumn", "EvaluationRunDataMappingStep", "EvaluationRunDataStep", "EvaluationRunDataStepInput", "EvaluationRunDataStepOrigin", "EvaluationRunDataStepType", "EvaluationRunEdit", "EvaluationRunFlags", "EvaluationRunIdResponse", "EvaluationRunIdsRequest", "EvaluationRunIdsResponse", "EvaluationRunQuery", "EvaluationRunQueryFlags", "EvaluationRunResponse", "EvaluationRunsResponse", "EvaluationScenario", "EvaluationScenarioCreate", "EvaluationScenarioEdit", "EvaluationScenarioIdResponse", "EvaluationScenarioIdsResponse", "EvaluationScenarioQuery", "EvaluationScenarioResponse", "EvaluationScenariosResponse", "EvaluationStatus", "Evaluator", "EvaluatorArtifactFlags", "EvaluatorArtifactQueryFlags", "EvaluatorCatalogPreset", "EvaluatorCatalogPresetResponse", "EvaluatorCatalogPresetsResponse", "EvaluatorCatalogTemplate", "EvaluatorCatalogTemplateResponse", "EvaluatorCatalogTemplatesResponse", "EvaluatorCatalogType", "EvaluatorCatalogTypesResponse", "EvaluatorCreate", "EvaluatorEdit", "EvaluatorFlags", "EvaluatorFork", "EvaluatorQuery", "EvaluatorResponse", "EvaluatorRevision", "EvaluatorRevisionCommit", "EvaluatorRevisionCreate", "EvaluatorRevisionDataInput", "EvaluatorRevisionDataInputHeadersValue", "EvaluatorRevisionDataInputRuntime", "EvaluatorRevisionDataOutput", "EvaluatorRevisionDataOutputHeadersValue", "EvaluatorRevisionDataOutputRuntime", "EvaluatorRevisionEdit", "EvaluatorRevisionFlags", "EvaluatorRevisionFork", "EvaluatorRevisionQuery", "EvaluatorRevisionQueryFlags", "EvaluatorRevisionResolveResponse", "EvaluatorRevisionResponse", "EvaluatorRevisionsLog", "EvaluatorRevisionsResponse", "EvaluatorTemplate", "EvaluatorTemplatesResponse", "EvaluatorVariant", "EvaluatorVariantCreate", "EvaluatorVariantEdit", "EvaluatorVariantFlags", "EvaluatorVariantFork", "EvaluatorVariantResponse", "EvaluatorVariantsResponse", "EvaluatorsResponse", "Event", "EventQuery", "EventType", "EventsQueryResponse", "ExistenceOperator", "FilteringInput", "FilteringInputConditionsItem", "FilteringOutput", "FilteringOutputConditionsItem", "Focus", "Folder", "FolderCreate", "FolderEdit", "FolderIdResponse", "FolderKind", "FolderQuery", "FolderQueryKinds", "FolderResponse", "FoldersResponse", "Format", "Formatting", typing.Any, typing.Any, "Header", "HttpValidationError", "InviteRequest", "Invocation", "InvocationCreate", "InvocationCreateLinks", "InvocationEdit", "InvocationEditLinks", "InvocationLinkResponse", "InvocationLinks", "InvocationQuery", "InvocationQueryLinks", "InvocationResponse", "InvocationsResponse", "JsonSchemasInput", "JsonSchemasOutput", typing.Any, typing.Any, "LegacyLifecycleDto", "ListApiKeysResponse", "ListOperator", "ListOptions", "LogicalOperator", "MetricSpec", "MetricType", "MetricsBucket", "NumericOperator", "OTelEventInput", "OTelEventInputTimestamp", "OTelEventOutput", "OTelEventOutputTimestamp", "OTelHashInput", "OTelHashOutput", "OTelLinkInput", "OTelLinkOutput", "OTelLinksResponse", "OTelReferenceInput", "OTelReferenceOutput", "OTelSpanKind", "OTelStatusCode", "OTelTracingRequest", "OTelTracingResponse", "OldAnalyticsResponse", "OrganizationDetails", "OrganizationDomainResponse", "OrganizationProviderResponse", "OrganizationUpdate", "OssSrcModelsApiOrganizationModelsOrganization", "Permission", "ProjectsResponse", "QueriesResponse", "Query", "QueryCreate", "QueryEdit", "QueryFlags", "QueryQueryFlags", "QueryResponse", "QueryRevision", "QueryRevisionCommit", "QueryRevisionCreate", "QueryRevisionDataInput", "QueryRevisionDataOutput", "QueryRevisionEdit", "QueryRevisionQuery", "QueryRevisionResponse", "QueryRevisionsLog", "QueryRevisionsResponse", "QueryVariant", "QueryVariantCreate", "QueryVariantEdit", "QueryVariantQuery", "QueryVariantResponse", "QueryVariantsResponse", "Reference", "ReferenceRequestModelInput", "ReferenceRequestModelOutput", "RequestType", "ResolutionInfo", "RetrievalInfo", "RevisionFork", "SecretDto", "SecretDtoData", "SecretKind", "SecretResponseDto", "SecretResponseDtoData", "SessionIdsResponse", "SimpleApplication", "SimpleApplicationCreate", "SimpleApplicationDataInput", "SimpleApplicationDataInputHeadersValue", "SimpleApplicationDataInputRuntime", "SimpleApplicationDataOutput", "SimpleApplicationDataOutputHeadersValue", "SimpleApplicationDataOutputRuntime", "SimpleApplicationEdit", "SimpleApplicationFlags", "SimpleApplicationQuery", "SimpleApplicationQueryFlags", "SimpleApplicationResponse", "SimpleApplicationsResponse", "SimpleEnvironment", "SimpleEnvironmentCreate", "SimpleEnvironmentEdit", "SimpleEnvironmentQuery", "SimpleEnvironmentResponse", "SimpleEnvironmentsResponse", "SimpleEvaluation", "SimpleEvaluationCreate", "SimpleEvaluationData", "SimpleEvaluationDataApplicationSteps", "SimpleEvaluationDataApplicationStepsOneValue", "SimpleEvaluationDataEvaluatorSteps", "SimpleEvaluationDataEvaluatorStepsOneValue", "SimpleEvaluationDataQuerySteps", "SimpleEvaluationDataQueryStepsOneValue", "SimpleEvaluationDataTestsetSteps", "SimpleEvaluationDataTestsetStepsOneValue", "SimpleEvaluationEdit", "SimpleEvaluationIdResponse", "SimpleEvaluationQuery", "SimpleEvaluationResponse", "SimpleEvaluationsResponse", "SimpleEvaluator", "SimpleEvaluatorCreate", "SimpleEvaluatorDataInput", "SimpleEvaluatorDataInputHeadersValue", "SimpleEvaluatorDataInputRuntime", "SimpleEvaluatorDataOutput", "SimpleEvaluatorDataOutputHeadersValue", "SimpleEvaluatorDataOutputRuntime", "SimpleEvaluatorEdit", "SimpleEvaluatorFlags", "SimpleEvaluatorQuery", "SimpleEvaluatorQueryFlags", "SimpleEvaluatorResponse", "SimpleEvaluatorsResponse", "SimpleQueriesResponse", "SimpleQuery", "SimpleQueryCreate", "SimpleQueryEdit", "SimpleQueryQuery", "SimpleQueryResponse", "SimpleQueue", "SimpleQueueCreate", "SimpleQueueData", "SimpleQueueDataEvaluators", "SimpleQueueDataEvaluatorsOneValue", "SimpleQueueIdResponse", "SimpleQueueKind", "SimpleQueueQuery", "SimpleQueueResponse", "SimpleQueueScenariosQuery", "SimpleQueueScenariosResponse", "SimpleQueueSettings", "SimpleQueuesResponse", "SimpleTestset", "SimpleTestsetCreate", "SimpleTestsetEdit", "SimpleTestsetQuery", "SimpleTestsetResponse", "SimpleTestsetsResponse", "SimpleTrace", "SimpleTraceChannel", "SimpleTraceCreate", "SimpleTraceCreateLinks", "SimpleTraceEdit", "SimpleTraceEditLinks", "SimpleTraceKind", "SimpleTraceLinkResponse", "SimpleTraceLinks", "SimpleTraceOrigin", "SimpleTraceQuery", "SimpleTraceQueryLinks", "SimpleTraceReferences", "SimpleTraceResponse", "SimpleTracesResponse", "SimpleWorkflow", "SimpleWorkflowCreate", "SimpleWorkflowDataInput", "SimpleWorkflowDataInputHeadersValue", "SimpleWorkflowDataInputRuntime", "SimpleWorkflowDataOutput", "SimpleWorkflowDataOutputHeadersValue", "SimpleWorkflowDataOutputRuntime", "SimpleWorkflowEdit", "SimpleWorkflowFlags", "SimpleWorkflowQuery", "SimpleWorkflowQueryFlags", "SimpleWorkflowResponse", "SimpleWorkflowsResponse", "SpanInput", "SpanInputEndTime", "SpanInputStartTime", "SpanOutput", "SpanOutputEndTime", "SpanOutputStartTime", "SpanResponse", "SpanType", "SpansNodeInput", "SpansNodeInputEndTime", "SpansNodeInputSpansValue", "SpansNodeInputStartTime", "SpansNodeOutput", "SpansNodeOutputEndTime", "SpansNodeOutputSpansValue", "SpansNodeOutputStartTime", "SpansResponse", "SpansTreeInput", "SpansTreeInputSpansValue", "SpansTreeOutput", "SpansTreeOutputSpansValue", "SsoProviderDto", "SsoProviderInfo", "SsoProviderSettingsDto", "SsoProviders", "StandardProviderDto", "StandardProviderKind", "StandardProviderSettingsDto", "Status", "StringOperator", "TestcaseInput", "TestcaseOutput", "TestcaseResponse", "TestcasesResponse", "Testset", "TestsetCreate", "TestsetEdit", "TestsetFlags", "TestsetQuery", "TestsetResponse", "TestsetRevision", "TestsetRevisionCommit", "TestsetRevisionCreate", "TestsetRevisionDataInput", "TestsetRevisionDataOutput", "TestsetRevisionDelta", "TestsetRevisionDeltaColumns", "TestsetRevisionDeltaRows", "TestsetRevisionEdit", "TestsetRevisionQuery", "TestsetRevisionResponse", "TestsetRevisionsLog", "TestsetRevisionsResponse", "TestsetVariant", "TestsetVariantCreate", "TestsetVariantEdit", "TestsetVariantQuery", "TestsetVariantResponse", "TestsetVariantsResponse", "TestsetsResponse", "TextOptions", "ToolAuthScheme", "ToolCallData", "ToolCallFunction", "ToolCallResponse", "ToolCatalogAction", "ToolCatalogActionDetails", "ToolCatalogActionResponse", "ToolCatalogActionResponseAction", "ToolCatalogActionsResponse", "ToolCatalogActionsResponseActionsItem", "ToolCatalogIntegration", "ToolCatalogIntegrationDetails", "ToolCatalogIntegrationResponse", "ToolCatalogIntegrationResponseIntegration", "ToolCatalogIntegrationsResponse", "ToolCatalogIntegrationsResponseIntegrationsItem", "ToolCatalogProvider", "ToolCatalogProviderDetails", "ToolCatalogProviderResponse", "ToolCatalogProviderResponseProvider", "ToolCatalogProvidersResponse", "ToolCatalogProvidersResponseProvidersItem", "ToolConnection", "ToolConnectionCreate", "ToolConnectionCreateData", "ToolConnectionResponse", "ToolConnectionStatus", "ToolConnectionsResponse", "ToolProviderKind", "ToolResult", "ToolResultData", "TraceIdResponse", "TraceIdsResponse", "TraceInput", "TraceInputSpansValue", "TraceOutput", "TraceOutputSpansValue", "TraceRequest", "TraceResponse", "TraceType", "TracesRequest", "TracesResponse", "TracingQuery", "UserIdsResponse", "ValidationError", "ValidationErrorLocItem", "VariantFork", "WebhookDeliveriesResponse", "WebhookDelivery", "WebhookDeliveryCreate", "WebhookDeliveryData", "WebhookDeliveryQuery", "WebhookDeliveryResponse", "WebhookDeliveryResponseInfo", "WebhookEventType", "WebhookProviderDto", "WebhookProviderSettingsDto", "WebhookSubscription", "WebhookSubscriptionCreate", "WebhookSubscriptionData", "WebhookSubscriptionDataAuthMode", "WebhookSubscriptionEdit", "WebhookSubscriptionQuery", "WebhookSubscriptionResponse", "WebhookSubscriptionsResponse", "Windowing", "WindowingOrder", "Workflow", "WorkflowArtifactFlags", "WorkflowCatalogFlags", "WorkflowCatalogPreset", "WorkflowCatalogPresetResponse", "WorkflowCatalogPresetsResponse", "WorkflowCatalogTemplate", "WorkflowCatalogTemplateResponse", "WorkflowCatalogTemplatesResponse", "WorkflowCatalogType", "WorkflowCatalogTypeResponse", "WorkflowCatalogTypesResponse", "WorkflowCreate", "WorkflowEdit", "WorkflowFlags", "WorkflowFork", "WorkflowResponse", "WorkflowRevisionCommit", "WorkflowRevisionCreate", "WorkflowRevisionDataInput", "WorkflowRevisionDataInputHeadersValue", "WorkflowRevisionDataInputRuntime", "WorkflowRevisionDataOutput", "WorkflowRevisionDataOutputHeadersValue", "WorkflowRevisionDataOutputRuntime", "WorkflowRevisionEdit", "WorkflowRevisionFlags", "WorkflowRevisionFork", "WorkflowRevisionInput", "WorkflowRevisionOutput", "WorkflowRevisionResolveResponse", "WorkflowRevisionResponse", "WorkflowRevisionsLog", "WorkflowRevisionsResponse", "WorkflowVariant", "WorkflowVariantCreate", "WorkflowVariantEdit", "WorkflowVariantFlags", "WorkflowVariantFork", "WorkflowVariantResponse", "WorkflowVariantsResponse", "WorkflowsResponse", "Workspace", "WorkspaceMemberResponse", "WorkspacePermission", "WorkspaceResponse"] +__all__ = ["AdminAccountCreateOptions", "AdminAccountRead", "AdminAccountsCreate", "AdminAccountsDelete", "AdminAccountsDeleteTarget", "AdminAccountsResponse", "AdminApiKeyCreate", "AdminApiKeyResponse", "AdminDeleteResponse", "AdminDeletedEntities", "AdminDeletedEntity", "AdminOrganizationCreate", "AdminOrganizationMembershipCreate", "AdminOrganizationMembershipRead", "AdminOrganizationRead", "AdminProjectCreate", "AdminProjectMembershipCreate", "AdminProjectMembershipRead", "AdminProjectRead", "AdminSimpleAccountCreate", "AdminSimpleAccountDeleteEntry", "AdminSimpleAccountRead", "AdminSimpleAccountsApiKeysCreate", "AdminSimpleAccountsCreate", "AdminSimpleAccountsDelete", "AdminSimpleAccountsOrganizationsCreate", "AdminSimpleAccountsOrganizationsMembershipsCreate", "AdminSimpleAccountsOrganizationsTransferOwnership", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjects", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeProjectsZero", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspaces", "AdminSimpleAccountsOrganizationsTransferOwnershipIncludeWorkspacesZero", "AdminSimpleAccountsOrganizationsTransferOwnershipResponse", "AdminSimpleAccountsProjectsCreate", "AdminSimpleAccountsProjectsMembershipsCreate", "AdminSimpleAccountsResponse", "AdminSimpleAccountsUsersCreate", "AdminSimpleAccountsUsersIdentitiesCreate", "AdminSimpleAccountsUsersResetPassword", "AdminSimpleAccountsWorkspacesCreate", "AdminSimpleAccountsWorkspacesMembershipsCreate", "AdminStructuredError", "AdminSubscriptionCreate", "AdminSubscriptionRead", "AdminUserCreate", "AdminUserIdentityCreate", "AdminUserIdentityRead", "AdminUserIdentityReadStatus", "AdminUserRead", "AdminWorkspaceCreate", "AdminWorkspaceMembershipCreate", "AdminWorkspaceMembershipRead", "AdminWorkspaceRead", "Analytics", "AnalyticsResponse", "Annotation", "AnnotationCreate", "AnnotationCreateLinks", "AnnotationEdit", "AnnotationEditLinks", "AnnotationLinkResponse", "AnnotationLinks", "AnnotationQuery", "AnnotationQueryLinks", "AnnotationResponse", "AnnotationsResponse", "Application", "ApplicationArtifactFlags", "ApplicationArtifactQueryFlags", "ApplicationCatalogPreset", "ApplicationCatalogPresetResponse", "ApplicationCatalogPresetsResponse", "ApplicationCatalogTemplate", "ApplicationCatalogTemplateResponse", "ApplicationCatalogTemplatesResponse", "ApplicationCatalogType", "ApplicationCatalogTypesResponse", "ApplicationCreate", "ApplicationEdit", "ApplicationFlags", "ApplicationFork", "ApplicationQuery", "ApplicationResponse", "ApplicationRevision", "ApplicationRevisionCommit", "ApplicationRevisionCreate", "ApplicationRevisionDataInput", "ApplicationRevisionDataInputHeadersValue", "ApplicationRevisionDataInputRuntime", "ApplicationRevisionDataOutput", "ApplicationRevisionDataOutputHeadersValue", "ApplicationRevisionDataOutputRuntime", "ApplicationRevisionEdit", "ApplicationRevisionFlags", "ApplicationRevisionFork", "ApplicationRevisionQuery", "ApplicationRevisionQueryFlags", "ApplicationRevisionResolveResponse", "ApplicationRevisionResponse", "ApplicationRevisionsLog", "ApplicationRevisionsResponse", "ApplicationVariant", "ApplicationVariantCreate", "ApplicationVariantEdit", "ApplicationVariantFlags", "ApplicationVariantFork", "ApplicationVariantResponse", "ApplicationVariantsResponse", "ApplicationsResponse", "BodyConfigsFetchVariantsConfigsFetchPost", "Bucket", "CollectStatusResponse", "ComparisonOperator", "Condition", "ConditionOperator", "ConditionOptions", "ConditionValue", "ConfigResponseModel", "CustomModelSettingsDto", "CustomProviderDto", "CustomProviderKind", "CustomProviderSettingsDto", "DictOperator", "DiscoverResponse", "DiscoverResponseMethodsValue", "EeSrcModelsApiOrganizationModelsOrganization", "EntityRef", "Environment", "EnvironmentCreate", "EnvironmentEdit", "EnvironmentFlags", "EnvironmentQueryFlags", "EnvironmentResponse", "EnvironmentRevision", "EnvironmentRevisionCommit", "EnvironmentRevisionCreate", "EnvironmentRevisionData", "EnvironmentRevisionDelta", "EnvironmentRevisionEdit", "EnvironmentRevisionResolveResponse", "EnvironmentRevisionResponse", "EnvironmentRevisionsLog", "EnvironmentRevisionsResponse", "EnvironmentVariant", "EnvironmentVariantCreate", "EnvironmentVariantEdit", "EnvironmentVariantResponse", "EnvironmentVariantsResponse", "EnvironmentsResponse", "ErrorPolicy", "EvaluationMetrics", "EvaluationMetricsCreate", "EvaluationMetricsIdsResponse", "EvaluationMetricsQuery", "EvaluationMetricsQueryScenarioIds", "EvaluationMetricsQueryTimestamps", "EvaluationMetricsRefresh", "EvaluationMetricsResponse", "EvaluationQueue", "EvaluationQueueCreate", "EvaluationQueueData", "EvaluationQueueEdit", "EvaluationQueueFlags", "EvaluationQueueIdResponse", "EvaluationQueueIdsResponse", "EvaluationQueueQuery", "EvaluationQueueQueryFlags", "EvaluationQueueResponse", "EvaluationQueueScenariosQuery", "EvaluationQueuesResponse", "EvaluationResult", "EvaluationResultCreate", "EvaluationResultIdResponse", "EvaluationResultIdsResponse", "EvaluationResultQuery", "EvaluationResultResponse", "EvaluationResultsResponse", "EvaluationRun", "EvaluationRunCreate", "EvaluationRunData", "EvaluationRunDataConcurrency", "EvaluationRunDataMapping", "EvaluationRunDataMappingColumn", "EvaluationRunDataMappingStep", "EvaluationRunDataStep", "EvaluationRunDataStepInput", "EvaluationRunDataStepOrigin", "EvaluationRunDataStepType", "EvaluationRunEdit", "EvaluationRunFlags", "EvaluationRunIdResponse", "EvaluationRunIdsRequest", "EvaluationRunIdsResponse", "EvaluationRunQuery", "EvaluationRunQueryFlags", "EvaluationRunResponse", "EvaluationRunsResponse", "EvaluationScenario", "EvaluationScenarioCreate", "EvaluationScenarioEdit", "EvaluationScenarioIdResponse", "EvaluationScenarioIdsResponse", "EvaluationScenarioQuery", "EvaluationScenarioResponse", "EvaluationScenariosResponse", "EvaluationStatus", "Evaluator", "EvaluatorArtifactFlags", "EvaluatorArtifactQueryFlags", "EvaluatorCatalogPreset", "EvaluatorCatalogPresetResponse", "EvaluatorCatalogPresetsResponse", "EvaluatorCatalogTemplate", "EvaluatorCatalogTemplateResponse", "EvaluatorCatalogTemplatesResponse", "EvaluatorCatalogType", "EvaluatorCatalogTypesResponse", "EvaluatorCreate", "EvaluatorEdit", "EvaluatorFlags", "EvaluatorFork", "EvaluatorQuery", "EvaluatorResponse", "EvaluatorRevision", "EvaluatorRevisionCommit", "EvaluatorRevisionCreate", "EvaluatorRevisionDataInput", "EvaluatorRevisionDataInputHeadersValue", "EvaluatorRevisionDataInputRuntime", "EvaluatorRevisionDataOutput", "EvaluatorRevisionDataOutputHeadersValue", "EvaluatorRevisionDataOutputRuntime", "EvaluatorRevisionEdit", "EvaluatorRevisionFlags", "EvaluatorRevisionFork", "EvaluatorRevisionQuery", "EvaluatorRevisionQueryFlags", "EvaluatorRevisionResolveResponse", "EvaluatorRevisionResponse", "EvaluatorRevisionsLog", "EvaluatorRevisionsResponse", "EvaluatorTemplate", "EvaluatorTemplatesResponse", "EvaluatorVariant", "EvaluatorVariantCreate", "EvaluatorVariantEdit", "EvaluatorVariantFlags", "EvaluatorVariantFork", "EvaluatorVariantResponse", "EvaluatorVariantsResponse", "EvaluatorsResponse", "Event", "EventQuery", "EventType", "EventsQueryResponse", "ExistenceOperator", "FilteringInput", "FilteringInputConditionsItem", "FilteringOutput", "FilteringOutputConditionsItem", "Focus", "Folder", "FolderCreate", "FolderEdit", "FolderIdResponse", "FolderKind", "FolderQuery", "FolderQueryKinds", "FolderResponse", "FoldersResponse", "Format", "Formatting", typing.Any, typing.Any, "Header", "HttpValidationError", "InviteRequest", "Invocation", "InvocationCreate", "InvocationCreateLinks", "InvocationEdit", "InvocationEditLinks", "InvocationLinkResponse", "InvocationLinks", "InvocationQuery", "InvocationQueryLinks", "InvocationResponse", "InvocationsResponse", "JsonSchemasInput", "JsonSchemasOutput", typing.Any, typing.Any, "LegacyLifecycleDto", "ListApiKeysResponse", "ListOperator", "ListOptions", "LogicalOperator", "MetricSpec", "MetricType", "MetricsBucket", "NumericOperator", "OTelEventInput", "OTelEventInputTimestamp", "OTelEventOutput", "OTelEventOutputTimestamp", "OTelHashInput", "OTelHashOutput", "OTelLinkInput", "OTelLinkOutput", "OTelLinksResponse", "OTelReferenceInput", "OTelReferenceOutput", "OTelSpanKind", "OTelStatusCode", "OTelTracingRequest", "OTelTracingResponse", "OldAnalyticsResponse", "OrganizationDetails", "OrganizationDomainResponse", "OrganizationProviderResponse", "OrganizationUpdate", "OssSrcModelsApiOrganizationModelsOrganization", "Permission", "ProjectsResponse", "QueriesResponse", "Query", "QueryCreate", "QueryEdit", "QueryFlags", "QueryQueryFlags", "QueryResponse", "QueryRevision", "QueryRevisionCommit", "QueryRevisionCreate", "QueryRevisionDataInput", "QueryRevisionDataOutput", "QueryRevisionEdit", "QueryRevisionQuery", "QueryRevisionResponse", "QueryRevisionsLog", "QueryRevisionsResponse", "QueryVariant", "QueryVariantCreate", "QueryVariantEdit", "QueryVariantQuery", "QueryVariantResponse", "QueryVariantsResponse", "Reference", "ReferenceRequestModelInput", "ReferenceRequestModelOutput", "RequestType", "ResolutionInfo", "RetrievalInfo", "RevisionFork", "SecretDto", "SecretDtoData", "SecretKind", "SecretResponseDto", "SecretResponseDtoData", "SessionIdsResponse", "SimpleApplication", "SimpleApplicationCreate", "SimpleApplicationDataInput", "SimpleApplicationDataInputHeadersValue", "SimpleApplicationDataInputRuntime", "SimpleApplicationDataOutput", "SimpleApplicationDataOutputHeadersValue", "SimpleApplicationDataOutputRuntime", "SimpleApplicationEdit", "SimpleApplicationFlags", "SimpleApplicationQuery", "SimpleApplicationQueryFlags", "SimpleApplicationResponse", "SimpleApplicationsResponse", "SimpleEnvironment", "SimpleEnvironmentCreate", "SimpleEnvironmentEdit", "SimpleEnvironmentQuery", "SimpleEnvironmentResponse", "SimpleEnvironmentsResponse", "SimpleEvaluation", "SimpleEvaluationCreate", "SimpleEvaluationData", "SimpleEvaluationDataApplicationSteps", "SimpleEvaluationDataApplicationStepsOneValue", "SimpleEvaluationDataEvaluatorSteps", "SimpleEvaluationDataEvaluatorStepsOneValue", "SimpleEvaluationDataQuerySteps", "SimpleEvaluationDataQueryStepsOneValue", "SimpleEvaluationDataTestsetSteps", "SimpleEvaluationDataTestsetStepsOneValue", "SimpleEvaluationEdit", "SimpleEvaluationIdResponse", "SimpleEvaluationQuery", "SimpleEvaluationResponse", "SimpleEvaluationsResponse", "SimpleEvaluator", "SimpleEvaluatorCreate", "SimpleEvaluatorDataInput", "SimpleEvaluatorDataInputHeadersValue", "SimpleEvaluatorDataInputRuntime", "SimpleEvaluatorDataOutput", "SimpleEvaluatorDataOutputHeadersValue", "SimpleEvaluatorDataOutputRuntime", "SimpleEvaluatorEdit", "SimpleEvaluatorFlags", "SimpleEvaluatorQuery", "SimpleEvaluatorQueryFlags", "SimpleEvaluatorResponse", "SimpleEvaluatorsResponse", "SimpleQueriesResponse", "SimpleQuery", "SimpleQueryCreate", "SimpleQueryEdit", "SimpleQueryQuery", "SimpleQueryResponse", "SimpleQueue", "SimpleQueueCreate", "SimpleQueueData", "SimpleQueueDataEvaluators", "SimpleQueueDataEvaluatorsOneValue", "SimpleQueueIdResponse", "SimpleQueueKind", "SimpleQueueQuery", "SimpleQueueResponse", "SimpleQueueScenariosQuery", "SimpleQueueScenariosResponse", "SimpleQueueSettings", "SimpleQueuesResponse", "SimpleTestset", "SimpleTestsetCreate", "SimpleTestsetEdit", "SimpleTestsetQuery", "SimpleTestsetResponse", "SimpleTestsetsResponse", "SimpleTrace", "SimpleTraceChannel", "SimpleTraceCreate", "SimpleTraceCreateLinks", "SimpleTraceEdit", "SimpleTraceEditLinks", "SimpleTraceKind", "SimpleTraceLinkResponse", "SimpleTraceLinks", "SimpleTraceOrigin", "SimpleTraceQuery", "SimpleTraceQueryLinks", "SimpleTraceReferences", "SimpleTraceResponse", "SimpleTracesResponse", "SimpleWorkflow", "SimpleWorkflowCreate", "SimpleWorkflowDataInput", "SimpleWorkflowDataInputHeadersValue", "SimpleWorkflowDataInputRuntime", "SimpleWorkflowDataOutput", "SimpleWorkflowDataOutputHeadersValue", "SimpleWorkflowDataOutputRuntime", "SimpleWorkflowEdit", "SimpleWorkflowFlags", "SimpleWorkflowQuery", "SimpleWorkflowQueryFlags", "SimpleWorkflowResponse", "SimpleWorkflowsResponse", "SpanInput", "SpanInputEndTime", "SpanInputStartTime", "SpanOutput", "SpanOutputEndTime", "SpanOutputStartTime", "SpanResponse", "SpanType", "SpansNodeInput", "SpansNodeInputEndTime", "SpansNodeInputSpansValue", "SpansNodeInputStartTime", "SpansNodeOutput", "SpansNodeOutputEndTime", "SpansNodeOutputSpansValue", "SpansNodeOutputStartTime", "SpansResponse", "SpansTreeInput", "SpansTreeInputSpansValue", "SpansTreeOutput", "SpansTreeOutputSpansValue", "SsoProviderDto", "SsoProviderInfo", "SsoProviderSettingsDto", "SsoProviders", "StandardProviderDto", "StandardProviderKind", "StandardProviderSettingsDto", "Status", "StringOperator", "TestcaseInput", "TestcaseOutput", "TestcaseResponse", "TestcasesResponse", "Testset", "TestsetCreate", "TestsetEdit", "TestsetFlags", "TestsetQuery", "TestsetResponse", "TestsetRevision", "TestsetRevisionCommit", "TestsetRevisionCreate", "TestsetRevisionDataInput", "TestsetRevisionDataOutput", "TestsetRevisionDelta", "TestsetRevisionDeltaColumns", "TestsetRevisionDeltaRows", "TestsetRevisionEdit", "TestsetRevisionQuery", "TestsetRevisionResponse", "TestsetRevisionsLog", "TestsetRevisionsResponse", "TestsetVariant", "TestsetVariantCreate", "TestsetVariantEdit", "TestsetVariantQuery", "TestsetVariantResponse", "TestsetVariantsResponse", "TestsetsResponse", "TextOptions", "ToolAuthScheme", "ToolCallData", "ToolCallFunction", "ToolCallResponse", "ToolCatalogAction", "ToolCatalogActionDetails", "ToolCatalogActionResponse", "ToolCatalogActionResponseAction", "ToolCatalogActionsResponse", "ToolCatalogActionsResponseActionsItem", "ToolCatalogIntegration", "ToolCatalogIntegrationDetails", "ToolCatalogIntegrationResponse", "ToolCatalogIntegrationResponseIntegration", "ToolCatalogIntegrationsResponse", "ToolCatalogIntegrationsResponseIntegrationsItem", "ToolCatalogProvider", "ToolCatalogProviderDetails", "ToolCatalogProviderResponse", "ToolCatalogProviderResponseProvider", "ToolCatalogProvidersResponse", "ToolCatalogProvidersResponseProvidersItem", "ToolConnection", "ToolConnectionCreate", "ToolConnectionCreateData", "ToolConnectionResponse", "ToolConnectionStatus", "ToolConnectionsResponse", "ToolProviderKind", "ToolResult", "ToolResultData", "TraceIdResponse", "TraceIdsResponse", "TraceInput", "TraceInputSpansValue", "TraceOutput", "TraceOutputSpansValue", "TraceRequest", "TraceResponse", "TraceType", "TracesRequest", "TracesResponse", "TracingQuery", "UserIdsResponse", "ValidationError", "ValidationErrorLocItem", "VariantFork", "WebhookDeliveriesResponse", "WebhookDelivery", "WebhookDeliveryCreate", "WebhookDeliveryData", "WebhookDeliveryQuery", "WebhookDeliveryResponse", "WebhookDeliveryResponseInfo", "WebhookEventType", "WebhookProviderDto", "WebhookProviderSettingsDto", "WebhookSubscription", "WebhookSubscriptionCreate", "WebhookSubscriptionData", "WebhookSubscriptionDataAuthMode", "WebhookSubscriptionEdit", "WebhookSubscriptionQuery", "WebhookSubscriptionResponse", "WebhookSubscriptionsResponse", "Windowing", "WindowingOrder", "Workflow", "WorkflowArtifactFlags", "WorkflowCatalogFlags", "WorkflowCatalogPreset", "WorkflowCatalogPresetResponse", "WorkflowCatalogPresetsResponse", "WorkflowCatalogTemplate", "WorkflowCatalogTemplateResponse", "WorkflowCatalogTemplatesResponse", "WorkflowCatalogType", "WorkflowCatalogTypeResponse", "WorkflowCatalogTypesResponse", "WorkflowCreate", "WorkflowEdit", "WorkflowFlags", "WorkflowFork", "WorkflowResponse", "WorkflowRevisionCommit", "WorkflowRevisionCreate", "WorkflowRevisionDataInput", "WorkflowRevisionDataInputHeadersValue", "WorkflowRevisionDataInputRuntime", "WorkflowRevisionDataOutput", "WorkflowRevisionDataOutputHeadersValue", "WorkflowRevisionDataOutputRuntime", "WorkflowRevisionEdit", "WorkflowRevisionFlags", "WorkflowRevisionFork", "WorkflowRevisionInput", "WorkflowRevisionOutput", "WorkflowRevisionResolveResponse", "WorkflowRevisionResponse", "WorkflowRevisionsLog", "WorkflowRevisionsResponse", "WorkflowVariant", "WorkflowVariantCreate", "WorkflowVariantEdit", "WorkflowVariantFlags", "WorkflowVariantFork", "WorkflowVariantResponse", "WorkflowVariantsResponse", "WorkflowsResponse", "Workspace", "WorkspaceMemberResponse", "WorkspacePermission", "WorkspaceResponse"] diff --git a/clients/python/agenta_client/types/evaluation_metrics_edit.py b/clients/python/agenta_client/types/evaluation_metrics_edit.py deleted file mode 100644 index 44de82e0c7..0000000000 --- a/clients/python/agenta_client/types/evaluation_metrics_edit.py +++ /dev/null @@ -1,30 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -from __future__ import annotations - -import typing - -import pydantic -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel, update_forward_refs -from .evaluation_status import EvaluationStatus - - -class EvaluationMetricsEdit(UniversalBaseModel): - flags: typing.Optional[typing.Dict[str, typing.Any]] = None - tags: typing.Optional[typing.Dict[str, typing.Any]] = None - meta: typing.Optional[typing.Dict[str, typing.Any]] = None - id: typing.Optional[str] = None - version: typing.Optional[str] = None - status: typing.Optional[EvaluationStatus] = None - data: typing.Optional[typing.Dict[str, typing.Any]] = None - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow -from .label_json_input import LabelJsonInput # noqa: E402, I001 -from .full_json_input import FullJsonInput # noqa: E402, I001 -update_forward_refs(EvaluationMetricsEdit, FullJsonInput=FullJsonInput, LabelJsonInput=LabelJsonInput) diff --git a/clients/python/agenta_client/types/evaluation_queue_flags.py b/clients/python/agenta_client/types/evaluation_queue_flags.py index 6aca799bc0..d6e5b3f603 100644 --- a/clients/python/agenta_client/types/evaluation_queue_flags.py +++ b/clients/python/agenta_client/types/evaluation_queue_flags.py @@ -8,6 +8,7 @@ class EvaluationQueueFlags(UniversalBaseModel): is_sequential: typing.Optional[bool] = None + is_default: typing.Optional[bool] = None if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 diff --git a/clients/python/agenta_client/types/evaluation_queue_query.py b/clients/python/agenta_client/types/evaluation_queue_query.py index ac5095e78d..5d553524a4 100644 --- a/clients/python/agenta_client/types/evaluation_queue_query.py +++ b/clients/python/agenta_client/types/evaluation_queue_query.py @@ -15,6 +15,7 @@ class EvaluationQueueQuery(UniversalBaseModel): meta: typing.Optional[typing.Dict[str, typing.Any]] = None name: typing.Optional[str] = None description: typing.Optional[str] = None + include_archived: typing.Optional[bool] = None user_id: typing.Optional[str] = None user_ids: typing.Optional[typing.List[str]] = None run_id: typing.Optional[str] = None diff --git a/clients/python/agenta_client/types/evaluation_queue_query_flags.py b/clients/python/agenta_client/types/evaluation_queue_query_flags.py index 21a34f8379..b53163ca48 100644 --- a/clients/python/agenta_client/types/evaluation_queue_query_flags.py +++ b/clients/python/agenta_client/types/evaluation_queue_query_flags.py @@ -8,6 +8,7 @@ class EvaluationQueueQueryFlags(UniversalBaseModel): is_sequential: typing.Optional[bool] = None + is_default: typing.Optional[bool] = None if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 diff --git a/clients/python/agenta_client/types/evaluation_result_edit.py b/clients/python/agenta_client/types/evaluation_result_edit.py deleted file mode 100644 index cc892e7f61..0000000000 --- a/clients/python/agenta_client/types/evaluation_result_edit.py +++ /dev/null @@ -1,33 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -from __future__ import annotations - -import typing - -import pydantic -from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel, update_forward_refs -from .evaluation_status import EvaluationStatus - - -class EvaluationResultEdit(UniversalBaseModel): - flags: typing.Optional[typing.Dict[str, typing.Any]] = None - tags: typing.Optional[typing.Dict[str, typing.Any]] = None - meta: typing.Optional[typing.Dict[str, typing.Any]] = None - id: typing.Optional[str] = None - version: typing.Optional[str] = None - hash_id: typing.Optional[str] = None - trace_id: typing.Optional[str] = None - testcase_id: typing.Optional[str] = None - error: typing.Optional[typing.Dict[str, typing.Any]] = None - status: typing.Optional[EvaluationStatus] = None - - if IS_PYDANTIC_V2: - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 - else: - class Config: - frozen = True - smart_union = True - extra = pydantic.Extra.allow -from .label_json_input import LabelJsonInput # noqa: E402, I001 -from .full_json_input import FullJsonInput # noqa: E402, I001 -update_forward_refs(EvaluationResultEdit, FullJsonInput=FullJsonInput, LabelJsonInput=LabelJsonInput) diff --git a/clients/python/agenta_client/types/evaluation_run_data.py b/clients/python/agenta_client/types/evaluation_run_data.py index c71e05048b..19b1ded27e 100644 --- a/clients/python/agenta_client/types/evaluation_run_data.py +++ b/clients/python/agenta_client/types/evaluation_run_data.py @@ -4,6 +4,7 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from .evaluation_run_data_concurrency import EvaluationRunDataConcurrency from .evaluation_run_data_mapping import EvaluationRunDataMapping from .evaluation_run_data_step import EvaluationRunDataStep @@ -11,6 +12,7 @@ class EvaluationRunData(UniversalBaseModel): steps: typing.Optional[typing.List[EvaluationRunDataStep]] = None repeats: typing.Optional[int] = None + concurrency: typing.Optional[EvaluationRunDataConcurrency] = None mappings: typing.Optional[typing.List[EvaluationRunDataMapping]] = None if IS_PYDANTIC_V2: diff --git a/clients/python/agenta_client/types/evaluation_run_data_concurrency.py b/clients/python/agenta_client/types/evaluation_run_data_concurrency.py new file mode 100644 index 0000000000..25e259e1fc --- /dev/null +++ b/clients/python/agenta_client/types/evaluation_run_data_concurrency.py @@ -0,0 +1,20 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel + + +class EvaluationRunDataConcurrency(UniversalBaseModel): + batch_size: typing.Optional[int] = None + max_retries: typing.Optional[int] = None + retry_delay: typing.Optional[float] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/clients/python/agenta_client/types/evaluation_run_flags.py b/clients/python/agenta_client/types/evaluation_run_flags.py index 86a37a6a9a..9558042685 100644 --- a/clients/python/agenta_client/types/evaluation_run_flags.py +++ b/clients/python/agenta_client/types/evaluation_run_flags.py @@ -15,6 +15,8 @@ class EvaluationRunFlags(UniversalBaseModel): is_split: typing.Optional[bool] = None has_queries: typing.Optional[bool] = None has_testsets: typing.Optional[bool] = None + has_traces: typing.Optional[bool] = None + has_testcases: typing.Optional[bool] = None has_evaluators: typing.Optional[bool] = None has_custom: typing.Optional[bool] = None has_human: typing.Optional[bool] = None diff --git a/clients/python/agenta_client/types/evaluation_run_query_flags.py b/clients/python/agenta_client/types/evaluation_run_query_flags.py index fd632af306..3afa6465b5 100644 --- a/clients/python/agenta_client/types/evaluation_run_query_flags.py +++ b/clients/python/agenta_client/types/evaluation_run_query_flags.py @@ -15,6 +15,8 @@ class EvaluationRunQueryFlags(UniversalBaseModel): is_split: typing.Optional[bool] = None has_queries: typing.Optional[bool] = None has_testsets: typing.Optional[bool] = None + has_traces: typing.Optional[bool] = None + has_testcases: typing.Optional[bool] = None has_evaluators: typing.Optional[bool] = None has_custom: typing.Optional[bool] = None has_human: typing.Optional[bool] = None diff --git a/clients/python/agenta_client/types/simple_evaluation_data.py b/clients/python/agenta_client/types/simple_evaluation_data.py index afcec7aab5..62b5723cc6 100644 --- a/clients/python/agenta_client/types/simple_evaluation_data.py +++ b/clients/python/agenta_client/types/simple_evaluation_data.py @@ -4,6 +4,7 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from .evaluation_run_data_concurrency import EvaluationRunDataConcurrency from .evaluation_status import EvaluationStatus from .simple_evaluation_data_application_steps import SimpleEvaluationDataApplicationSteps from .simple_evaluation_data_evaluator_steps import SimpleEvaluationDataEvaluatorSteps @@ -18,6 +19,7 @@ class SimpleEvaluationData(UniversalBaseModel): application_steps: typing.Optional[SimpleEvaluationDataApplicationSteps] = None evaluator_steps: typing.Optional[SimpleEvaluationDataEvaluatorSteps] = None repeats: typing.Optional[int] = None + concurrency: typing.Optional[EvaluationRunDataConcurrency] = None if IS_PYDANTIC_V2: model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 diff --git a/clients/python/agenta_client/types/simple_queue_kind.py b/clients/python/agenta_client/types/simple_queue_kind.py index 0f4e53725b..373d6407cd 100644 --- a/clients/python/agenta_client/types/simple_queue_kind.py +++ b/clients/python/agenta_client/types/simple_queue_kind.py @@ -2,4 +2,4 @@ import typing -SimpleQueueKind = typing.Union[typing.Literal["traces", "testcases"], typing.Any] +SimpleQueueKind = typing.Union[typing.Literal["queries", "testsets", "traces", "testcases"], typing.Any] diff --git a/clients/python/agenta_client/workspaces/client.py b/clients/python/agenta_client/workspaces/client.py index d95c7f4b2e..1659e6f9db 100644 --- a/clients/python/agenta_client/workspaces/client.py +++ b/clients/python/agenta_client/workspaces/client.py @@ -203,7 +203,7 @@ def get_all_workspace_roles(self, *, request_options: typing.Optional[RequestOpt Returns a list of all available workspace roles. Returns: - List[WorkspaceRoleResponse]: A list of WorkspaceRole objects representing the available workspace roles. + List[WorkspaceRoleResponse]: A list of DefaultRole objects representing the available workspace roles. Raises: HTTPException: If an error occurs while retrieving the workspace roles. @@ -491,7 +491,7 @@ async def get_all_workspace_roles(self, *, request_options: typing.Optional[Requ Returns a list of all available workspace roles. Returns: - List[WorkspaceRoleResponse]: A list of WorkspaceRole objects representing the available workspace roles. + List[WorkspaceRoleResponse]: A list of DefaultRole objects representing the available workspace roles. Raises: HTTPException: If an error occurs while retrieving the workspace roles. diff --git a/clients/python/agenta_client/workspaces/raw_client.py b/clients/python/agenta_client/workspaces/raw_client.py index 2f329cf079..5e5ccf4567 100644 --- a/clients/python/agenta_client/workspaces/raw_client.py +++ b/clients/python/agenta_client/workspaces/raw_client.py @@ -244,7 +244,7 @@ def get_all_workspace_roles(self, *, request_options: typing.Optional[RequestOpt Returns a list of all available workspace roles. Returns: - List[WorkspaceRoleResponse]: A list of WorkspaceRole objects representing the available workspace roles. + List[WorkspaceRoleResponse]: A list of DefaultRole objects representing the available workspace roles. Raises: HTTPException: If an error occurs while retrieving the workspace roles. @@ -556,7 +556,7 @@ async def get_all_workspace_roles(self, *, request_options: typing.Optional[Requ Returns a list of all available workspace roles. Returns: - List[WorkspaceRoleResponse]: A list of WorkspaceRole objects representing the available workspace roles. + List[WorkspaceRoleResponse]: A list of DefaultRole objects representing the available workspace roles. Raises: HTTPException: If an error occurs while retrieving the workspace roles. diff --git a/clients/python/uv.lock b/clients/python/uv.lock index ca0ce1dac1..eb0b72ac57 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -87,11 +87,11 @@ wheels = [ [[package]] name = "idna" -version = "3.17" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/28/99c51f664567218d824af024c0251650fb27e4ca066df188dab0769c5b91/idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f", size = 196048, upload-time = "2026-05-28T14:32:38.55Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/a7/f76514cc40ad6234098ecdebda08732d75964776c51a42845b7da10649e2/idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c", size = 65316, upload-time = "2026-05-28T14:32:37.035Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] diff --git a/docs/design/unified-eval-loops/sdk-vs-api.md b/docs/design/unified-eval-loops/sdk-vs-api.md new file mode 100644 index 0000000000..0b3f979970 --- /dev/null +++ b/docs/design/unified-eval-loops/sdk-vs-api.md @@ -0,0 +1,123 @@ +# SDK vs API: evaluation subsystem comparison + +Snapshot **after** the unification + batching work (Option A: the API re-execute +path now issues one `process_sources` call over all scenarios, matching the SDK +and the design's `process_slice(all scenarios)`). + +The headline: **the engine is one shared code object, not two implementations.** +The API package imports the SDK's `process_sources`, `EvaluationPlanner`, +`runtime.models`, `runtime.topology`, and the adapter protocols directly. The +question is therefore which *seams* each side fills the same, similarly, or +differently — not "do they reimplement each other." + +Key files: +- Engine (shared): `sdks/python/agenta/sdk/evaluations/runtime/{processor,planner,topology,models,executor}.py` +- SDK driver: `sdks/python/agenta/sdk/evaluations/{preview/evaluate.py, runtime/executor.py, runtime/adapters.py}` + clients `{runs,scenarios,results,metrics}.py` +- API driver: `api/oss/src/core/evaluations/{tasks/processor.py, tasks/run.py, runtime/adapters.py, runtime/tensor.py}` + +--- + +## 1. EQUAL — one shared code object + +Imported from the SDK package; there is no API copy. A change here changes both +paths at once. + +| Component | File | +|---|---| +| `process_sources` (the engine) — per-scenario `gather` + semaphore, cell planning/execution, retries, inline `refresh_metrics(scenario_id)` + end-of-slice `refresh_metrics(run_id, None)`, in-loop `edit_scenario` status write | `runtime/processor.py` | +| `EvaluationPlanner.plan` — cell graph (scenario × step × repeat) | `runtime/planner.py` | +| `scenario_status` / `run_status` — verdict ranking (ERRORS > PENDING > SUCCESS; run = ERRORS/RUNNING/SUCCESS) | `runtime/processor.py` | +| `classify_steps_topology` | `runtime/topology.py` | +| `ResolvedSourceItem`, `PlannedCell`, `EvaluationStep`, `TensorSlice`, `WorkflowExecution*`, `ResultLogRequest`, `ProcessSummary` | `runtime/models.py` | +| `DEFAULT_BATCH_SIZE = 10` (default in-slice concurrency) | `runtime/processor.py` | +| Adapter protocols: `ResultSetter.set`, `RefreshMetrics`, `EditScenario`, `CreateScenario`, `TraceLoader`, `WorkflowRunner` | `runtime/executor.py` | +| **Engine call shape: ONE slice over all scenarios** (post-Option-A; was API-divergent) | both drivers | +| **Scenario-factory shape: ordered, lock-guarded cursor** (`_PreMintedScenarios` / `_OrderedScenarios`) | both drivers | + +Consequence: lifecycle logging (`[SLICE]/[SCENARIO]/[STEP]/[METRICS]`), concurrency +model, status computation, and metric-refresh *shape* are identical because they +are the same code. + +--- + +## 2. SIMILAR BUT DIFFERENT — mirrored adapters (same protocol, different backend) + +Each side injects a named adapter class implementing a shared protocol. **SDK +adapters are HTTP clients; API adapters are in-process service calls.** + +| Seam (protocol) | SDK | API | Difference | +|---|---|---|---| +| Workflow runner | `SDKWorkflowRunner` → `invoke_application`/`invoke_evaluator` **decorators in-process** | `APIWorkflowRunner` → `workflows_service.invoke_workflow` **HTTP**, wrapped by `APICachedRunner` (hashed-trace reuse) | SDK runs the user's local Python; API calls the workflow service. **`execute_batch` is now concurrent + semaphore-bounded on BOTH** (the SDK was sequential pre-fix). | +| Result setter (`.set`) | `SDKResultSetter` → `apopulate(results=[cell])` (`POST /simple/evaluations/{id}/populate`) | `APIResultSetter` → `evaluations_service.set_results([cell])` | Both **live per-cell**. API binds `timestamp`/`interval` at construction; SDK has no temporal axis. | +| Metrics refresh | `SDKMetricsRefresher` → `arefresh(run_id, scenario_id?)` (`POST /evaluations/metrics/refresh`) | `APIMetricsRefresher` → `evaluations_service.refresh_metrics(...)` | Same two engine calls (variational per scenario, global at end). API adapter also handles **temporal** buckets (timestamps+interval) and rejects scenario+timestamp; SDK is scenario-or-global only. | +| Scenario editor (`edit_scenario`) | `SDKScenarioEditor` → `aedit_scenario(scenario_id, status, tags, meta)` (`PATCH /evaluations/scenarios/{id}`) | `APIScenarioEditor` → `evaluations_service.edit_scenario(EvaluationScenarioEdit(...))` | Both carry `tags`/`meta` and tolerate a run closed mid-flight (SDK: HTTP 409 → None; API: `except EvaluationClosedConflict`). | +| Trace fetcher (`fetch_trace`) | `SDKTraceFetcher` → `afetch_trace` | `APITraceFetcher` → `fetch_trace(tracing_service, ...)` | Same callable contract; different fetch backend. | +| Scenario factory (`create_scenario`) | `_PreMintedScenarios` cursor over bulk-minted scenarios | `_OrderedScenarios` cursor over recovered scenarios | Same shape now (both ordered, lock-guarded cursors over a pre-collected list). | +| Run create / close | `acreate`/`aclose` (HTTP) | `evaluations_service.create_run`/`edit_run` (in-process) | SDK is an HTTP client of the same endpoints the API serves internally. | + +Pattern: same protocol, SDK = HTTP-client / API = in-process-service. The +`SDK*`/`API*` class names are intentional peers. + +--- + +## 3. DIFFERENT — genuine structural divergence (mostly API-only capability) + +| Aspect | SDK | API | +|---|---|---| +| Driver | `process_run_locally` — value-resolved, single-shot, in-process `await` | `APISliceProcessor.process` — run-resolved, worker-driven | +| Entity resolution | in the caller (`evaluate.py` → revision ids; `executor._retrieve_revisions` → objects) | inside `process` (`_resolve_runners_and_revisions`), from the persisted run graph | +| Re-execution / retry | none (always mint fresh, run once) | first-class: `TensorSlice` (scenario_ids × step_keys × repeat_idxs), `process_mode: fill-missing\|force`, `seed_bindings`, reuse counting, `_cell_is_addressed`, per-scenario source recovery from input cells | +| Run finalization | `aclose(status=run_status.value)` — single inline driver, no floor | `_finalize_run_after_slice` — re-fetch current run, severity-floor across concurrent slices, flip `is_active`, tolerate closed-run | +| Liveness / temporal | absent (batch only) | `is_live`, scheduler re-ticks (`newest`/`oldest` windows), temporal metric buckets, run stays RUNNING | +| Worker orchestration | none (library call) | Taskiq tasks, `_with_job_lock(allow_concurrency=...)` (singleton for `run_from_source`, per-job for `run_from_batch`/`rerun`), heartbeat | +| Tensor ops | uses `populate` + `process` + `refresh` only | all five: `probe`, `populate`, `process`, `prune`, `refresh` | +| Queue / online | none | `SimpleQueueSettings` / `EvaluationQueueData` — user assignment, `batch_size`/`batch_offset` | +| Metrics read-back | `aquery_global` for the `aevaluate` return (`{run, scenarios, metrics}`) | none (worker writes only; consumed via UI/API) | +| Aggregate refresh | engine's end-of-slice global only | `tensor.refresh`: variational + temporal-or-global by `is_live` | + +--- + +## 4. Flow taxonomy — three orthogonal axes (API) + +"Live / batch / source / slice" are not one list; they are three independent axes +that combine into the topology dispatches. + +| Axis | Values | Controls | +|---|---|---| +| **Source** | source-ingest (testset/query → mint) · direct-batch (explicit trace/testcase ids → mint) · slice-reexecute (existing scenarios, recover from cells) | where scenarios/inputs come from | +| **Liveness** | batch (`is_live=False`: one-shot, global metrics, finalizes) · live (`is_live=True`: scheduler re-ticks, temporal buckets, stays RUNNING) | temporal semantics | +| **Process mode** | `force` (ingest: fill all cells) · `fill-missing` (retry: only empty cells) | re-execution strategy | + +Entry points (`tasks/run.py`): `run_from_source` (topology-dispatched: `live_query`, +`batch_query`, `batch_testset`, `batch_invocation`, `queue_*`), `run_from_batch` +(direct id batches into open queue runs), `rerun` (slice re-execute by coordinate; +owns the `tensor.refresh` boundary). + +**The SDK implements exactly one cell of this matrix:** `batch · source-ingest · +force` — the `batch_testset` shape (testset → app → evaluator), single-shot. Everything +live, query, queue, retry, and worker-orchestration is API-only by design. The SDK +is a thin in-process client of one flow, sharing the engine + planner + topology +classifier + `TensorSlice` model but not the orchestration around them. + +--- + +## 5. What Option A changed + +Before: the API looped per scenario into the engine on **every** path (ingest and +re-execute), so cross-scenario concurrency never materialized (a 100-scenario run +ran serially; each `process_sources` call got a fresh semaphore used only for one +scenario's repeats). + +After: `APISliceProcessor.process` does a **recovery pass** (per scenario: resolve +source, compute addressed/target cells + reuse, recover upstream context) then a +**single** `_run_sdk_source_slice` over all scenarios with: +- an ordered cursor `create_scenario` (`_OrderedScenarios`), +- a per-`(scenario, cell)` `plan_cell_filter` (keyed by `cell.scenario_id`), +- a per-scenario `initial_context_by_repeat` **callable** (lazy, memory-bounded — + the engine resolves each scenario's recovered context under the `gather`). + +This closed the last two SDK/API divergences in the shared cell (engine call shape, +scenario factory) and gives the API cross-scenario concurrency, bounded by the same +`batch_size` semaphore. `timestamp`/`interval` are constant per call (verified: +ingest seeds one value; rerun passes None), so no per-scenario threading was needed +for them. diff --git a/docs/designs/access-controls-refactor-plan.md b/docs/designs/access-controls-refactor-plan.md new file mode 100644 index 0000000000..ce557a1771 --- /dev/null +++ b/docs/designs/access-controls-refactor-plan.md @@ -0,0 +1,214 @@ +# Access / Entitlements / Permissions Refactor — Migration Plan + +Status: PROPOSED (no edits applied). For review before execution. + +## Goal (per user) + +Restructure the EE access surface so concerns are cleanly separated: + +- `core/access/controls.py` — **shared** build/orchestration: env loading, + `_build_controls()`, the singleton, `controls_hash`, shared validators/schemas. + **Depends on** `entitlements/controls.py` + `permissions/controls.py` (one-way, + downward — `access` is the composition layer on top). +- `core/entitlements/controls.py` — **plans only**: plan/quota/counter/gauge/ + throttle parsing + overlays + `get_plan*` accessors. +- `core/permissions/controls.py` — **roles only**: role builders/parsers/overlays + + `get_role*` accessors. +- Move `utils/entitlements.py` + `utils/permissions.py` into `core/` (they are + stateful service layers, not utils). + +Dependency direction (no cycles): +``` +core/{entitlements,permissions}/types.py (leaf types) + ▲ +core/entitlements/controls.py (plans) core/permissions/controls.py (roles) + ▲ ▲ + └───────────────┬────────────────────────┘ + core/access/controls.py (shared build + singleton + hash) + ▲ + utils/* (services) , apis/fastapi/access/router.py , db_manager_ee, ... +``` + +## Current `entitlements/controls.py` function inventory (single file today) + +PLAN cluster: +- `_PlanOverride` (47), `_default_plans` (76), `_validate_flag_key` (82), + `_validate_counter_key` (89), `_validate_gauge_key` (96), + `_parse_plans_override` (103) +- `_ThrottleOverlay` (565), `_DefaultPlanOverlay` (578), `_merge_quota` (591), + `_merge_throttle` (602), `_parse_default_plan_overlay` (614), + `_apply_default_plan_overlay` (646), `_resolve_default_plan_slug` (721) +- accessors: `get_plans` (812), `get_plan` (817), `get_plan_entitlements` (824), + `get_plan_description` (831) +- `_DEFAULT_PLAN_DESCRIPTIONS` (module-level, near _default_plans) + +ROLE cluster: +- `_RoleOverride` (57), `_read_only_permissions` (165), + `_viewer_permissions_for_scope` (175), `_admin_permissions_for_scope` (186), + `_minima_for` (197), `_default_roles` (236), `_parse_roles_override` (280), + `_RoleOverlayEntry` (413), `_parse_roles_overlay` (426), + `_apply_roles_overlay` (506) +- accessors: `get_roles` (838), `get_role` (845), `get_role_permissions` (853), + `get_role_description` (861) + +SHARED: +- `_validate_permission` (270) — used by role parsers; depends on `Permission`. + (Move to permissions/controls.py — it is role/permission-only despite the name.) +- `_build_controls` (740) — orchestrates plan+role parse+overlay, builds the + combined `controls_hash`, logs sources. +- singleton: `_PLANS, _PLAN_DESCRIPTIONS, _ROLES, _CONTROLS_HASH = _build_controls()` (804) +- `get_controls_hash` (868) +- imports: `env`, `hashlib`, `dumps`, pydantic; `entitlements.types` (Tracker, + Flag, Counter, Gauge, DEFAULT_ENTITLEMENTS, DefaultPlan, OWNER_PERMISSIONS, + Quota, SCOPES, Throttle); `permissions.types` (Permission, DefaultRole, RequiredRole) + +Verdict: plan and role clusters have **zero cross-calls**. Only `_build_controls` ++ the shared `controls_hash` + `SCOPES` couple them. Clean to split. + +## Target module contents + +### `core/entitlements/controls.py` (plans) +- All PLAN-cluster functions above. +- New internal: `build_plan_controls() -> (plans, descriptions)` — does the + env-or-default + plan-overlay logic currently inlined in `_build_controls` + lines 746-766. +- Public `get_plan*` accessors read a module-level `_PLANS`/`_PLAN_DESCRIPTIONS` + populated by `access/controls.py` via a registration hook (see "singleton" + below) OR keep accessors in access/controls.py. **Decision needed — see Q1.** +- Imports `entitlements.types` only. No permissions import. + +### `core/permissions/controls.py` (roles) +- All ROLE-cluster functions + `_validate_permission`. +- New internal: `build_role_controls() -> roles` — env-or-default + role-overlay + logic currently inlined in `_build_controls` lines 768-779. +- Public `get_role*` accessors. Same singleton question (Q1). +- Imports `permissions.types` (+ `SCOPES`, `OWNER_PERMISSIONS` from + entitlements.types — note this is a small permissions→entitlements.types dep; + acceptable since types are leaf. Alternatively move `SCOPES`/`OWNER_PERMISSIONS` + to a neutral spot — see Q2). + +### `core/access/controls.py` (shared orchestration) +- `_build_controls()` — calls `build_plan_controls()` + `build_role_controls()`, + computes the combined `controls_hash`, logs sources. +- The singleton init. +- `get_controls_hash()`. +- Imports BOTH `entitlements.controls` and `permissions.controls`. + +### `utils/ → core/` service move +- `utils/entitlements.py` → `core/entitlements/service.py` + (NOTE: an `entitlements/service.py` was just deleted — the dead + `EntitlementsService`. This new file is the *check_entitlements* service, a + different thing. Name ok, or use `core/entitlements/checks.py` — see Q3.) +- `utils/permissions.py` → `core/permissions/service.py` (or `checks.py`). + +## The singleton question (Q1 — the crux) + +Today: `get_plans`/`get_roles` read module-level `_PLANS`/`_ROLES` built at import +of `controls.py`. If plans/roles accessors live in their *own* modules but the +build is orchestrated in `access/controls.py`, the accessor modules can't build +their own singletons (they'd each only know their half, and env-override parsing ++ hashing is centralized). + +Two clean designs: + +- **(A) Accessors stay with the data, access registers into them.** + `entitlements/controls.py` owns `_PLANS` + `get_plan*`; `permissions/controls.py` + owns `_ROLES` + `get_role*`. Each exposes `build_*_controls()` (pure, no global) + AND a `register_*_controls(...)` setter. `access/controls.py` at import calls + build for both, computes hash, then registers each half into its module + + stores the hash. Accessors raise if not yet registered. + - Pro: accessors live with their domain; clean reads. + - Con: import order matters — *something* must import `access/controls.py` at + startup before any accessor is called. Today that is implicit (importing + controls.py runs the build). Need an explicit bootstrap call (mirrors how + `utils/entitlements.py` already does `register_entitlements_services`). + +- **(B) access/controls.py owns the singleton AND all accessors.** + `entitlements/controls.py` + `permissions/controls.py` are pure (build + parse, + no globals). `access/controls.py` builds, holds `_PLANS`/`_ROLES`/`_HASH`, and + exposes ALL accessors (`get_plans`, `get_roles`, ...). Every importer switches + to `from ee.src.core.access.controls import get_plans/get_roles/...`. + - Pro: one singleton, one build, no registration hook, no import-order trap + (importing access.controls runs the build, exactly like today). + - Con: accessors live in `access`, not in their domain module. But that is + arguably correct — they read the *combined effective* controls, which is an + access-layer concern. entitlements/permissions controls.py become pure + builder/parser libraries. + +**Recommendation: (B).** It preserves today's "import runs the build" semantics +(no new bootstrap-order bug), keeps the split pure, and matches the existing +`apis/fastapi/access/` framing (access = the combined surface). entitlements/ +permissions `controls.py` become stateless builders; `access/controls.py` is the +stateful composition root. This is also the lowest-cycle-risk design. + +## Open questions for sign-off + +- **Q1 (singleton design):** A (register into domain modules) vs **B (access owns + singleton + all accessors)**. Recommend B. +- **Q2 (`SCOPES`/`OWNER_PERMISSIONS` location):** they sit in `entitlements.types` + but are permission/role concepts. Leave (small types-level dep from permissions + → entitlements.types), or move to `permissions.types` / a neutral `access.types`? + Recommend: leave for now (types are leaf, no cycle); revisit later. +- **Q3 (service file naming):** `core/{entitlements,permissions}/service.py` vs + `checks.py`. `service.py` collides conceptually with the just-deleted + EntitlementsService; `checks.py` is more descriptive (`check_entitlements`, + `check_action_access`). Recommend `service.py` (domain convention) — the + collision was a *deleted* file, so no real clash. +- **Q4 (scope):** do utils→core move in the SAME change as the controls split, or + separate? The controls split is ~15 accessor importers; the utils move is + ~24 (entitlements) + ~30 (permissions) + worker entrypoints. Recommend + **two separate changes**: (1) controls split, (2) utils→core move. + +## Blast radius (importer rewrites) + +Controls split (design B — all accessors move to `access.controls`): +- Plan-accessor importers (6): billing/router, tracing/service, subscriptions/ + settings, events/service, throttling, utils/entitlements. +- Role-accessor importers (5): oss workspace_router, oss organization_router, + ee workspace_router, utils/permissions, workspace_manager. +- `get_plan_description` + `get_plans`/`get_roles` in access/router (1). +- db_manager_ee (imports get_roles? — it imports from entitlements.controls; check). +- `test_controls_env_override.py`: ~28 in-string imports (`from + ...entitlements.controls import get_plans/get_roles/...`) → rewrite to + `...access.controls`. + +utils→core move: +- `utils/entitlements.py`: 24 importer files. +- `utils/permissions.py`: 30 importer files. +- plus worker entrypoints referencing `bootstrap_entitlements_services`. + +## Execution order (proposed) + +Change 1 — controls split (design B): +1. Create `core/entitlements/controls.py` content = pure plan builder/parser + (`build_plan_controls`, `_parse_plans_override`, overlays, `_default_plans`, + `_DEFAULT_PLAN_DESCRIPTIONS`). No globals, no accessors. +2. Create `core/permissions/controls.py` = pure role builder/parser + (`build_role_controls`, role parsers/overlays/minima, `_validate_permission`). + No globals, no accessors. +3. Create `core/access/controls.py` = imports both, `_build_controls()`, + singleton, ALL accessors (`get_plans/get_plan*/get_roles/get_role*`), + `get_controls_hash`. +4. Delete the role/plan logic from the old `entitlements/controls.py` (file + becomes the pure-plan module from step 1). +5. Rewire ~15 accessor importers + the test's in-string imports → `access.controls`. +6. Verify: ruff + per-file tests (test_access_controls, test_controls_env_override). + +Change 2 — utils→core service move (separate): +1. `git mv utils/entitlements.py core/entitlements/service.py`, + `git mv utils/permissions.py core/permissions/service.py`. +2. Rewire ~54 importers + worker entrypoints. +3. Verify. + +## Risks + +- **Import-order / cycle:** design B avoids the bootstrap-order trap. Confirm + `access.controls` importing both domain controls creates no cycle (it won't: + domain controls import only `*.types`, which import nothing). +- **The combined `controls_hash`** stays unified (computed in access) — no + behavior change. +- **Large importer churn**, esp. the utils move (~54 files) and the test file's + 28 in-string imports — mechanical but wide; do with sed + per-file verify. +- **Working tree already large/uncommitted** — recommend committing current + session work before Change 1, and doing Change 1 and Change 2 as separate + commits. diff --git a/docs/designs/dynamic-access-and-billing/findings.md b/docs/designs/dynamic-access-and-billing/findings.md index f0190cee35..bc420e1925 100644 --- a/docs/designs/dynamic-access-and-billing/findings.md +++ b/docs/designs/dynamic-access-and-billing/findings.md @@ -29,7 +29,7 @@ Initial scan surfaced 13 findings: one P0 (project-scope RBAC regression), four - **Closed (sixth sync pass — Copilot review at `91a60dd6`):** FIND-030 (P3, `wontfix` — the every-minute echo is an intentional cron-alive heartbeat present in both `events.txt` and `spans.txt`, not a leftover; Copilot misread it as debug code), FIND-031 (P3, fixed — added explicit `if not plan:` guard in the Stripe webhook handler so missing metadata returns a clear "Missing plan metadata on Stripe event" 400 instead of `Unknown plan 'None'`), FIND-032 (P2, fixed — removed the two `AGENTA_BILLING_TRIAL_*` env-var rows + the "Trial vars" sub-section from `research.md` and replaced with the per-pricing-entry `{trial: N}` marker, matching what shipped), FIND-033 (P3, `wontfix` — the wildcard-discards-extras behavior is correct: `owner = ["*"]` means everything; mixed inputs would be redundant, and non-enum slugs are filtered upstream at the access-controls boundary), FIND-034 (P3, `wontfix` — Pydantic evaluates field defaults at class-body time; the resulting parsed dict is a singleton, but no code mutates `EnvironSettings` instances so the latent mutability concern is theoretical), FIND-035 (P3, fixed by adding a one-line comment documenting that the warn-once flags race-but-don't-break; Copilot's `lru_cache` / logging-filter alternatives are deferred — the actual once-per-process semantics already work). - **Closed (docs style pass):** FIND-026 (P2, roles-overlay docs now lead with a deployment-wide root-slug shape instead of public `project` targeting), FIND-027 (P3, redundant default-plan overlay merge-semantics table removed), FIND-028 (P3, `Retention` enum reference table removed from operator docs), FIND-029 (P3, implementation-heavy reference section removed and the page trimmed toward the neighboring self-host how-to style). - **Closed (PR #4330 + migration audit):** FIND-014 (P2, design-doc retention defaults updated to match shipped code), FIND-015 (P3, override-vs-overlay terminology tightened in MDX), FIND-016 (P1, acceptance tests fixed by renaming `member`→`viewer`), FIND-017 (P2, `assert` replaced with explicit `if/raise EventException` in trial-checkout), FIND-018 (P3, `_normalize_pricing_entry` now rejects empty `{}` with a slug-pointing error; covered by new unit test), FIND-019 (P3, `existing_type=sa.String()` threaded through both `alter_column` calls in the member→viewer migration), FIND-020 (P0, EE `core` migration tree fork resolved by chaining `a1b2c3d4e5f7` after `9d3e8f0a1b2c`). -- **Closed (earlier sync passes):** FIND-021 (P1, `STRIPE_METER_NAMES` mapping added in `entitlements/types.py`; `meters/service.py` looks up event name + price via the mapping with a skip-path log; `subscriptions/settings.py` switched to flat pricing shape with operator-owned `traces` / `users` slot names plus reserved `free` / `trial` markers; `migrate_stripe_pricing.py` rewritten as an annotation helper for `free` / `trial`), FIND-022 (P3, mechanical `Counter.EVENTS` → `Counter.EVENTS_INGESTED` and `Counter.TRACES` → `Counter.TRACES_INGESTED` rename across `data-retention/README.md`, `data-retention/data-retention-periods.initial.specs.md`, and `ee-self-hosting/research.md`), FIND-023 (P2, `test_access_controls.py` fixture and tests migrated to `Counter.TRACES_INGESTED` / `Period.MONTHLY` / `Retention.MONTHLY` / `Flag.ACCESS`; 61/61 pass), FIND-024 (P3, Copilot-flagged stale `scripts/migrate_stripe_pricing.py` path in a validator error message — already resolved as a side-effect of the FIND-021 pricing rewrite; verified `grep` returns no matches), FIND-025 (P3, Copilot-flagged stale `Revises: e6f7a8b9c0d1` docstring header on `a1b2c3d4e5f7_unify_org_member_role_to_viewer.py` left over from the FIND-020 rebase — docstring updated to `Revises: 9d3e8f0a1b2c` to match `down_revision`). +- **Closed (earlier sync passes):** FIND-021 (P1, `REPORTS` mapping added in `entitlements/types.py`; `meters/service.py` looks up event name + price via the mapping with a skip-path log; `subscriptions/settings.py` switched to flat pricing shape with operator-owned `traces` / `users` slot names plus reserved `free` / `trial` markers; `migrate_stripe_pricing.py` rewritten as an annotation helper for `free` / `trial`), FIND-022 (P3, mechanical `Counter.EVENTS` → `Counter.EVENTS_INGESTED` and `Counter.TRACES` → `Counter.TRACES_INGESTED` rename across `data-retention/README.md`, `data-retention/data-retention-periods.initial.specs.md`, and `ee-self-hosting/research.md`), FIND-023 (P2, `test_access_controls.py` fixture and tests migrated to `Counter.TRACES_INGESTED` / `Period.MONTHLY` / `Retention.MONTHLY` / `Flag.ACCESS`; 61/61 pass), FIND-024 (P3, Copilot-flagged stale `scripts/migrate_stripe_pricing.py` path in a validator error message — already resolved as a side-effect of the FIND-021 pricing rewrite; verified `grep` returns no matches), FIND-025 (P3, Copilot-flagged stale `Revises: e6f7a8b9c0d1` docstring header on `a1b2c3d4e5f7_unify_org_member_role_to_viewer.py` left over from the FIND-020 rebase — docstring updated to `Revises: 9d3e8f0a1b2c` to match `down_revision`). EE unit suite green after the prior fixes: `test_access_controls.py` 61/61, `test_billing_settings.py` + `test_controls_env_override.py` 83/83. The three fixes from this pass (FIND-031 Stripe-event guard, FIND-032 doc cleanup, FIND-035 comment) are localized and don't change behavior beyond the failure messages. @@ -325,7 +325,7 @@ _None._ # The Stripe-side names are operator-owned (configured once on the Stripe # dashboard); changing an internal Counter value here does not require any # Stripe-dashboard change. - STRIPE_METER_NAMES: dict[str, str] = { + REPORTS: dict[str, str] = { Counter.TRACES_INGESTED.value: "traces", Gauge.USERS.value: "users", # Add a row here when a new counter/gauge becomes Stripe-reportable @@ -335,14 +335,14 @@ _None._ ``` - Use the mapping in two places: - 1. [meters/service.py:250](../../../api/ee/src/core/meters/service.py#L250) — `event_name = STRIPE_METER_NAMES[meter.key.value]` (with a clear `KeyError` → `[report] Skipping ... no Stripe meter mapping` log line that falls through the existing skipped-meter path). - 2. [subscriptions/settings.py](../../../api/ee/src/core/subscriptions/settings.py) — change `_VALID_METER_KEYS` to use `set(STRIPE_METER_NAMES.values())` (the **Stripe-side** names) instead of `{c.value for c in Counter} | {g.value for g in Gauge}`. The operator's `AGENTA_BILLING_PRICING.stripe.meters..price` is then keyed on Stripe-side names (matching what they configured on Stripe), and `get_stripe_meter_price(plan, meter.key.value)` does the lookup via `STRIPE_METER_NAMES[meter.key.value]`. + 1. [meters/service.py:250](../../../api/ee/src/core/meters/service.py#L250) — `event_name = REPORTS[meter.key.value]` (with a clear `KeyError` → `[report] Skipping ... no Stripe meter mapping` log line that falls through the existing skipped-meter path). + 2. [subscriptions/settings.py](../../../api/ee/src/core/subscriptions/settings.py) — change `_VALID_METER_KEYS` to use `set(REPORTS.values())` (the **Stripe-side** names) instead of `{c.value for c in Counter} | {g.value for g in Gauge}`. The operator's `AGENTA_BILLING_PRICING.stripe.meters..price` is then keyed on Stripe-side names (matching what they configured on Stripe), and `get_stripe_meter_price(plan, meter.key.value)` does the lookup via `REPORTS[meter.key.value]`. - Update the converter at [migrate_stripe_pricing.py](migrate_stripe_pricing.py) to be a no-op transform on the meter slot-names — it reads `"traces"` and writes `"traces"`, no remap needed because the schema now agrees with the Stripe-side names. Tighten the script's docstring to reflect this. - - Update the operator docs at [04-dynamic-access-controls.mdx](../../docs/self-host/04-dynamic-access-controls.mdx) and [05-dynamic-billing-settings.mdx](../../docs/self-host/05-dynamic-billing-settings.mdx) to document that the `stripe.meters` keys are the **Stripe-side meter event names**, not the internal Counter slugs, and that the set of valid keys is determined by the `STRIPE_METER_NAMES` table. + - Update the operator docs at [04-dynamic-access-controls.mdx](../../docs/self-host/04-dynamic-access-controls.mdx) and [05-dynamic-billing-settings.mdx](../../docs/self-host/05-dynamic-billing-settings.mdx) to document that the `stripe.meters` keys are the **Stripe-side meter event names**, not the internal Counter slugs, and that the set of valid keys is determined by the `REPORTS` table. - Alternatives: - Rename the Stripe-side meters on every operator's Stripe dashboard to match the new internal slug (`"traces"` → `"traces_ingested"`). Rejected: operationally invasive, requires coordinated downtime per deployment, and any future internal rename would require the same dance again. - Make the mapping operator-configurable per plan via an extra `stripe.meters..event_name` field. Could be added later as a layer on top of the global default if any deployment needs to override the canonical map; not required for the initial fix. -- Resolution: Applied the user-suggested mapping plus the broader pricing reshape that fell out of the conversation. `STRIPE_METER_NAMES: dict[str, str]` added at module scope in [`entitlements/types.py`](../../../api/ee/src/core/entitlements/types.py) keying internal Counter/Gauge values to Stripe-side meter event names (`Counter.TRACES_INGESTED.value` → `"traces"`, `Gauge.USERS.value` → `"users"`). Runtime in [`meters/service.py`](../../../api/ee/src/core/meters/service.py) uses `STRIPE_METER_NAMES.get(meter.key.value)` for both `event_name` and the price-id lookup, with a clear skip-path log if a meter is not Stripe-reportable. [`subscriptions/settings.py`](../../../api/ee/src/core/subscriptions/settings.py) was rewritten to a flat pricing shape mirroring the original `STRIPE_PRICING` (`{slug: {free?, trial?, : {price, quantity?}}}`); slot names are operator-owned and no longer validated against an internal enum, so `"traces"` and `"users"` flow end-to-end. The `AGENTA_BILLING_TRIAL_PLAN` / `AGENTA_BILLING_TRIAL_DAYS` env vars were collapsed into a per-entry `{trial: N}` marker in the pricing dict, and the corresponding fields were dropped from `BillingSettings` in [`api/oss/src/utils/env.py`](../../../api/oss/src/utils/env.py). [`migrate_stripe_pricing.py`](migrate_stripe_pricing.py) was rewritten as a small annotation helper (`annotate(pricing, free_slug, trial)`) that adds the `free` / `trial` markers without touching slot names. Operator-facing billing docs (`05-dynamic-billing-settings.mdx`) were deleted per user direction; access-control docs (`04-dynamic-access-controls.mdx`, `02-configuration.mdx`, both `env.ee.*.example` files) were stripped of all billing / Stripe references. +- Resolution: Applied the user-suggested mapping plus the broader pricing reshape that fell out of the conversation. `REPORTS: dict[str, str]` added at module scope in [`entitlements/types.py`](../../../api/ee/src/core/entitlements/types.py) keying internal Counter/Gauge values to Stripe-side meter event names (`Counter.TRACES_INGESTED.value` → `"traces"`, `Gauge.USERS.value` → `"users"`). Runtime in [`meters/service.py`](../../../api/ee/src/core/meters/service.py) uses `REPORTS.get(meter.key.value)` for both `event_name` and the price-id lookup, with a clear skip-path log if a meter is not Stripe-reportable. [`subscriptions/settings.py`](../../../api/ee/src/core/subscriptions/settings.py) was rewritten to a flat pricing shape mirroring the original `STRIPE_PRICING` (`{slug: {free?, trial?, : {price, quantity?}}}`); slot names are operator-owned and no longer validated against an internal enum, so `"traces"` and `"users"` flow end-to-end. The `AGENTA_BILLING_TRIAL_PLAN` / `AGENTA_BILLING_TRIAL_DAYS` env vars were collapsed into a per-entry `{trial: N}` marker in the pricing dict, and the corresponding fields were dropped from `BillingSettings` in [`api/oss/src/utils/env.py`](../../../api/oss/src/utils/env.py). [`migrate_stripe_pricing.py`](migrate_stripe_pricing.py) was rewritten as a small annotation helper (`annotate(pricing, free_slug, trial)`) that adds the `free` / `trial` markers without touching slot names. Operator-facing billing docs (`05-dynamic-billing-settings.mdx`) were deleted per user direction; access-control docs (`04-dynamic-access-controls.mdx`, `02-configuration.mdx`, both `env.ee.*.example` files) were stripped of all billing / Stripe references. - Sources: User-direction ("the Stripe meters and prices are associated to 'traces' not 'traces_ingested'", "maybe add a mapping as a global var", "drop the docs about billing", "drop mentions to BILLING or STRIPE ENV VARS in mdx docs", "drop from .env.example files too"); PR #4330 Copilot review (comment id 3260473541, thread `PRRT_kwDOJbjazM6C5oCs`) flagged the converter-surface symptom of the same root cause. ### FIND-022 — [CLOSED] Stale `Counter.EVENTS.retention` / `Counter.TRACES.retention` references in data-retention design docs diff --git a/docs/designs/dynamic-access-and-billing/summary.md b/docs/designs/dynamic-access-and-billing/summary.md index 76589346c2..7bfa7d0f49 100644 --- a/docs/designs/dynamic-access-and-billing/summary.md +++ b/docs/designs/dynamic-access-and-billing/summary.md @@ -16,7 +16,7 @@ A second hard cut shipped alongside: **events retention split out of billing**. **`settings.py` — the billing-settings builder.** `api/ee/src/core/subscriptions/settings.py` parses `AGENTA_BILLING_CATALOG` (list of `_CatalogEntry` Pydantic models with `extra="allow"` so operators can add display-only fields the frontend renders; `type ∈ {"standard", "custom"}` is enforced) and `AGENTA_BILLING_PRICING` (object keyed by plan slug — flat shape `{slug: {free?, trial?, : {price, quantity?}}}` that mirrors the original `STRIPE_PRICING` plus the two reserved markers). Reserved keys are `free` (bool) and `trial` (positive int days); every other key on a pricing entry is a Stripe-side slot name owned by the operator (the same names they used on the Stripe dashboard — typically `"traces"` and `"users"`) and is not validated against an internal enum. Cross-cutting validation runs in `_build_settings()`: every catalog `plan` slug must exist in the effective plan set, every pricing slug must exist in the effective plan set, at most one entry may carry `"free": true`, at most one entry may carry `"trial": N`, the free-plan fallback must be resolvable, and `AGENTA_ACCESS_DEFAULT_PLAN` (when set) must be in the effective plan set. Accessors `get_catalog()`, `get_catalog_plan(slug)`, `get_pricing()`, `get_pricing_plan(slug)`, `get_stripe_line_items(slug)`, `get_stripe_meter_price(plan, meter)`, `get_free_plan()`, `get_trial_plan()`, `get_trial_days()`, `trial_enabled()` are how the billing router, the meter reporting job, the subscription service, and the signup flow read the effective settings. `[billing-settings] catalog=defaults|env pricing=defaults|env free_plan=|None trial=/d|disabled` is logged at startup. -**Stripe meter name mapping.** Internal `Counter` / `Gauge` slugs (code identifiers) and Stripe-side meter event names (external configuration the operator owns on the Stripe dashboard) are independent surfaces. `api/ee/src/core/entitlements/types.py` carries a single source of truth at module scope: `STRIPE_METER_NAMES: dict[str, str] = {Counter.TRACES_INGESTED.value: "traces", Gauge.USERS.value: "users"}`. The runtime in `meters/service.py` uses `STRIPE_METER_NAMES.get(meter.key.value)` for both the `stripe.billing.MeterEvent.create(event_name=...)` argument and the per-meter price-id lookup; meters not in the map fall through the existing skipped-meter log path with a clear message. The pricing-config slot names in `AGENTA_BILLING_PRICING` agree with the Stripe-side names (operator-owned), so an internal rename of `Counter.TRACES_INGESTED` does not require a coordinated Stripe-dashboard change or an env-var rewrite on every deployment. Adding a new Stripe-reportable counter or gauge requires exactly two edits: add the `Counter` / `Gauge` member to `REPORTS`, and add the row to `STRIPE_METER_NAMES`. +**Stripe meter name mapping.** Internal `Counter` / `Gauge` slugs (code identifiers) and Stripe-side meter event names (external configuration the operator owns on the Stripe dashboard) are independent surfaces. `api/ee/src/core/entitlements/types.py` carries a single source of truth at module scope: `REPORTS: dict[str, str] = {Counter.TRACES_INGESTED.value: "traces", Gauge.USERS.value: "users"}`. The runtime in `meters/service.py` uses `REPORTS.get(meter.key.value)` for both the `stripe.billing.MeterEvent.create(event_name=...)` argument and the per-meter price-id lookup; meters not in the map fall through the existing skipped-meter log path with a clear message. The pricing-config slot names in `AGENTA_BILLING_PRICING` agree with the Stripe-side names (operator-owned), so an internal rename of `Counter.TRACES_INGESTED` does not require a coordinated Stripe-dashboard change or an env-var rewrite on every deployment. Adding a new Stripe-reportable counter or gauge requires exactly one edit: add the row to `REPORTS` (its key marks the meter reportable, its value is the Stripe-side name). **Default-plan overlay.** `AGENTA_ACCESS_DEFAULT_PLAN_OVERLAY` lets self-hosted operators tweak individual entitlement values on the default plan without restating the rest. The shape mirrors `_PlanOverride` (`description`, `flags`, `counters`, `gauges`, `throttles`) with one divergence: `throttles` is a map keyed by category slug instead of a list, so per-category patches don't require restating the whole throttle list. Merge semantics — `description` replaces, `flags` per-key replace, `counters`/`gauges` field-merge inside each quota (overlay-set fields replace, omitted fields keep the base; pass `null` to clear), and `throttles[category]` looks up the existing single-category throttle on the base plan and field-merges its `bucket`. Operators who need multi-category or endpoint-keyed throttle changes use `AGENTA_ACCESS_PLANS` instead. Resolution happens after the base plan map is built but before the public dicts are frozen, so all downstream accessors see the merged state. diff --git a/docs/designs/eval-loops/desired-architecture.md b/docs/designs/eval-loops/desired-architecture.md deleted file mode 100644 index 56d38165bd..0000000000 --- a/docs/designs/eval-loops/desired-architecture.md +++ /dev/null @@ -1,1227 +0,0 @@ -# Evaluation System - Desired Architecture - -**Created:** 2026-02-16 -**Status:** Design Proposal -**Related:** [Current State - Iteration Patterns](./iteration-patterns.md) - ---- - -## Executive Summary - -This document outlines the desired architecture for the evaluation system, focusing on: - -1. **Unification** - SDK loops become the canonical implementation used by both SDK and backend -2. **Separation of Concerns** - Clear boundaries between graph management, execution, tensor population, and interpretation -3. **Ports & Adapters** - Dependency injection for persistence, enabling different adapters (API, DAO, JSON) for the same core logic -4. **Consistency** - Same loops and patterns across human evaluations, auto evaluations, and live evaluations - ---- - -## Table of Contents - -- [Guiding Principles](#guiding-principles) -- [Four Concerns](#four-concerns) -- [Architectural Goals](#architectural-goals) -- [Ports & Adapters Design](#ports--adapters-design) -- [Loop Unification Strategy](#loop-unification-strategy) -- [Execution Modes](#execution-modes) -- [Migration Path](#migration-path) -- [Open Questions](#open-questions) - ---- - -## Guiding Principles - -### 1. Single Source of Truth for Iteration Logic - -**Current Problem:** -- SDK has one set of loops (`sdk/agenta/sdk/evaluations/preview/evaluate.py`) -- API has different loops (`api/oss/src/core/evaluations/tasks/legacy.py`, `live.py`) -- Logic duplication and divergence over time -- Changes must be made in multiple places - -**Desired State:** -- **SDK loops become the canonical implementation** -- Backend uses the same loops via dependency injection -- One place to modify iteration logic -- Guaranteed consistency between SDK and API - ---- - -### 2. Separation of Concerns - -The evaluation system has **four distinct concerns** that should not be conflated: - -| Concern | Responsibility | Should NOT | -|---------|---------------|------------| -| **Graph Management** | Create, modify steps (`add_step` / `remove_step`) | Execute the graph or populate results | -| **Orchestration (`process`)** | Drive execution across a `TensorSlice` | Manage the graph structure or directly store results | -| **Tensor Interface (`populate` / `probe` / `prune`)** | Write, read, and delete results in the tensor | Execute workflows or interpret data | -| **Tensor Interpretation** | Analyze and aggregate metrics | Modify the tensor or execute workflows | - -**Current Problem:** -- These concerns are mixed in the same functions -- Graph execution code also handles persistence -- Tensor population is tightly coupled to execution - -**Desired State:** -- Clean interfaces between concerns -- Each concern is independently testable -- Can swap implementations without affecting others - ---- - -### 3. Dependency Injection via Ports & Adapters - -**Current Problem:** -- SDK and API have hardcoded persistence mechanisms -- SDK makes HTTP calls directly -- API writes to database directly -- No way to test without external dependencies - -**Desired State:** -- Core iteration logic is pure (no I/O) -- Persistence is injected via interfaces (ports) -- Different adapters for different contexts: - - **SDK Context:** Adapter calls remote API - - **Backend Context:** Adapter calls DAO layer - - **Test Context:** Adapter writes to JSON or in-memory store - ---- - -### 4. Consistency Across Evaluation Types - -**Current Problem:** -- Human evaluations use different code than auto evaluations -- Live evaluations have separate iteration logic -- Batch evaluations have different patterns - -**Desired State:** -- **Same core loops** for all evaluation types -- Differences handled through: - - Configuration (which evaluators to run) - - Execution mode (batch, live, sliced) - - Data source (testset, live traces) -- Consistency reduces bugs and improves maintainability - ---- - -## Four Concerns - -### 1. Graph Management - -**What:** Define and modify the evaluation graph structure - -**Responsibilities:** -- Add/remove steps (`add_step` / `remove_step`) with `type` (input / invocation / annotation) and `origin` (human / custom / auto) -- Validate graph structure -- Persist step list in `run.data.steps` - -**Operations:** -```python -# Graph Management Interface -class EvaluationGraphManager(Protocol): - async def add_step( - self, - *, - type: Literal["input", "invocation", "annotation"], - origin: Literal["human", "custom", "auto"], - references: dict, - ) -> Step - - async def remove_step(self, *, step_key: str) -> None - async def get_steps(self, *, run_id: UUID) -> list[Step] -``` - -**Impacts on Tensor:** -- Adding a step adds a column dimension to the tensor -- Removing a step should be followed by `prune(TensorSlice(steps=[step_key]))` to delete stale results -- Steps are immutable by reference once added — change = `remove_step` + `add_step` - -**Examples:** -- User adds a new evaluator → `add_step(type="annotation", origin="auto", ...)` -- User removes an evaluator → `remove_step(step_key=...)` then `prune` stale results -- User changes evaluator config → new revision = new `step_key` - ---- - -### 2. Orchestration (`process`) - -**What:** Drive execution across a `TensorSlice` — the implementation of `process` - -**Responsibilities:** -- Resolve which cells of the tensor need work (guided by `TensorSlice` + optional `probe`) -- Invoke steps in the correct order (topological sort of the step list) -- Call `populate` for each result produced -- Handle errors by recording them in results (collect-errors mode — see Decisions) -- Concurrency model is up to the implementor (sequential, parallel, distributed) - -**Operations:** -```python -# Orchestration Interface -class EvaluationOrchestrator(Protocol): - async def process( - self, - *, - run: EvaluationRun, - slice: TensorSlice, - persistence: EvaluationPersistence, # provides populate/probe/prune - ) -> ProcessSummary -``` - -**Execution modes** (all expressed as `process(TensorSlice(...))`): -- **Full run:** `TensorSlice(scenarios="all", steps="all", repeats="all")` -- **Live (temporal):** `TensorSlice` scoped to scenarios derived from a trace query window -- **Targeted re-run:** `TensorSlice(scenarios=[id1, id2], steps=["evaluator-x"])` - -**Does NOT:** -- Persist results directly — delegates to `populate` -- Modify graph structure -- Aggregate metrics (delegates to tensor interpretation) - ---- - -### 3. Tensor Interface (`populate` / `probe` / `prune`) - -**What:** The three primitive operations on tensor cells — the persistence port that `process` depends on - -**Responsibilities:** -- `populate` — write a result for a `(scenario, step, repeat)` cell; result holds a `trace_id` reference and/or an `error` by value -- `probe` — read results for a `TensorSlice`; used by `process` to skip already-successful cells -- `prune` — delete results for a `TensorSlice`; used after step removal or to clear stale data - -**Operations:** -```python -# Tensor Interface (persistence port) -class EvaluationPersistence(Protocol): - async def populate( - self, - *, - run_id: UUID, - scenario_id: UUID, - step_key: str, - repeat_idx: int, - trace_id: Optional[UUID] = None, - testcase_id: Optional[UUID] = None, - error: Optional[str] = None, - ) -> EvaluationResult - - async def probe( - self, - *, - run_id: UUID, - slice: TensorSlice, - ) -> list[EvaluationResult] - - async def prune( - self, - *, - run_id: UUID, - slice: TensorSlice, - ) -> int # number of results deleted -``` - -**Storage Adapters:** -- **Remote API Adapter (SDK):** POSTs/queries `/evaluations/results` -- **DAO Adapter (Backend):** Writes/reads `evaluation_results` table -- **In-Memory Adapter (Testing):** Stores in dict - -**Does NOT:** -- Execute workflows -- Aggregate metrics (that's interpretation) -- Modify graph structure - ---- - -### 4. Tensor Interpretation - -**What:** Analyze and aggregate results from the tensor - -**Responsibilities:** -- Aggregate metrics across scenarios (mean, p95, etc.) -- Compute derived metrics -- Generate metrics tensors (aggregated view) -- Support different aggregation scopes (run, scenario, temporal) - -**Operations:** -```python -# Tensor Interpretation Interface -class EvaluationTensorInterpreter(Protocol): - async def aggregate_metrics( - self, - run_id: Optional[UUID], - scenario_id: Optional[UUID], - timestamp: Optional[datetime], - step_keys: list[str], - ) -> MetricsBucket - - async def get_metrics( - self, - scope: AggregationScope, - ) -> EvaluationMetrics - - async def compute_derived_metric( - self, - metric_name: str, - results: list[EvaluationResult], - ) -> Any -``` - -**Aggregation Scopes:** -- **Run-level:** Aggregate across all scenarios in a run -- **Scenario-level:** Metrics for a single scenario -- **Temporal:** Aggregate across time windows (e.g., last hour) -- **Step-level:** Metrics for a specific step across scenarios - -**Does NOT:** -- Execute workflows -- Modify results -- Change graph structure - ---- - -## Architectural Goals - -### Goal 1: SDK Loops as Canonical Implementation - -**Rationale:** -- SDK loops in `evaluate.py` are cleaner and more recent -- Better structured with clear separation of testsets → testcases → apps → evaluators -- Already handles multiple testset revisions, application variants, and evaluators -- More maintainable than legacy API loops - -**Approach:** -1. Extract SDK loops into a **shared core module** -2. Make loops **pure functions** (no I/O, side effects injected) -3. Backend imports and uses the same loops -4. Persistence injected via adapters - -**Example Structure:** -``` -agenta/ -├── core/ -│ └── evaluations/ -│ ├── engine/ -│ │ ├── graph.py # Graph management -│ │ ├── executor.py # Graph execution (canonical loops) -│ │ ├── tensor.py # Tensor population -│ │ └── metrics.py # Tensor interpretation -│ └── interfaces/ -│ ├── persistence.py # Ports (protocols) -│ └── adapters/ -│ ├── api.py # API adapter (SDK uses this) -│ ├── dao.py # DAO adapter (backend uses this) -│ └── json.py # JSON adapter (testing uses this) -``` - ---- - -### Goal 2: Clean Interfaces via Ports & Adapters - -**Port:** Interface (protocol) defining operations -**Adapter:** Implementation of the port for a specific context - -#### Example: Persistence Port - -```python -# Port (interface) -class EvaluationPersistence(Protocol): - """Port for persisting evaluation results.""" - - async def save_scenario(self, scenario: ScenarioCreate) -> Scenario: - """Save a scenario and return with ID.""" - ... - - async def save_result(self, result: ResultCreate) -> EvaluationResult: - """Save a single result.""" - ... - - async def save_results_batch( - self, - results: list[ResultCreate], - ) -> list[EvaluationResult]: - """Save multiple results in a batch.""" - ... - - async def get_result( - self, - run_id: UUID, - scenario_id: UUID, - step_key: str, - ) -> Optional[EvaluationResult]: - """Retrieve a specific result.""" - ... -``` - -#### Adapter 1: Remote API (for SDK) - -```python -class RemoteAPIPersistence: - """Adapter that persists via HTTP calls to backend API.""" - - def __init__(self, api_client: AgentaAPIClient): - self.client = api_client - - async def save_scenario(self, scenario: ScenarioCreate) -> Scenario: - response = await self.client.post( - "/evaluations/scenarios", - json=scenario.model_dump(), - ) - return Scenario(**response.json()) - - async def save_results_batch( - self, - results: list[ResultCreate], - ) -> list[EvaluationResult]: - response = await self.client.post( - "/evaluations/results/batch", - json=[r.model_dump() for r in results], - ) - return [EvaluationResult(**r) for r in response.json()] - - # ... other methods -``` - -#### Adapter 2: DAO (for Backend) - -```python -class DAOPersistence: - """Adapter that persists directly to database via DAO.""" - - def __init__(self, evaluations_dao: EvaluationsDAO): - self.dao = evaluations_dao - - async def save_scenario(self, scenario: ScenarioCreate) -> Scenario: - return await self.dao.create_scenario(scenario=scenario) - - async def save_results_batch( - self, - results: list[ResultCreate], - ) -> list[EvaluationResult]: - return await self.dao.create_results(results=results) - - # ... other methods -``` - -#### Adapter 3: JSON (for Testing) - -```python -class JSONPersistence: - """Adapter that persists to JSON file for testing.""" - - def __init__(self, file_path: Path): - self.file_path = file_path - self.data = {"scenarios": [], "results": []} - - async def save_scenario(self, scenario: ScenarioCreate) -> Scenario: - scenario_with_id = Scenario( - id=uuid4(), - **scenario.model_dump(), - ) - self.data["scenarios"].append(scenario_with_id.model_dump()) - self._write_to_file() - return scenario_with_id - - async def save_results_batch( - self, - results: list[ResultCreate], - ) -> list[EvaluationResult]: - results_with_ids = [ - EvaluationResult(id=uuid4(), **r.model_dump()) - for r in results - ] - self.data["results"].extend([r.model_dump() for r in results_with_ids]) - self._write_to_file() - return results_with_ids - - def _write_to_file(self): - with open(self.file_path, "w") as f: - json.dump(self.data, f, indent=2, default=str) - - # ... other methods -``` - ---- - -### Goal 3: Loop Unification and Refactoring - -#### Current State Analysis - -From [iteration-patterns.md](./iteration-patterns.md), we have: - -| Component | Loops | Pattern | -|-----------|-------|---------| -| SDK | Testsets → Testcases → Apps → Evaluators | 4-level nesting | -| Legacy API | Testcases → Evaluators | 2-level nesting | -| Live API | Query Steps → Traces → Evaluators | 3-level nesting | - -#### Desired State: Unified Loop Structure - -**Single canonical loop structure** that handles all cases: - -```python -async def process( - *, - run: EvaluationRun, - slice: TensorSlice, - persistence: EvaluationPersistence, # provides populate/probe/prune -) -> ProcessSummary: - """ - Canonical process implementation. - - Used by: - - SDK (with RemoteAPIPersistence adapter) - - Backend batch evaluation (with DAOPersistence adapter) - - Backend live evaluation (with DAOPersistence adapter) - - Tests (with InMemoryPersistence adapter) - """ - - # 1. Resolve scenarios from slice - scenarios = await _resolve_scenarios(run, slice) - - # 2. For each scenario - for scenario in scenarios: - node_outputs = {} - - # 3. Execute steps in topological order - for step in run.data.steps: - if slice.steps != "all" and step.key not in slice.steps: - continue - - for repeat_idx in _resolve_repeats(slice): - # Optionally probe to skip already-successful cells - # (idempotency — collect-errors mode means errors are not skipped) - - # Invoke the step (app/evaluator call) - output = await _invoke_step(step, scenario, node_outputs) - - # Populate result - await persistence.populate( - run_id=run.id, - scenario_id=scenario.id, - step_key=step.key, - repeat_idx=repeat_idx, - trace_id=output.trace_id, - error=output.error, - ) - - node_outputs[step.key] = output - - return ProcessSummary(...) -``` - -**Key Benefits:** -- **Same loop** for batch, live, sliced execution -- **Persistence injected** → works in SDK, backend, tests -- **Graph-based** → supports arbitrary node structures -- **Clean separation** → execution logic independent of I/O - ---- - -#### Refactoring Required - -To achieve unification, we need to: - -##### 1. **Split Loops** - -**Current:** SDK has monolithic loop that does execution + persistence + aggregation - -**Desired:** Split into: -- `execute_evaluation()` - Pure execution logic -- `persist_results()` - Delegated to adapter -- `aggregate_metrics()` - Separate interpreter - ---- - -##### 2. **Merge Loops** - -**Current:** Legacy and Live have separate, similar loops - -**Desired:** Single `execute_evaluation()` that handles both via `execution_mode`: -- `ExecutionMode.BATCH` → iterate over testset -- `ExecutionMode.LIVE` → iterate over trace query results -- `ExecutionMode.SLICED` → iterate over subset - ---- - -##### 3. **Change Loop Structure** - -**Current:** Some loops iterate by index (`for idx in range(nof_testcases)`) - -**Desired:** Iterate by object (`for scenario in scenarios`) -- More Pythonic -- Easier to slice/filter -- Better error handling - ---- - -##### 4. **Extract Pure Functions** - -**Current:** Loops have side effects (DB writes, API calls) inline - -**Desired:** Extract to pure functions: -```python -# Pure function (no I/O) -def prepare_evaluator_inputs( - testcase: Testcase, - app_output: dict, - trace: Trace, -) -> dict: - """Prepare inputs for evaluator invocation.""" - return { - "inputs": testcase.inputs, - "output": app_output, - "expected_output": testcase.expected_output, - "trace": trace, - } - -# Execution loop calls pure function, then injects I/O -evaluator_inputs = prepare_evaluator_inputs(testcase, app_output, trace) -result = await persistence.save_result(...) -``` - ---- - -### Goal 4: Support Multiple Execution Modes via `TensorSlice` - -All execution modes are expressed as `process(slice)` — the `TensorSlice` determines what subset of the tensor to work on. - -#### TensorSlice - -```python -@dataclass -class TensorSlice: - """Defines which cells of the tensor to target.""" - - scenarios: Literal["all", "none"] | list[UUID] = "all" - steps: Literal["all", "none"] | list[str] = "all" - repeats: Literal["all", "none"] | list[int] = "all" -``` - -#### Examples - -**Full run (all scenarios × all steps × all repeats):** -```python -await orchestrator.process( - run=run, - slice=TensorSlice(), # defaults to "all" on every dimension - persistence=dao_persistence, -) -``` - -**Live evaluation (scenarios derived from a trace query window):** -```python -trace_scenario_ids = await resolve_live_scenarios(run, start_time, end_time) -await orchestrator.process( - run=run, - slice=TensorSlice(scenarios=trace_scenario_ids, steps="all", repeats="all"), - persistence=dao_persistence, -) -``` - -**Targeted re-run (specific scenarios and evaluators):** -```python -await orchestrator.process( - run=run, - slice=TensorSlice( - scenarios=[scenario_1_id, scenario_2_id], - steps=["evaluator-accuracy", "evaluator-hallucination"], - repeats="all", - ), - persistence=dao_persistence, -) -``` - -**Fill missing results (probe first, then process gaps):** -```python -existing = await persistence.probe(run_id=run.id, slice=TensorSlice()) -missing_slice = compute_missing(full_tensor, existing) -await orchestrator.process(run=run, slice=missing_slice, persistence=dao_persistence) -``` -``` - ---- - -## Loop Unification Strategy - -### Phase 1: Extract Core Execution Logic - -**Goal:** Create pure, testable execution functions - -**Steps:** -1. Identify common iteration patterns across SDK and API loops -2. Extract pure functions (no I/O, no side effects) -3. Create `agenta/core/evaluations/engine/executor.py` with canonical loops -4. Add comprehensive unit tests (no DB, no API required) - -**Example Pure Function:** -```python -def build_execution_plan( - graph: EvaluationGraph, - scenarios: list[Scenario], -) -> list[ExecutionStep]: - """ - Build execution plan: which nodes to run for which scenarios. - - Pure function - no I/O, fully deterministic. - """ - plan = [] - for scenario in scenarios: - for node in graph.topological_order(): - plan.append( - ExecutionStep( - scenario_id=scenario.id, - node_id=node.id, - depends_on=[...], - ) - ) - return plan -``` - ---- - -### Phase 2: Define Ports (Interfaces) - -**Goal:** Define clean contracts for I/O operations - -**Steps:** -1. Create `agenta/core/evaluations/interfaces/persistence.py` with protocols -2. Create `agenta/core/evaluations/interfaces/invocation.py` for app/evaluator calls -3. Create `agenta/core/evaluations/interfaces/data_sources.py` for testsets/traces - -**Example Invocation Port:** -```python -class WorkflowInvoker(Protocol): - """Port for invoking applications and evaluators.""" - - async def invoke( - self, - workflow_id: UUID, - inputs: dict[str, Any], - ) -> InvocationResult: - """Invoke a workflow (app or evaluator) with inputs.""" - ... -``` - ---- - -### Phase 3: Implement Adapters - -**Goal:** Create concrete implementations for each context - -**Steps:** -1. Implement `RemoteAPIPersistence` (SDK uses this) -2. Implement `DAOPersistence` (backend uses this) -3. Implement `JSONPersistence` (tests use this) -4. Implement `InMemoryPersistence` (fast tests use this) - ---- - -### Phase 4: Migrate SDK to Use Core Loops - -**Goal:** SDK uses canonical implementation with remote adapter - -**Steps:** -1. Update `sdk/agenta/sdk/evaluations/preview/evaluate.py` -2. Import `execute_evaluation` from core -3. Pass `RemoteAPIPersistence` adapter -4. Remove duplicate loop logic -5. Ensure backward compatibility (same API surface) - -**Before:** -```python -# sdk/agenta/sdk/evaluations/preview/evaluate.py -async def aevaluate(...): - # 300+ lines of loop logic + HTTP calls - for testset in testsets: - for testcase in testcases: - # ... execute ... - await api_client.post("/results", ...) # Hardcoded API call -``` - -**After:** -```python -# sdk/agenta/sdk/evaluations/preview/evaluate.py -from agenta.core.evaluations.engine.executor import process -from agenta.core.evaluations.interfaces.adapters.api import RemoteAPIPersistence -from agenta.core.evaluations.types import TensorSlice - -async def aevaluate(...): - # Create persistence adapter - persistence = RemoteAPIPersistence(api_client=agenta_api) - - # Execute using canonical process - summary = await process( - run=run, - slice=TensorSlice(), # all scenarios × all steps × all repeats - persistence=persistence, - ) - - return summary -``` - ---- - -### Phase 5: Migrate Backend to Use Core Loops - -**Goal:** Backend uses same loops with DAO adapter - -**Steps:** -1. Update `api/oss/src/core/evaluations/tasks/legacy.py` -2. Import `execute_evaluation` from core (shared module) -3. Pass `DAOPersistence` adapter -4. Remove duplicate loop logic -5. Maintain same task API for worker - -**Before:** -```python -# api/oss/src/core/evaluations/tasks/legacy.py -async def evaluate_batch_testset(...): - # 700+ lines of loop logic + DAO calls - for idx in range(nof_testcases): - for jdx in range(nof_annotations): - # ... execute ... - await dao.create_result(...) # Hardcoded DAO call -``` - -**After:** -```python -# api/oss/src/core/evaluations/tasks/legacy.py -from agenta.core.evaluations.engine.executor import process -from agenta.core.evaluations.interfaces.adapters.dao import DAOPersistence -from agenta.core.evaluations.types import TensorSlice - -async def evaluate_batch_testset(...): - # Create persistence adapter - persistence = DAOPersistence(evaluations_dao=evaluations_dao) - - # Execute using canonical process - summary = await process( - run=run, - slice=TensorSlice(), # all scenarios × all steps × all repeats - persistence=persistence, - ) - - # Update run status - await update_run_status(run.id, summary) -``` - ---- - -### Phase 6: Consolidate Live and Batch - -**Goal:** Same loop handles live and batch via execution mode - -**Steps:** -1. Merge `live.py` and `legacy.py` into single task -2. Use `ExecutionMode.BATCH` vs `ExecutionMode.LIVE` -3. Data source (testset vs traces) resolved by mode -4. Remove code duplication - -**Example:** -```python -# api/oss/src/core/evaluations/tasks/evaluate.py -async def evaluate( - run_id: UUID, - slice: TensorSlice, -): - """ - Universal evaluation task. - - Handles: - - Batch testset evaluation: TensorSlice() - - Live trace evaluation: TensorSlice(scenarios=[...derived from trace query...]) - - Targeted re-run: TensorSlice(scenarios=[...], steps=[...]) - """ - run = await runs_dao.get(run_id) - persistence = DAOPersistence(evaluations_dao=evaluations_dao) - - summary = await process( - run=run, - slice=slice, - persistence=persistence, - ) - - return summary -``` - ---- - -## Execution Modes - -All modes are expressed as `process(run, TensorSlice(...), persistence)`. - -### Full Run (Testset Evaluation) - -**Data Source:** Testset scenarios already in the tensor -**Slice:** `TensorSlice()` — all scenarios × all steps × all repeats -**Use Case:** Evaluate a full testset against applications/evaluators - -**Step structure:** -``` -input step (testset) - → invocation step (application) - → annotation step (evaluator 1) - → annotation step (evaluator 2) -``` - ---- - -### Live Mode (Trace Evaluation) - -**Data Source:** Trace query result (temporal window → scenario IDs) -**Slice:** `TensorSlice(scenarios=[...trace-derived ids...], steps="all", repeats="all")` -**Use Case:** Continuously evaluate production traces - -**Step structure:** -``` -input step (query/traces) - → annotation step (evaluator 1) - → annotation step (evaluator 2) -``` - ---- - -### Targeted Re-run (Partial Execution) - -**Data Source:** Existing scenarios in the tensor -**Slice:** `TensorSlice(scenarios=[id1, id2], steps=["evaluator-x"], repeats="all")` -**Use Case:** Re-run specific evaluators on specific scenarios (e.g., after evaluator update) - ---- - -### Fill Gaps (Idempotent Completion) - -**Slice:** Computed from `probe` result — the set of missing or failed cells -**Use Case:** Resume an interrupted run; fill in cells that errored - -```python -existing = await persistence.probe(run_id=run.id, slice=TensorSlice()) -gap_slice = compute_missing(expected_full_tensor, existing) -await process(run=run, slice=gap_slice, persistence=persistence) -``` - ---- - -## Migration Path - -### Stage 1: Foundation (Weeks 1-2) - -**Goal:** Establish core architecture without breaking existing code - -**Tasks:** -1. ✅ Document current state ([iteration-patterns.md](./iteration-patterns.md)) -2. ✅ Document desired state (this document) -3. Create `agenta/core/evaluations/engine/` module structure -4. Define ports (protocols) in `interfaces/` -5. Extract pure functions from SDK loops -6. Write unit tests for pure functions (no I/O) - -**Deliverables:** -- Core module structure in place -- Ports defined and documented -- Pure execution logic extracted and tested -- No changes to existing SDK or API code yet - ---- - -### Stage 2: Adapters (Weeks 3-4) - -**Goal:** Implement persistence adapters - -**Tasks:** -1. Implement `DAOPersistence` adapter -2. Implement `RemoteAPIPersistence` adapter -3. Implement `InMemoryPersistence` adapter (for tests) -4. Write integration tests for each adapter -5. Document adapter usage patterns - -**Deliverables:** -- All adapters implemented and tested -- Adapter selection guide documented -- Integration tests passing -- Still no changes to existing SDK/API code - ---- - -### Stage 3: SDK Migration (Weeks 5-6) - -**Goal:** Migrate SDK to use core loops - -**Tasks:** -1. Update `sdk/agenta/sdk/evaluations/preview/evaluate.py` -2. Replace loop logic with `execute_evaluation()` call -3. Use `RemoteAPIPersistence` adapter -4. Maintain backward compatibility (same API surface) -5. Run full SDK test suite -6. Update SDK documentation - -**Deliverables:** -- SDK uses canonical loops -- All SDK tests passing -- Backward compatible (no breaking changes) -- SDK documentation updated - ---- - -### Stage 4: Backend Migration - Batch (Weeks 7-8) - -**Goal:** Migrate backend batch evaluation to use core loops - -**Tasks:** -1. Update `api/oss/src/core/evaluations/tasks/legacy.py` -2. Replace loop logic with `execute_evaluation()` call -3. Use `DAOPersistence` adapter -4. Maintain same task API for workers -5. Run full API test suite -6. Performance benchmarking - -**Deliverables:** -- Backend batch evaluation uses canonical loops -- All API tests passing -- Performance meets or exceeds baseline -- Task API unchanged (backward compatible) - ---- - -### Stage 5: Backend Migration - Live (Weeks 9-10) - -**Goal:** Migrate backend live evaluation to use core loops - -**Tasks:** -1. Update `api/oss/src/core/evaluations/tasks/live.py` -2. Replace loop logic with `execute_evaluation()` call -3. Use `DAOPersistence` adapter -4. Use `ExecutionMode.LIVE` with temporal scope -5. Run full API test suite -6. Performance benchmarking - -**Deliverables:** -- Backend live evaluation uses canonical loops -- All API tests passing -- Performance meets or exceeds baseline -- Temporal execution working correctly - ---- - -### Stage 6: Consolidation (Weeks 11-12) - -**Goal:** Merge batch and live into single task - -**Tasks:** -1. Create unified `evaluate()` task -2. Handle batch/live via `ExecutionMode` parameter -3. Deprecate `evaluate_batch_testset` and `evaluate_live_query` -4. Update all callers to use new unified task -5. Remove deprecated code after migration period -6. Update documentation - -**Deliverables:** -- Single evaluation task handles all modes -- Deprecated tasks removed (or marked for removal) -- All callers migrated -- Documentation complete - ---- - -### Stage 7: Metrics & Interpretation (Weeks 13-14) - -**Goal:** Apply same architecture to metrics aggregation - -**Tasks:** -1. Extract metrics aggregation into `TensorInterpreter` -2. Define `MetricsPort` interface -3. Implement adapters for different analytics backends -4. Migrate `refresh_metrics()` to use interpreter -5. Add support for custom aggregation functions - -**Deliverables:** -- Metrics aggregation uses ports & adapters -- Custom aggregations supported -- All metrics tests passing -- Performance validated - ---- - -## Decisions - -### 1. Graph Representation - -**Decided: Declarative configuration** — `run.data.steps` is a list of steps with `type`, `origin`, and `references`. The execution engine derives topology from this list. No explicit nodes+edges graph object. - ---- - -### 2. Error Handling - -**Decided: Collect errors** — execution continues across all cells; failures are recorded as error-valued results. - -This is a natural fit for the data model: every `EvaluationResult` holds either a `trace_id` (reference to a successful trace) or an `error` stored by value — or both if a trace partially succeeded. Errors are visible in the tensor alongside successes without stopping the run. - ---- - -### 3. Concurrency - -**Decided: Implementation concern for `process`** — the architecture does not mandate sequential vs. parallel vs. distributed execution. The `TensorSlice` contract specifies _what_ to run; each `process` implementation (SDK, Taskiq worker, future distributed executor) chooses its own concurrency model. The persistence port (`populate`/`probe`/`prune`) must be safe to call concurrently. - ---- - -### 4. Tensor Sparsity - -**Decided: Handled by `process` and `populate`** — missing cells are a natural outcome of partial runs, failures, or slice-targeted execution. The mechanism for filling gaps is `process(TensorSlice(...))` targeting the missing subset, with `probe` used to discover what is missing. No separate sparsity-handling primitive is needed. - ---- - -### 5. Shared Module Packaging - -**Decided: Monorepo shared module for now.** Revisit if/when the canonical loop is stable and independent versioning becomes necessary. - ---- - -### 6. Backward Compatibility - -**Strategy:** -- Keep old task signatures; delegate to new implementation internally -- New stack ships under `/preview/*` while old endpoints remain mounted -- Remove old loops only after both SDK and backend are migrated and tested - ---- - -## Success Criteria - -### Technical Metrics - -- [ ] **Single Source of Truth:** SDK and API use same execution loops -- [ ] **Test Coverage:** >90% coverage for core execution logic -- [ ] **Performance:** New implementation matches or exceeds current performance -- [ ] **Modularity:** Can swap persistence adapters without changing execution logic -- [ ] **Consistency:** Same loops used for batch, live, sliced, incremental modes - -### Code Quality Metrics - -- [ ] **Reduced Duplication:** <10% code duplication between SDK and API evaluation logic -- [ ] **Pure Functions:** >80% of execution logic is pure (no I/O) -- [ ] **Interface Adherence:** All I/O goes through defined ports -- [ ] **Documentation:** All ports, adapters, and execution modes documented - -### Functional Metrics - -- [ ] **Backward Compatibility:** All existing SDK and API tests pass -- [ ] **Feature Parity:** New implementation supports all current features -- [ ] **Error Handling:** Graceful handling of failures with clear error messages -- [ ] **Observability:** Execution progress and errors are logged/traced - ---- - -## Related Documentation - -- [Current State - Iteration Patterns](./iteration-patterns.md) -- [API Architecture Patterns](../../../AGENTS.md#api-architecture-patterns-oss--ee) -- [Testing Documentation](../testing/README.md) - ---- - -## Appendix: Example End-to-End Flow - -### Before (Current State) - -**SDK:** -```python -# sdk/agenta/sdk/evaluations/preview/evaluate.py -async def aevaluate(...): - for testset in testsets: - for testcase in testcases: - for app in apps: - result = await invoke_app(...) - await agenta_api.post("/results", result) # Direct API call - for evaluator in evaluators: - result = await invoke_evaluator(...) - await agenta_api.post("/results", result) # Direct API call -``` - -**Backend:** -```python -# api/oss/src/core/evaluations/tasks/legacy.py -async def evaluate_batch_testset(...): - for idx in range(nof_testcases): - for jdx in range(nof_annotations): - result = await invoke_evaluator(...) - await dao.create_result(result) # Direct DAO call -``` - -**Problems:** -- Duplicate loop logic (SDK and backend) -- Hardcoded persistence (API calls vs DAO) -- Different iteration patterns (objects vs indices) -- Hard to test without external dependencies - ---- - -### After (Desired State) - -**Core Module (Shared):** -```python -# agenta/core/evaluations/engine/executor.py -async def process( - *, - run: EvaluationRun, - slice: TensorSlice, - persistence: EvaluationPersistence, # Injected — provides populate/probe/prune -) -> ProcessSummary: - scenarios = await resolve_scenarios(run, slice) - - for scenario in scenarios: - node_outputs = {} - - for step in run.data.steps: - for repeat_idx in resolve_repeats(slice): - output = await invoke_step(step, scenario, node_outputs) - - await persistence.populate( - run_id=run.id, - scenario_id=scenario.id, - step_key=step.key, - repeat_idx=repeat_idx, - trace_id=output.trace_id, - error=output.error, - ) - node_outputs[step.key] = output - - return ProcessSummary(...) -``` - -**SDK:** -```python -# sdk/agenta/sdk/evaluations/preview/evaluate.py -from agenta.core.evaluations.engine.executor import process -from agenta.core.evaluations.interfaces.adapters.api import RemoteAPIPersistence -from agenta.core.evaluations.types import TensorSlice - -async def aevaluate(...): - persistence = RemoteAPIPersistence(agenta_api) - return await process(run=run, slice=TensorSlice(), persistence=persistence) -``` - -**Backend:** -```python -# api/oss/src/core/evaluations/tasks/evaluate.py -from agenta.core.evaluations.engine.executor import process -from agenta.core.evaluations.interfaces.adapters.dao import DAOPersistence -from agenta.core.evaluations.types import TensorSlice - -async def evaluate(run_id: UUID, slice: TensorSlice): - run = await runs_dao.get(run_id) - persistence = DAOPersistence(evaluations_dao) - return await process(run=run, slice=slice, persistence=persistence) -``` - -**Benefits:** -- ✅ Single source of truth (shared `process`) -- ✅ Persistence injected via `populate`/`probe`/`prune` port -- ✅ Same iteration pattern (objects, not indices) -- ✅ Testable without external dependencies (use `InMemoryPersistence`) -- ✅ Consistent across SDK, backend, tests - ---- - -**Document Status:** Draft for review and discussion -**Next Steps:** Review with team, refine based on feedback, begin Stage 1 implementation diff --git a/docs/designs/eval-loops/evaluation-operations.md b/docs/designs/eval-loops/evaluation-operations.md deleted file mode 100644 index ac12edf59f..0000000000 --- a/docs/designs/eval-loops/evaluation-operations.md +++ /dev/null @@ -1,540 +0,0 @@ -# Evaluation Operations - -**Created:** 2026-02-17 -**Purpose:** Document the supported operations on an evaluation — graph mutations, tensor interface, and orchestration -**Related:** -- [Evaluation Structure](./evaluation-structure.md) -- [Iteration Patterns](./iteration-patterns.md) -- [Desired Architecture](./desired-architecture.md) - ---- - -## Table of Contents - -- [Design Principle: Everything is a Mutation](#design-principle-everything-is-a-mutation) -- [Layer Model](#layer-model) -- [Step Model](#step-model) -- [Creation](#creation) -- [Graph Mutations](#graph-mutations) - - [add_step](#add_step) - - [remove_step](#remove_step) - - [Edit Step (via Remove + Add)](#edit-step-via-remove--add) -- [Tensor Mutations](#tensor-mutations) - - [add_scenario / remove_scenario](#add_scenario--remove_scenario) - - [increase_repeats / decrease_repeats](#increase_repeats--decrease_repeats) - - [populate / prune](#populate--prune) - - [probe](#probe) -- [Metrics](#metrics) - - [refresh_metrics](#refresh_metrics) -- [Flag Operations](#flag-operations) - - [get_flags / set_flags](#get_flags--set_flags) - - [set_flag](#set_flag) -- [Orchestration: process](#orchestration-process) -- [TensorSlice](#tensorslice) -- [Operation Summary](#operation-summary) - ---- - -## Design Principle: Everything is a Mutation - -**Creation with a graph definition is sugar for: create empty + apply mutations.** - -``` -create(graph_definition) -≡ -create_empty() -+ add_step(type="input", ...) [for each data source in graph] -+ add_step(type="invocation", ...) [for each application in graph] -+ add_step(type="annotation", ...) [for each evaluator in graph] -``` - -This makes the operation model explicit: -- If a mutation is supported, it is supported at creation time -- If a mutation is not supported (or gated by a flag), that applies at creation time too -- There are no "creation-only" special cases - ---- - -## Layer Model - -Operations are organized into two symmetric layers, plus an orchestration layer above both: - -``` -┌─────────────────────────────────────────────────────────┐ -│ Orchestration │ -│ process(slice) │ -│ SDK | Backend | Frontend — each impl. │ -│ All converge on the tensor layer below ↓ │ -└─────────────────────────────────────────────────────────┘ - -┌──────────────────────────┐ ┌──────────────────────────┐ -│ Graph Layer │ │ Tensor Layer │ -│ │ │ │ -│ add_step │ │ add_scenario │ -│ remove_step ───────────┼─►│ remove_scenario │ -│ │ │ │ -│ (steps define the │ │ increase_repeats │ -│ shape of the tensor) │ │ decrease_repeats │ -│ │ │ │ -│ │ │ populate (write) │ -│ │ │ prune (delete) │ -│ │ │ probe (read) │ -└──────────────────────────┘ └──────────────────────────┘ -``` - -**Graph layer** defines the structure: which steps exist, what they reference. -**Tensor layer** operates on the data: scenarios, repeats, and result cells. -**Cross-layer cascade:** `remove_step` triggers tensor-level operations (`remove_scenario` for input steps, `prune` for results of that step). - -**Tensor interface** (`probe` / `prune` / `populate`) is the shared contract. `process` is not a single implementation but a role: the SDK, the backend task runner, and the frontend each implement it differently, but all call `populate` to commit results. - ---- - -## Step Model - -All graph nodes are **steps**. A step has a `type` and an `origin`. - -### Step type - -**Field:** `type: Literal["input", "invocation", "annotation"]` -**Source:** `api/oss/src/core/evaluations/types.py:27` - -| Type | Meaning | Examples | -|------|---------|---------| -| `"input"` | Data source — provides inputs | Testset, Query | -| `"invocation"` | Invokes a workflow, produces a trace | Application/variant | -| `"annotation"` | Evaluates/scores, produces a trace | Evaluator/judge | - -### Step origin - -**Field:** `origin: Literal["human", "custom", "auto"]` -**Source:** `api/oss/src/core/evaluations/types.py:28` - -| Origin | Who populates results | Web behavior | Backend behavior | -|--------|----------------------|--------------|-----------------| -| `"auto"` | Backend | Transparent | Runs step automatically | -| `"human"` | A person via the UI | Prompts user for data | Waits — does not run | -| `"custom"` | External / programmatic | Transparent | Does not run | - -**Key rule:** `process` only runs `"auto"` steps. `"human"` and `"custom"` steps are populated via direct `populate` calls from outside the process orchestration. - -### Step definition (from codebase) - -```python -class EvaluationRunDataStep(BaseModel): - key: str # step_key — unique per run - type: Literal["input", "invocation", "annotation"] - origin: Literal["human", "custom", "auto"] - references: Dict[str, Reference] # points to specific revision - inputs: Optional[List[EvaluationRunDataStepInput]] = None -``` - -**Source:** `api/oss/src/core/evaluations/types.py:113-118` - -Nodes are **immutable by reference** — `references` points to a specific revision. It cannot be changed once set. - ---- - -## Creation - -### `create_empty()` - -Creates an evaluation with no graph and no tensor. All flags at defaults. - -``` -EvaluationRun: - steps: [] - tensor: { scenarios: [], results: [], metrics: [] } - flags: { is_live: false, is_active: true, repeat_target: "application", - reuse_traces: false, is_closed: false, - allow_decrease_repeats: false } -``` - -### `create(graph_definition?)` - -Equivalent to `create_empty()` followed by `add_step(...)` for each step in the definition. All the same mutation rules apply. - ---- - -## Graph Mutations - ---- - -### `add_step` - -**Operation:** `add_step(type, origin, references, key?, inputs?)` - -Adds a step to the graph. Multiple steps of the same type are supported. - -```python -add_step( - type: Literal["input", "invocation", "annotation"], - origin: Literal["human", "custom", "auto"], - references: Dict[str, Reference], - key: Optional[str] = None, # auto-generated if not provided - inputs: Optional[List[StepInput]] = None, -) -``` - -**Compatibility check:** `is_live = true` rejects `type = "input"` steps backed by a testset. - -**Effect on tensor:** None immediately. Data is created during process or via direct populate. - -**Allowed when `is_closed`:** No. - ---- - -### `remove_step` - -**Operation:** `remove_step(step_key: str)` - -Removes a step and all its associated tensor data. - -**Effect:** -1. Remove step from graph -2. `prune(TensorSlice(steps=[step_key], scenarios="all", repeats="all"))` — results -3. Flush metrics referencing `step_key` -4. If `type = "input"`: also prune scenarios sourced exclusively from this step - -**Allowed when `is_closed`:** No. - ---- - -### Edit Step (via Remove + Add) - -**There is no `edit_step` operation.** References are immutable. - -Changing a reference = `remove_step(old_key)` + `add_step(new_references)`. The flush cost is intentional and visible. - ---- - -## Tensor Mutations - -Tensor operations work on the data inside the evaluation — scenarios, repeats, and result cells. They are organized as symmetric pairs plus a read operation. - ---- - -### `add_scenario` / `remove_scenario` - -**`add_scenario(source: TestcaseRef | TraceRef)`** — adds a row to the tensor. - -```python -class TestcaseRef: - testcase_id: UUID - -class TraceRef: - trace_id: UUID - timestamp: Optional[datetime] # required for is_live=true - interval: Optional[str] # required for is_live=true -``` - -Normally created automatically during `process`. Direct `add_scenario` supports manual or incremental construction. - ---- - -**`remove_scenario(scenario_id: UUID)`** — removes a row and all its results. - -Equivalent to: -```python -prune(TensorSlice(scenarios=[scenario_id], steps="all", repeats="all")) -# + delete the scenario row itself -``` - -Triggered automatically by `remove_step` when the removed step was the sole input source for a scenario. - -**Both allowed when `is_closed`:** No. - ---- - -### `increase_repeats` / `decrease_repeats` - -**`increase_repeats(new_count: int)`** — expands the repeat dimension. `new_count > current`. Non-destructive: no data is deleted. New slots are empty until filled by `populate` or `process`. - ---- - -**`decrease_repeats(new_count: int)`** — shrinks the repeat dimension. `new_count < current`. - -**Effect:** -1. `prune(TensorSlice(scenarios="all", steps="all", repeats=[n, ..., old_count-1]))` -2. Full metrics flush — recompute with `refresh_metrics()` - -**Gated by flag:** `allow_decrease_repeats: bool` (default `false`). - -**Both allowed when `is_closed`:** No. - ---- - -### `populate` / `prune` - -**`populate(slice: TensorSlice, results: List[ResultData])`** — writes result cells. Creates or overwrites. - -```python -class ResultData: - scenario_id: UUID - step_key: str - repeat_idx: int - trace_id: Optional[UUID] # reference to execution trace - error: Optional[str] # set if the step failed -``` - -**This is the convergence point.** All `process` implementations — SDK, backend, frontend — call `populate` to commit results, regardless of how they produced them: -- Backend `process`: calls `populate` after running `auto` steps -- Frontend `process`: calls `populate` after a human submits an annotation -- External `process`: calls `populate` after a custom/programmatic step completes - -Does not automatically refresh metrics — call `refresh_metrics()` after bulk populate. - ---- - -**`prune(slice: TensorSlice)`** — deletes result cells within the slice and flushes affected metrics. - -```python -# All results for a specific step -prune(TensorSlice(steps=["eval-rev-abc"], scenarios="all", repeats="all")) - -# High repeat indices (after decrease_repeats) -prune(TensorSlice(steps="all", scenarios="all", repeats=[3, 4, 5])) - -# All results for specific scenarios -prune(TensorSlice(steps="all", scenarios=[sid1, sid2], repeats="all")) -``` - -Metrics are always flushed conservatively — recompute with `refresh_metrics()`. - -**Both allowed when `is_closed`:** No. - ---- - -### `probe` - -**`probe(slice: TensorSlice, status: StatusFilter) → List[ResultRef]`** — read-only. Returns all cells within the slice that match the status. - -```python -StatusFilter = Literal["missing", "success", "failure", "any"] -``` - -| Status | Meaning | -|--------|---------| -| `"missing"` | Cell in slice has no result yet | -| `"success"` | Result exists with no error and a valid `trace_id` | -| `"failure"` | Result exists with a non-null `error` | -| `"any"` | All cells in the slice | - -**Returns:** List of `(scenario_id, step_key, repeat_idx)` composite keys. - -**Primary use:** Determine what to (re-)process, or audit completeness before closing. - -**Allowed when `is_closed`:** Yes. - ---- - -## Metrics - -Metrics are derived from results. They are not part of the core tensor interface but follow the same slice-scoped pattern. - -### `refresh_metrics` - -**Operation:** `refresh_metrics(scope?: GlobalScope | VariationalScope | TemporalScope)` - -Recomputes metrics from current results. - -``` -1. Fetch results matching scope -2. Extract trace_ids -3. Run SQL aggregations over trace data -4. Write metrics entities -``` - -**When to call:** -- After `process` completes -- After `decrease_repeats` -- After `remove_step` -- After bulk `populate` -- On user-initiated recalculate - -**Allowed when `is_closed`:** Yes (metrics are derived, not structural). - ---- - -## Flag Operations - -Flags are properties of the `EvaluationRun` that control behavior. At the storage level there is no per-flag update — flags are always read as a set and written as a set. The user-facing API provides `set_flag` which wraps that read-modify-write cycle. - ---- - -### `get_flags` / `set_flags` - -**`get_flags(run_id) → EvaluationFlags`** — returns all current flag values. - -```python -class EvaluationFlags: - is_live: bool # online vs offline execution - is_active: bool # pause/resume (only meaningful when is_live=true) - repeat_target: Literal["application", "evaluator"] - reuse_traces: bool - is_closed: bool # lock evaluation against mutations - allow_decrease_repeats: bool # gate on destructive repeat reduction -``` - ---- - -**`set_flags(run_id, flags: EvaluationFlags)`** — low-level write of the full flags object. Replaces all flags atomically. - -This is the primitive. Not intended to be called directly in most contexts — use `set_flag` instead. - ---- - -### `set_flag` - -**`set_flag(run_id, name: str, value: Any)`** — set a single flag. Internally performs: - -``` -flags = get_flags(run_id) -flags[name] = value -set_flags(run_id, flags) -``` - -**Available flags and their constraints:** - -| Flag | Type | Constraint on set | -|------|------|-------------------| -| `is_live` | `bool` | Setting `true` when testset input steps exist → rejected | -| `is_active` | `bool` | Only meaningful when `is_live = true` | -| `repeat_target` | `"application" \| "evaluator"` | Changing after results exist → requires `prune` first (or explicit override) | -| `reuse_traces` | `bool` | No structural constraint | -| `is_closed` | `bool` | Setting `true` gates all subsequent mutations; setting `false` reopens | -| `allow_decrease_repeats` | `bool` | Safety gate — must be set `true` before calling `decrease_repeats` | - -**Allowed when `is_closed`:** -- `is_closed` itself: Yes — you must be able to set it to `false` to reopen -- All other flags: No — closed evaluation is read-only - ---- - -## Orchestration: process - -**Operation:** `process(slice: TensorSlice)` - -Drives `auto`-origin steps within the slice, in graph-topological order. - -`process` is **not a single implementation** — it is a role. The SDK, the backend task runner, and the frontend each implement it differently. What they share is the contract: read from the graph, produce results, call `populate`. - -``` -process(slice): - for each scenario in slice: - ensure scenario exists (add_scenario if needed) - for each auto step in slice (topological order): - for each repeat in slice: - if probe({scenario}, {step}, {repeat}, "success"): - skip (already done) - run step → trace_id or error - populate({scenario, step, repeat, trace_id, error}) - refresh_metrics() -``` - -**What it does NOT do:** -- Does not touch `human` or `custom` steps — those are populated from outside -- Does not run steps outside the slice - -**Common patterns:** - -```python -# Full run -process(TensorSlice(scenarios="all", steps="all", repeats="all")) - -# Retry all failures -failed = probe(TensorSlice(scenarios="all", steps="all", repeats="all"), status="failure") -process(TensorSlice( - scenarios=[r.scenario_id for r in failed], - steps="all", - repeats="all", -)) - -# Fill missing results for one evaluator -process(TensorSlice(scenarios="all", steps=["eval-rev-abc"], repeats="all")) - -# Fill new repeat slots after increase_repeats -process(TensorSlice(scenarios="all", steps="all", repeats=[3, 4])) -``` - -**Allowed when `is_closed`:** No. - ---- - -## TensorSlice - -A `TensorSlice` specifies a subset of the tensor along all three dimensions. Used by `probe`, `prune`, `populate`, and `process`. - -```python -class TensorSlice: - scenarios: Literal["all", "none"] | List[UUID] # scenario_ids - steps: Literal["all", "none"] | List[str] # step_keys - repeats: Literal["all", "none"] | List[int] # repeat_idx values -``` - -| Value | Meaning | -|-------|---------| -| `"all"` | Every item in this dimension | -| `"none"` | Nothing (empty slice — no-op for write operations) | -| `[...]` | Only the listed items | - ---- - -## Operation Summary - -### Graph Mutations - -| Operation | Flushes Results | Flushes Metrics | Allowed when `is_closed` | -|-----------|-----------------|-----------------|--------------------------| -| `add_step(...)` | No | No | No | -| `remove_step(key)` | Yes — for `key` | Yes — for `key` | No | -| ~~`edit_step`~~ | — | — | Not supported; use remove + add | - -### Tensor Mutations - -| Operation | Pair | Deletes Data | Flushes Metrics | Gated by Flag | Allowed when `is_closed` | -|-----------|------|--------------|-----------------|---------------|--------------------------| -| `add_scenario(...)` | ↕ | No | No | No | No | -| `remove_scenario(id)` | ↕ | Yes — scenario + its results | Yes — for scenario | No | No | -| `increase_repeats(n)` | ↕ | No | No | No | No | -| `decrease_repeats(n)` | ↕ | Yes — pruned repeats | Yes — full flush | `allow_decrease_repeats` | No | -| `populate(slice, results)` | ↕ | No (overwrites) | No | No | No | -| `prune(slice)` | ↕ | Yes — per slice | Yes — per scope | No | No | -| `probe(slice, status)` | — | No | No | No | Yes | - -### Metrics - -| Operation | Allowed when `is_closed` | -|-----------|--------------------------| -| `refresh_metrics(scope?)` | Yes | - -### Flag Operations - -| Operation | Effect | Allowed when `is_closed` | -|-----------|--------|--------------------------| -| `get_flags(run_id)` | Read all flags | Yes | -| `set_flags(run_id, flags)` | Write all flags atomically (low-level) | Only for `is_closed` itself | -| `set_flag(run_id, name, value)` | Read-modify-write a single flag | Only for `is_closed` itself | - -### Orchestration - -| Operation | Calls | Respects `origin` | Allowed when `is_closed` | -|-----------|-------|-------------------|--------------------------| -| `process(slice)` | probe + populate + refresh_metrics | Yes — only runs `auto` steps | No | - -### Invariants - -1. **Steps are immutable by reference.** Use remove + add to change a reference. The flush is intentional. -2. **Graph and tensor are symmetric.** Graph has `add_step` / `remove_step`; tensor has matching add/remove pairs for scenarios, repeats, and cells. -3. **`remove_step` cascades to the tensor.** It triggers `remove_scenario` (for input steps) and `prune` (for results of that step). -4. **Closed evaluations are read-only for structure and results.** `probe` and `refresh_metrics` work when closed; all mutations do not. -5. **Decreasing repeats is destructive.** Gated by `allow_decrease_repeats`. -6. **Metrics are always derived.** Flushing is cache invalidation, not data loss. -7. **`process` only touches `auto` steps.** `human` and `custom` steps are populated from outside via direct `populate` calls. -8. **`populate` is the convergence point.** All process implementations — SDK, backend, frontend — call `populate` to commit results. -9. **Creation is mutation.** No operation available at creation is unavailable afterward, and vice versa. - ---- - -**Document Status:** Draft -**Next Action:** Review layer model and tensor interface contract with team diff --git a/docs/designs/eval-loops/evaluation-structure.md b/docs/designs/eval-loops/evaluation-structure.md deleted file mode 100644 index 2edcdc2dde..0000000000 --- a/docs/designs/eval-loops/evaluation-structure.md +++ /dev/null @@ -1,1983 +0,0 @@ -# Evaluation Structure - -**Created:** 2026-02-16 -**Purpose:** Document the evaluation structure — graph, tensor, flags, and constraints -**Related:** -- [Current State - Iteration Patterns](./iteration-patterns.md) -- [Desired Architecture](./desired-architecture.md) -- [Refactoring Analysis](./refactoring-analysis.md) -- [Evaluation Operations](./evaluation-operations.md) - ---- - -## Table of Contents - -- [Overview](#overview) -- [Graph Components](#graph-components) -- [Execution Flags](#execution-flags) -- [Data Source Types](#data-source-types) -- [Compatibility Constraints](#compatibility-constraints) -- [Tensor Structure](#tensor-structure) - - [Five Entities](#five-entities) - - [Tensor Dimensions](#tensor-dimensions) - - [EvaluationResult: the Cell](#evaluationeresult-the-cell) - - [Tensor Population Flow](#tensor-population-flow) - - [Metrics](#metrics) -- [Current Graph Structures](#current-graph-structures) -- [Future Graph Structures](#future-graph-structures) - ---- - -## Overview - -An evaluation graph defines: -1. **Data sources** (what to evaluate) -2. **Execution steps** (applications, evaluators) -3. **Execution mode** (online vs offline, batch vs sliced) -4. **Constraints** (compatibility rules between components) - -The graph structure determines: -- What can be executed together -- How data flows between steps -- When and how often execution occurs -- What results are collected - ---- - -## Graph Components - -### Node Types - -#### 1. Query Node (Data Source) -**Purpose:** Load traces from the tracing system - -**Properties:** -- `type: "query"` -- `query_spec`: Query criteria (filters, time window, etc.) -- `time_window`: Optional time bounds (start_time, end_time) - -**Behavior:** -- **Offline mode:** Executes once, returns snapshot of matching traces -- **Online mode:** Executes periodically, windowed by current interval - -**Example:** -```json -{ - "type": "query", - "query_spec": { - "filters": { - "status": "success", - "application_id": "app-123" - }, - "time_window": { - "start_time": "2026-02-16T00:00:00Z", - "end_time": "2026-02-16T23:59:59Z" - } - } -} -``` - ---- - -#### 2. Testset Node (Data Source) -**Purpose:** Load test cases from a testset revision - -**Properties:** -- `type: "testset"` -- `testset_id`: UUID -- `testset_revision_id`: UUID -- `testset_variant_id`: Optional UUID - -**Behavior:** -- **Offline mode:** Returns all test cases from the testset revision -- **Online mode:** ❌ Not compatible (re-execution yields same data) - -**Example:** -```json -{ - "type": "testset", - "testset_id": "ts-123", - "testset_revision_id": "rev-456", - "testset_variant_id": "var-789" -} -``` - ---- - -#### 3. Application Node (Execution Step) -**Purpose:** Invoke an application workflow with inputs - -**Properties:** -- `type: "application"` -- `application_id`: UUID -- `application_revision_id`: UUID -- `application_variant_id`: Optional UUID - -**Inputs:** -- From data source node: `testcase.data` or `trace inputs` - -**Outputs:** -- `trace_id`: Execution trace -- `outputs`: Application outputs (extracted from trace) - -**Example:** -```json -{ - "type": "application", - "application_id": "app-123", - "application_revision_id": "rev-456" -} -``` - ---- - -#### 4. Evaluator Node (Execution Step) -**Purpose:** Invoke an evaluator workflow to score/judge outputs - -**Properties:** -- `type: "evaluator"` -- `evaluator_id`: UUID -- `evaluator_revision_id`: UUID -- `evaluator_variant_id`: Optional UUID - -**Inputs:** -- From data source node: `testcase` or `trace` -- From application node: `outputs`, `trace` - -**Outputs:** -- `trace_id`: Evaluation trace -- `metrics`: Evaluation scores/judgments - -**Example:** -```json -{ - "type": "evaluator", - "evaluator_id": "eval-123", - "evaluator_revision_id": "rev-456" -} -``` - ---- - -## Execution Flags - -### Current Flags - -#### 1. `is_live` - Online vs Offline Evaluation - -**Type:** `boolean` -**Default:** `false` -**Location:** `EvaluationRun.is_live` (or similar field) - -**Purpose:** Determines execution mode - -##### `is_live = false` (Offline / Batch Evaluation) -- **Execution:** One-time, on-demand or scheduled -- **Scenarios:** Created once for all inputs -- **Data sources:** Testsets OR Queries (snapshot) -- **Use case:** Evaluate a specific dataset or time window - -**Behavior:** -``` -1. Load data source (testset or query) -2. Create scenarios (one per input) -3. Execute graph for all scenarios -4. Complete run -``` - ---- - -##### `is_live = true` (Online / Live Evaluation) -- **Execution:** Periodic, runs on interval (e.g., every hour) -- **Scenarios:** Include `timestamp` and `interval` fields -- **Data sources:** Queries ONLY (windowed by interval) -- **Use case:** Continuously evaluate production traffic - -**Behavior:** -``` -For each interval (e.g., hourly): -1. Overwrite query time window with current interval - Example: interval = "2026-02-16T10:00:00Z to 2026-02-16T11:00:00Z" -2. Execute query → get traces from that window -3. Create scenarios (one per trace) with timestamp + interval -4. Execute graph for scenarios -5. Wait for next interval -``` - -**Scenario fields:** -```python -class EvaluationScenario: - id: UUID - run_id: UUID - timestamp: Optional[datetime] # Only for is_live=true - interval: Optional[str] # Only for is_live=true (e.g., "1h") - status: EvaluationStatus -``` - ---- - -#### Why Testsets are Incompatible with Online Evaluation - -**Constraint:** `is_live = true` → Data source MUST be Query - -**Reason:** -- **Testsets are static:** Re-executing a testset always yields the same test cases -- **Queries are dynamic:** Re-executing a query with different time windows yields different traces - -**Example:** - -``` -Testset (static): - Interval 1 (10:00-11:00): [testcase_1, testcase_2, testcase_3] - Interval 2 (11:00-12:00): [testcase_1, testcase_2, testcase_3] ← Same data! - Interval 3 (12:00-13:00): [testcase_1, testcase_2, testcase_3] ← Same data! - -Query (dynamic): - Interval 1 (10:00-11:00): [trace_a, trace_b] from 10:00-11:00 - Interval 2 (11:00-12:00): [trace_c, trace_d, trace_e] from 11:00-12:00 ← Different! - Interval 3 (12:00-13:00): [trace_f] from 12:00-13:00 ← Different! -``` - -**Implication:** -- Online evaluation with testsets would create duplicate scenarios for the same test cases every interval -- No new information would be gained -- Wasteful and confusing - ---- - -### Compatibility Matrix - -| Data Source | `is_live = false` (Offline) | `is_live = true` (Online) | -|-------------|-----------------------------|---------------------------| -| **Testset** | ✅ Compatible | ❌ Not compatible | -| **Query** | ✅ Compatible (snapshot) | ✅ Compatible (windowed) | - ---- - -#### Companion Flag: `is_active` (Works with `is_live`) - -**Type:** `boolean` -**Default:** `true` (when `is_live = true`) -**Location:** `EvaluationRun.is_active` (or similar field) - -**Purpose:** Controls whether online evaluation should actually execute on periodic ticks - -**Interaction with `is_live`:** - -| `is_live` | `is_active` | Behavior | -|-----------|-------------|----------| -| `false` | N/A | Offline evaluation (not periodic) | -| `true` | `true` | Online evaluation **running** (executes on ticks) | -| `true` | `false` | Online evaluation **paused** (skipped on ticks) | - -**Use cases:** -- **Pause online evaluation:** Set `is_active = false` to temporarily stop execution without deleting the run -- **Resume online evaluation:** Set `is_active = true` to restart execution -- **Debugging:** Pause problematic evaluations without losing configuration - -**Behavior on periodic tick:** -```python -# Periodic scheduler (e.g., every hour) -for run in get_online_evaluations(): - if run.is_live and run.is_active: - # Execute this online evaluation - await execute_live_evaluation(run) - elif run.is_live and not run.is_active: - # Skip - evaluation is paused - continue -``` - -**Status:** Likely implemented (needs verification) - ---- - -### Graph Configuration Flags - -These are the key flags that control evaluation behavior: - ---- - -#### 2. `repeat_target` (CRITICAL - Currently Implicit) -**Current state:** Hardcoded (location TBD in code) -**Current value:** Unknown (needs investigation) -**Type:** `Literal["application", "evaluator"]` - -**Purpose:** Determines where repeats are applied in the graph - -This is a **critical degree of freedom** that fundamentally changes: -- What variability is being measured -- Tensor structure (rows vs columns) -- Execution order -- Cost (app invocations vs evaluator invocations) - ---- - -##### `repeat_target = "application"` (Test Application Variability) - -**Fanout location:** Between inputs and application invocations - -**What's being tested:** Variability/consistency of the application under test - -**Graph structure:** -``` -Input 1 → Application (repeat 1) → Evaluator 1 - → Evaluator 2 - → Application (repeat 2) → Evaluator 1 - → Evaluator 2 - → Application (repeat 3) → Evaluator 1 - → Evaluator 2 - -Input 2 → Application (repeat 1) → Evaluator 1 - → Evaluator 2 - → Application (repeat 2) → Evaluator 1 - → Evaluator 2 - ... -``` - -**Execution flow:** -```python -for testcase in testcases: - for repeat_idx in range(repeats): # REPEAT HERE - scenario = create_scenario( - testcase=testcase, - repeat_idx=repeat_idx, - ) - - # Invoke application (different each repeat) - outputs = invoke_application(testcase.inputs) - - # Invoke evaluators (once per app repeat) - for evaluator in evaluators: - metrics = invoke_evaluator(testcase, outputs) -``` - -**Tensor structure:** -``` -Rows: Scenarios (Input × Repeat) -Cols: Steps (Testset, Applications, Evaluators) - - Testset App-1 App-2 Eval-1 Eval-2 Eval-3 -Input-1-r1 ✓ ✓ ✓ ✓ ✓ ✓ -Input-1-r2 ✓ ✓ ✓ ✓ ✓ ✓ -Input-1-r3 ✓ ✓ ✓ ✓ ✓ ✓ -Input-2-r1 ✓ ✓ ✓ ✓ ✓ ✓ -Input-2-r2 ✓ ✓ ✓ ✓ ✓ ✓ -Input-2-r3 ✓ ✓ ✓ ✓ ✓ ✓ - -Each row = one scenario (input + repeat index) -Each column = one step (evaluators run once per row) -``` - -**Use cases:** -- Testing LLM temperature/sampling variability: "How consistent is GPT-4 at temp=0.7?" -- Testing RAG retrieval variability: "Do we get different docs each time?" -- A/B testing with random assignment: "Does feature flag affect results?" -- Measuring baseline noise: "What's the natural variation in our application?" - -**Cost implications:** -- Application invoked: `num_inputs × repeats` times -- Evaluators invoked: `num_inputs × repeats × num_evaluators` times -- Example: 100 inputs, 3 repeats, 2 evaluators = 300 app calls, 600 evaluator calls - ---- - -##### `repeat_target = "evaluator"` (Test Evaluator Variability) - -**Fanout location:** Between application outputs and evaluator invocations - -**What's being tested:** Variability/consistency of the evaluators - -**Graph structure:** -``` -Input 1 → Application (once) → Evaluator 1 (repeat 1) - → Evaluator 1 (repeat 2) - → Evaluator 1 (repeat 3) - → Evaluator 2 (repeat 1) - → Evaluator 2 (repeat 2) - → Evaluator 2 (repeat 3) - -Input 2 → Application (once) → Evaluator 1 (repeat 1) - → Evaluator 1 (repeat 2) - ... -``` - -**Execution flow:** -```python -for testcase in testcases: - scenario = create_scenario(testcase=testcase) - - # Invoke application (ONCE per input) - outputs = invoke_application(testcase.inputs) - - # Invoke evaluators (REPEAT HERE) - for evaluator in evaluators: - for repeat_idx in range(repeats): - metrics = invoke_evaluator(testcase, outputs) - log_result( - scenario=scenario, - step_key=f"{evaluator.id}-r{repeat_idx}", - metrics=metrics, - ) -``` - -**Tensor structure:** -``` -Rows: Scenarios (Input only, no repeat dimension) -Cols: Steps (Testset, Applications, Evaluators × Repeats) - - Testset App-1 Eval-1-r1 Eval-1-r2 Eval-1-r3 Eval-2-r1 Eval-2-r2 Eval-2-r3 -Input-1 ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ -Input-2 ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ -Input-3 ✓ ✓ ✓ ✓ ✓ ✓ ✓ ✓ - -Each row = one scenario (input only) -Each column = one step (evaluators have repeat dimension) -``` - -**Use cases:** -- Testing human evaluator agreement: "Do 3 humans agree on quality score?" -- Testing LLM-as-judge consistency: "How stable is GPT-4 as an evaluator?" -- Ensemble evaluation: "Get multiple independent judgments, aggregate later" -- Inter-rater reliability: "Measure Cohen's kappa across evaluator repeats" - -**Cost implications:** -- Application invoked: `num_inputs` times (no repeat multiplier!) -- Evaluators invoked: `num_inputs × num_evaluators × repeats` times -- Example: 100 inputs, 3 repeats, 2 evaluators = 100 app calls, 600 evaluator calls - -**Key difference:** Much cheaper when applications are expensive (e.g., long-running workflows, API costs) - ---- - -##### Comparison Table - -| Aspect | `repeat_target = "application"` | `repeat_target = "evaluator"` | -|--------|----------------------------------|-------------------------------| -| **Fanout location** | Input → Application | Application → Evaluator | -| **What's tested** | Application variability | Evaluator variability | -| **Scenario dimension** | Input × Repeat | Input only | -| **Step dimension** | Evaluators (no repeat) | Evaluators × Repeat | -| **Tensor rows** | `num_inputs × repeats` | `num_inputs` | -| **Tensor cols** | `num_steps` | `num_steps × repeats` | -| **App invocations** | `inputs × repeats` | `inputs` | -| **Eval invocations** | `inputs × repeats × evals` | `inputs × evals × repeats` | -| **Typical use case** | LLM consistency, RAG variability | Human agreement, LLM-judge stability | - ---- - -##### Current Implementation Status - -**Status:** HARDCODED (exact mode needs investigation) - -**Likely location in code:** -- SDK: `sdk/agenta/sdk/evaluations/preview/evaluate.py` (check where `repeats` is used) -- API: May not be implemented yet - -**Questions to answer:** -1. ✅ Is `repeats` currently implemented? -2. ✅ If yes, which mode is hardcoded? (`application` or `evaluator`) -3. ✅ Where in the loop does the repeat happen? -4. ✅ How does it affect scenario creation? -5. ✅ How does it affect the tensor structure in practice? - -**Future requirement:** -- Make `repeat_target` an explicit flag in graph definition -- Support both modes -- Default to `application` for backward compatibility (if that's current behavior) -- Add validation: `repeat_target = "application"` requires application nodes in graph - ---- - -#### 3. `reuse_traces` (CRITICAL - Currently Implicit) -**Current state:** Hardcoded (location TBD in code) -**Current value:** Unknown (likely `false` - needs investigation) -**Type:** `boolean` - -**Purpose:** Avoid re-computing expensive workflows by reusing existing traces with matching inputs - -This is a **critical optimization** that enables: -- Cost savings (reuse expensive LLM calls, long-running workflows) -- Reproducibility (use same outputs across evaluations) -- Faster iteration (skip computation, reuse results) - ---- - -##### Trace Reuse Mechanism - -**1. Compute stable hash:** -```python -hash = compute_hash( - node_config=node.config, - inputs=inputs, - references=references, - links=links, -) -``` - -**Utility location:** `compute_hash()` (exists in codebase - needs verification) - -**Hash represents:** "What this computation means" - deterministic based on: -- Node configuration (workflow, parameters) -- Inputs (testcase data, upstream outputs) -- References (testset, application, evaluator IDs/versions) -- Links (upstream trace IDs, span IDs) - ---- - -**2. Hash-based trace lookup:** - -**Key property:** Hash is **NOT a primary key** (one-to-many relationship) -- Multiple traces can share the same hash (from different runs, repeats) -- Fetch traces by hash index -- Filter out failed traces (with exceptions/errors) - -**Lookup logic:** -```python -if reuse_traces: - hash = compute_hash(node, inputs, references, links) - - # Fetch all traces matching this hash - existing_traces = fetch_traces_by_hash( - hash=hash, - exclude_errors=True, # Only successful traces - ) - - if existing_traces: - # Reuse existing trace - trace = select_trace(existing_traces) - return trace - else: - # No existing trace, compute fresh - trace = invoke_workflow(node, inputs) - return trace -else: - # Always compute fresh (ignore hash) - trace = invoke_workflow(node, inputs) - return trace -``` - ---- - -##### Critical Interaction with `repeat_target` - -The `reuse_traces` flag behaves differently depending on `repeat_target`: - ---- - -###### Case 1: `repeat_target = "application"` + `reuse_traces = true` - -**Constraint:** **CANNOT reuse same trace across repeats** within the same evaluation - -**Reason:** Testing application variability requires different outputs for each repeat - -**Behavior:** -```python -for testcase in testcases: - hash = compute_hash(testcase, application_config, ...) - existing_traces = fetch_traces_by_hash(hash, exclude_errors=True) - - for repeat_idx in range(repeats): - if existing_traces and len(existing_traces) > repeat_idx: - # Reuse DIFFERENT trace for each repeat - trace = existing_traces[repeat_idx] - else: - # Need to compute fresh trace - trace = invoke_application(testcase.inputs) - - # Create scenario with this trace - scenario = create_scenario( - testcase=testcase, - trace_id=trace.id, - repeat_idx=repeat_idx, - ) -``` - -**Requirements:** -- Fetch **multiple different traces** from hash index (one per repeat) -- If not enough traces cached, compute additional ones -- Each repeat gets a different trace (even if inputs are identical) - -**Example:** -``` -Evaluation run 1 (reuse_traces=false): - Input-1, repeat-1: Compute trace_a - Input-1, repeat-2: Compute trace_b - Input-1, repeat-3: Compute trace_c - → All 3 traces stored with same hash - -Evaluation run 2 (reuse_traces=true): - Input-1, repeat-1: Reuse trace_a (from hash) - Input-1, repeat-2: Reuse trace_b (from hash) - Input-1, repeat-3: Reuse trace_c (from hash) - → No computation needed! -``` - ---- - -###### Case 2: `repeat_target = "evaluator"` + `reuse_traces = true` - -**Constraint:** **MUST reuse same trace across evaluator repeats** within the same evaluation - -**Reason:** Testing evaluator variability requires the same app output for all repeats - -**Behavior:** -```python -for testcase in testcases: - hash = compute_hash(testcase, application_config, ...) - existing_traces = fetch_traces_by_hash(hash, exclude_errors=True) - - if existing_traces: - # Reuse SAME trace for all evaluator repeats - trace = existing_traces[0] # Pick any one - else: - # Compute fresh trace - trace = invoke_application(testcase.inputs) - - # Create single scenario (no repeat dimension) - scenario = create_scenario( - testcase=testcase, - trace_id=trace.id, - ) - - # All evaluator repeats use this same trace - for evaluator in evaluators: - for repeat_idx in range(repeats): - metrics = invoke_evaluator( - testcase=testcase, - outputs=trace.outputs, # Same for all repeats - ) -``` - -**Requirements:** -- Fetch **single trace** from hash index -- All evaluator repeats use this same trace -- If multiple traces available (from previous runs), pick any one - -**Example:** -``` -Evaluation run 1 (reuse_traces=false): - Input-1: Compute trace_a - Evaluator-1, repeat-1: Evaluate trace_a - Evaluator-1, repeat-2: Evaluate trace_a (same!) - Evaluator-1, repeat-3: Evaluate trace_a (same!) - → Only 1 trace computed, evaluated 3 times - -Evaluation run 2 (reuse_traces=true): - Input-1: Reuse trace_a (from hash) - Evaluator-1, repeat-1: Evaluate trace_a (same!) - Evaluator-1, repeat-2: Evaluate trace_a (same!) - Evaluator-1, repeat-3: Evaluate trace_a (same!) - → No computation needed! -``` - ---- - -##### Comparison Table - -| Aspect | `repeat_target = "application"` | `repeat_target = "evaluator"` | -|--------|----------------------------------|-------------------------------| -| **Trace reuse constraint** | Different trace per repeat | Same trace across repeats | -| **Traces fetched per input** | `repeats` traces | `1` trace | -| **Trace selection** | `existing_traces[repeat_idx]` | `existing_traces[0]` (any) | -| **Variability source** | Application (different traces) | Evaluator (same trace) | -| **Scenario creation** | One per repeat (with trace_id) | One per input (reused trace_id) | - ---- - -##### Scenario Creation with Trace Reuse - -Even when reusing traces, scenarios are still created: - -**Case 1: `repeat_target = "application"`** -```python -# Multiple scenarios (one per repeat), each with different trace -scenario_1 = Scenario( - testcase_id=testcase.id, - trace_id=trace_a.id, # Reused or fresh - repeat_idx=0, -) - -scenario_2 = Scenario( - testcase_id=testcase.id, - trace_id=trace_b.id, # Different trace - repeat_idx=1, -) - -scenario_3 = Scenario( - testcase_id=testcase.id, - trace_id=trace_c.id, # Different trace - repeat_idx=2, -) -``` - -**Case 2: `repeat_target = "evaluator"`** -```python -# Single scenario, same trace reused across evaluator repeats -scenario = Scenario( - testcase_id=testcase.id, - trace_id=trace_a.id, # Reused or fresh, but same for all evals -) - -# Evaluators reference this same scenario/trace -result_1 = EvaluationResult( - scenario_id=scenario.id, - step_key="evaluator-1-r0", - trace_id=evaluator_trace_1.id, -) - -result_2 = EvaluationResult( - scenario_id=scenario.id, # Same scenario! - step_key="evaluator-1-r1", - trace_id=evaluator_trace_2.id, -) -``` - ---- - -##### Filtering and Selection - -**Filter failed traces:** -```python -existing_traces = fetch_traces_by_hash( - hash=hash, - exclude_errors=True, # Only successful traces -) -``` - -**Selection when multiple available:** -- Order doesn't matter (everything is stochastic) -- Can pick traces in any order (e.g., by creation time, random) -- As long as different traces are used for different application repeats - -**Example:** -```python -# Fetch traces by hash -traces = [trace_a, trace_b, trace_c, trace_d] # All have same hash - -# Application repeats: Use different traces -repeat_0_trace = traces[0] # trace_a -repeat_1_trace = traces[1] # trace_b -repeat_2_trace = traces[2] # trace_c - -# Evaluator repeats: Use same trace -all_repeats_trace = traces[0] # trace_a (any one is fine) -``` - ---- - -##### Use Cases - -**With `repeat_target = "application"`:** -- Caching expensive app runs: "Run GPT-4 once, reuse across evaluations" -- A/B testing with fixed outputs: "Test different evaluators on same app outputs" -- Debugging: "Fix evaluator bug, re-evaluate without re-running app" - -**With `repeat_target = "evaluator"`:** -- Measuring human evaluator agreement: "Same output judged by multiple humans" -- LLM-as-judge stability: "Same output scored by LLM multiple times" -- Cost optimization: "Run app once, test evaluator variability cheaply" - ---- - -##### Current Implementation Status - -**Status:** Unknown - needs investigation - -**Likely locations in code:** -- Hash computation: Utility function `compute_hash()` or similar -- Trace lookup: Tracing service or DAO with hash index -- SDK: May not be implemented yet -- API: May exist in some form - -**Questions to answer:** -1. ✅ Is trace reuse currently implemented? -2. ✅ Where is `compute_hash()` located? -3. ✅ Is there a hash index on traces table? -4. ✅ How are traces fetched by hash? -5. ✅ Does it filter out failed traces? -6. ✅ How does it interact with repeats? - -**Future requirements:** -- Make `reuse_traces` an explicit flag in graph definition -- Implement hash-based trace lookup if not present -- Add validation: `repeat_target = "application"` + `reuse_traces = true` requires enough cached traces -- Document hash computation algorithm - ---- - -#### 4. `is_closed` (State Management - Likely Implemented) -**Current state:** Likely implemented -**Current value:** `false` by default, set to `true` when evaluation is finalized -**Type:** `boolean` - -**Purpose:** Controls whether the evaluation graph and tensor are editable - -**Behavior:** - -##### When `is_closed = false` (Open - Default) -- ✅ Can modify graph structure -- ✅ Can add/edit/delete scenarios -- ✅ Can add/edit/delete results -- ✅ Can re-run evaluation -- ✅ Can edit metadata (name, description) - -##### When `is_closed = true` (Closed - Finalized) -- ❌ **Cannot** modify graph structure -- ❌ **Cannot** add/edit/delete scenarios -- ❌ **Cannot** add/edit/delete results -- ❌ **Cannot** re-run evaluation (execution blocked) -- ✅ **Can still** edit metadata (name, description) -- ✅ **Can** reopen (with appropriate permissions/operations) - -**Enforcement:** -- **DAO level:** Database operations fail for closed evaluations -- **API level:** Returns error (e.g., 409 Conflict, 403 Forbidden) -- **UI level:** Disable edit buttons, show read-only mode - -**Reopening:** -```python -# Some operations can reopen a closed evaluation -await evaluations_dao.update_run( - run_id=run_id, - is_closed=False, # Reopen -) -``` - -**Use cases:** -- **Finalize results:** Prevent accidental modification after review -- **Archival:** Mark completed evaluations as read-only -- **Compliance:** Lock evaluations for audit trail -- **Versioning:** Freeze evaluation state at specific point in time - -**Metadata operations (allowed when closed):** -- Update run name -- Update run description -- Add tags/labels -- Change visibility/permissions - -**Status:** Likely implemented (needs verification) - ---- - -## Summary: Confirmed Flags - -| Flag | Type | Purpose | Status | -|------|------|---------|--------| -| `is_live` | `boolean` | Online (periodic) vs offline (one-time) execution | ✅ Explicit | -| `is_active` | `boolean` | Pause/resume online evaluations | ✅ Explicit (companion to `is_live`) | -| `repeat_target` | `"application" \| "evaluator"` | Where repeats fan out (critical for tensor structure) | ⚠️ Currently implicit | -| `reuse_traces` | `boolean` | Cost optimization via trace caching | ⚠️ Currently implicit | -| `is_closed` | `boolean` | Lock evaluation to prevent edits | ✅ Explicit (state management) | - ---- - -## Future Considerations (Not Confirmed as Flags) - -The following behaviors may need to be configurable in the future, but are not currently discussed as flags: - -### Execution Behavior -- **Concurrency:** Sequential vs parallel scenario execution -- **Error handling:** Fail fast vs collect all errors -- **Application invocation:** Whether to invoke apps (currently determined by graph structure) - -### Metrics Computation -- **Timing:** Immediate vs deferred vs on-demand -- **Scope:** Per-scenario vs per-run -- **Current behavior:** SDK computes immediately, API defers to separate task - -These should be documented separately if/when they become explicit configuration options. - ---- - -## Data Source Types - -### 1. Testset Data Source - -**Structure:** -```python -class TestsetDataSource: - type: Literal["testset"] - testset_id: UUID - testset_revision_id: UUID - testset_variant_id: Optional[UUID] -``` - -**Resolution:** -```python -async def resolve_testset_scenarios( - data_source: TestsetDataSource, -) -> list[Scenario]: - """ - Load test cases from testset revision. - - Returns one scenario per test case. - """ - testset_revision = await testsets_dao.get_revision( - testset_revision_id=data_source.testset_revision_id, - ) - - scenarios = [] - for testcase in testset_revision.testcases: - scenarios.append( - Scenario( - data_source_type="testset", - testcase=testcase, - ) - ) - - return scenarios -``` - -**Properties:** -- **Static:** Same test cases every execution -- **Bounded:** Known size (number of test cases) -- **Versioned:** Tied to specific revision -- **Portable:** Can be shared across projects - -**Compatible with:** -- ✅ Offline evaluation -- ❌ Online evaluation - ---- - -### 2. Query Data Source - -**Structure:** -```python -class QueryDataSource: - type: Literal["query"] - query_spec: QuerySpec - time_window: Optional[TimeWindow] - -class QuerySpec: - filters: dict[str, Any] - limit: Optional[int] - order_by: Optional[str] - -class TimeWindow: - start_time: datetime - end_time: datetime -``` - -**Resolution (Offline):** -```python -async def resolve_query_scenarios_offline( - data_source: QueryDataSource, -) -> list[Scenario]: - """ - Execute query once, return snapshot of traces. - - Returns one scenario per trace. - """ - traces = await tracing_service.query_traces( - filters=data_source.query_spec.filters, - start_time=data_source.time_window.start_time, - end_time=data_source.time_window.end_time, - limit=data_source.query_spec.limit, - ) - - scenarios = [] - for trace in traces: - scenarios.append( - Scenario( - data_source_type="query", - trace=trace, - ) - ) - - return scenarios -``` - -**Resolution (Online):** -```python -async def resolve_query_scenarios_online( - data_source: QueryDataSource, - interval_start: datetime, - interval_end: datetime, -) -> list[Scenario]: - """ - Execute query for current interval window. - - Overwrites data_source.time_window with interval bounds. - Returns one scenario per trace. - """ - traces = await tracing_service.query_traces( - filters=data_source.query_spec.filters, - start_time=interval_start, # Overwritten! - end_time=interval_end, # Overwritten! - limit=data_source.query_spec.limit, - ) - - scenarios = [] - for trace in traces: - scenarios.append( - Scenario( - data_source_type="query", - trace=trace, - timestamp=interval_start, # Online-specific - interval=f"{interval_end - interval_start}", # Online-specific - ) - ) - - return scenarios -``` - -**Properties:** -- **Dynamic:** Different traces every execution (if time window changes) -- **Unbounded:** Size depends on query results (use `limit` for safety) -- **Temporal:** Can be windowed by time -- **Live:** Reflects current system state - -**Compatible with:** -- ✅ Offline evaluation (snapshot) -- ✅ Online evaluation (windowed) - ---- - -## Tensor Structure - -The evaluation tensor is the data structure that holds all results produced by running a graph. It is a 3D table indexed by **scenarios × steps × repeats**, stored as five related entities. - ---- - -### Five Entities - -``` -EvaluationRun -└── EvaluationScenario (one per row) - └── step_key (column label — string, not an entity) - └── repeat_idx (depth label — integer, not an entity) - └── EvaluationResult (the cell) -``` - -| Entity | Nature | Description | -|--------|--------|-------------| -| `EvaluationRun` | Entity (DB row) | The evaluation job. Owns the graph definition and all flag values. | -| `EvaluationScenario` | Entity (DB row) | One row in the tensor. Represents one input × (optionally) one application repeat. | -| `step_key` | String label | Column label — identifies which graph step produced the result (e.g. `"app-123"`, `"eval-456"`, `"eval-456-r2"`). | -| `repeat_idx` | Integer label | Depth label — which evaluator repeat this result belongs to (0-based). | -| `EvaluationResult` | Entity (DB row) | The cell. Identified by `(scenario_id, step_key, repeat_idx)`. | - -**There is no dedicated "Step" or "Repeat" entity** — `step_key` and `repeat_idx` are fields on the result row, forming a composite key together with `scenario_id`. - ---- - -### Tensor Dimensions - -#### Rows: Scenarios - -Each `EvaluationScenario` is one row. Its meaning depends on `repeat_target`: - -| `repeat_target` | Row = | -|-----------------|-------| -| `"application"` | One input × one application repeat (repeat dimension is in the row) | -| `"evaluator"` | One input (repeat dimension is in the column) | - -For online evaluations (`is_live = true`), each scenario also carries: -- `timestamp`: The start of the interval that produced it -- `interval`: The interval duration (e.g. `"1h"`) - -#### Columns: Steps - -Each distinct `step_key` is one logical column. Step keys are stable strings derived from the graph node identity: - -``` -"testset-{testset_id}" ← testset node result -"app-{application_revision_id}" ← application node result -"eval-{evaluator_revision_id}" ← evaluator node result (no repeat) -"eval-{evaluator_revision_id}-r{repeat_idx}" ← evaluator repeat (repeat_target="evaluator") -``` - -#### Depth: Repeats - -`repeat_idx` is the 0-based repeat index. Its role depends on `repeat_target`: - -| `repeat_target` | `repeat_idx` in result | -|-----------------|------------------------| -| `"application"` | Always `0` (repeats are rows, not depth) | -| `"evaluator"` | `0..N-1` across evaluator repeats | - -In practice, for `repeat_target = "application"`, the scenario itself carries the repeat index, and results within a scenario have `repeat_idx = 0`. - ---- - -### EvaluationResult: the Cell - -**Composite key:** `(scenario_id, step_key, repeat_idx)` - -There is no dedicated UUID primary key for a result — the three-field composite uniquely identifies the cell. - -#### Result Content: By Reference Only - -Results do **not** copy values inline. They store references: - -```python -class EvaluationResult: - # Identity - scenario_id: UUID - step_key: str - repeat_idx: int - - # References (no inline values) - testcase_id: Optional[UUID] # Reference to the testcase used as input - trace_id: Optional[UUID] # Reference to the execution trace (app or evaluator) - error: Optional[str] # Error message if the step failed - - # Status - status: EvaluationStatus # pending | running | completed | failed -``` - -**Key principle:** To read the actual inputs or outputs of a result, you follow the reference to the trace and read it there. The result row is only a pointer. - -**Why by reference:** -- Traces are already stored in the tracing system with full span detail -- Copying data into results would duplicate storage and create drift -- Metrics computation reads trace IDs from results and queries the tracing system directly - ---- - -### Tensor Population Flow - -The tensor is populated in three stages that correspond to the execution lifecycle: - -``` -1. Create EvaluationRun → Defines graph, flags, metadata - (tensor shape is implied by graph + inputs) - -2. Create EvaluationScenario → Adds a row - Happens as inputs are enumerated - (one per testcase for offline; one per trace for online) - -3. Create/Update EvaluationResult → Populates a cell - Happens as each step completes - (one per scenario × step × repeat) -``` - -**Example: Offline testset evaluation (2 testcases, 1 application, 2 evaluators, 1 repeat)** - -``` -Stage 1: Create run - run_id = "run-abc" - -Stage 2: Create scenarios (2 rows) - scenario_1 = Scenario(run_id="run-abc", testcase_id="tc-1") - scenario_2 = Scenario(run_id="run-abc", testcase_id="tc-2") - -Stage 3: Populate cells - # Application step - result(scenario_1, step_key="app-rev-1", repeat_idx=0) = { trace_id: "trace-a1" } - result(scenario_2, step_key="app-rev-1", repeat_idx=0) = { trace_id: "trace-a2" } - - # Evaluator steps - result(scenario_1, step_key="eval-rev-1", repeat_idx=0) = { trace_id: "trace-e1" } - result(scenario_1, step_key="eval-rev-2", repeat_idx=0) = { trace_id: "trace-e2" } - result(scenario_2, step_key="eval-rev-1", repeat_idx=0) = { trace_id: "trace-e3" } - result(scenario_2, step_key="eval-rev-2", repeat_idx=0) = { trace_id: "trace-e4" } -``` - -Resulting tensor: - -``` - app-rev-1 eval-rev-1 eval-rev-2 -scenario_1 trace-a1 trace-e1 trace-e2 -scenario_2 trace-a2 trace-e3 trace-e4 -``` - ---- - -### Metrics - -Metrics are **separate entities** from the tensor cells. They are computed after results are collected, by fetching trace IDs from results and running SQL aggregations over the traces. - -#### Metrics Computation - -``` -1. Fetch results for the run (or scenario) -2. Extract trace_ids from those results -3. Query the tracing system for those trace IDs -4. Run SQL aggregations over the trace data -5. Write metrics entity -``` - -This means metrics are always **derived, not stored inline**. They can be recomputed at any time by repeating steps 1–5. - -#### Three Metrics Types - -| Type | Scope | Compatible with | Description | -|------|-------|-----------------|-------------| -| **Global** | Entire run (all scenarios × all repeats) | `is_live = false` (offline only) | Aggregate statistics across the full evaluation | -| **Variational** | Per scenario (across all repeats for that scenario) | Both | Per-input statistics, showing variation across repeats | -| **Temporal** | Per interval / timestamp | `is_live = true` (online only) | Global statistics binned by time interval | - ---- - -#### Global Metrics (Offline Only) - -**Scope:** All scenarios × all repeats in the run. - -**Structure:** Per step → per output key → statistics. - -```python -class GlobalMetrics: - run_id: UUID - - # Nested: step → output_key → stats - steps: dict[str, dict[str, MetricStats]] - # step_key output_key - -class MetricStats: - # Numeric outputs - mean: Optional[float] - std: Optional[float] - min: Optional[float] - max: Optional[float] - p50: Optional[float] - p95: Optional[float] - count: int - - # Categorical / score outputs - distribution: Optional[dict[str, int]] # value → count -``` - -**Example:** -```python -GlobalMetrics( - run_id="run-abc", - steps={ - "eval-rev-1": { - "score": MetricStats(mean=0.72, std=0.15, p50=0.75, count=50), - "category": MetricStats(distribution={"pass": 38, "fail": 12}), - }, - "eval-rev-2": { - "latency_ms": MetricStats(mean=320, p95=850, count=50), - }, - } -) -``` - -**Not available for online evaluations** — there is no well-defined "total" population for a continuously-running evaluation; use Temporal metrics instead. - ---- - -#### Variational Metrics (Both Modes) - -**Scope:** Per scenario, across all repeats for that scenario. - -**Purpose:** Show how variable the results are for a given input. For `repeat_target = "application"` this reveals application stochasticity; for `repeat_target = "evaluator"` it reveals evaluator disagreement. - -**Structure:** Same nested structure as Global, but one entity per scenario. - -```python -class VariationalMetrics: - scenario_id: UUID - - # Nested: step → output_key → stats (across repeats for this scenario) - steps: dict[str, dict[str, MetricStats]] - # step_key output_key -``` - -**Example:** -```python -VariationalMetrics( - scenario_id="scenario-1", # Input: "What is 2+2?" - steps={ - "eval-rev-1": { - # 3 repeats scored this input as: 0.8, 0.7, 0.9 - "score": MetricStats(mean=0.8, std=0.082, min=0.7, max=0.9, count=3), - }, - } -) -``` - -**Available for both offline and online evaluations.** - ---- - -#### Temporal Metrics (Online Only) - -**Scope:** Per time interval (aggregated across all scenarios in that interval). - -**Purpose:** Show how quality changes over time in a live evaluation. - -**Structure:** Same as Global, but keyed by interval. - -```python -class TemporalMetrics: - run_id: UUID - interval_start: datetime - interval_end: datetime - - # Nested: step → output_key → stats (all scenarios in this interval) - steps: dict[str, dict[str, MetricStats]] - # step_key output_key -``` - -**Example:** -```python -TemporalMetrics( - run_id="run-abc", - interval_start=datetime(2026, 2, 17, 10, 0), - interval_end=datetime(2026, 2, 17, 11, 0), - steps={ - "eval-rev-1": { - "score": MetricStats(mean=0.68, count=42), - }, - } -) -``` - -**Not available for offline evaluations** — offline runs have no meaningful interval breakdown; use Global metrics instead. - ---- - -#### Metrics Compatibility Summary - -| Metrics Type | Offline (`is_live = false`) | Online (`is_live = true`) | -|--------------|-----------------------------|-----------------------------| -| **Global** | ✅ Primary aggregate view | ❌ Not applicable | -| **Variational** | ✅ Per-input variation | ✅ Per-input variation | -| **Temporal** | ❌ Not applicable | ✅ Primary time-series view | - ---- - -#### Metrics Entity Relationship - -``` -EvaluationRun -├── GlobalMetrics (1 per run, offline only) -├── TemporalMetrics (N per run, online only — one per interval) -└── EvaluationScenario (N per run) - ├── VariationalMetrics (1 per scenario) - └── EvaluationResult (M per scenario — one per step × repeat) - └── → trace_id (reference into tracing system) -``` - ---- - -## Current Graph Structures - -### Structure 1: Batch Testset Evaluation (SDK) - -**Current implementation:** `sdk/agenta/sdk/evaluations/preview/evaluate.py` - -**Graph:** -``` -Testset → Application(s) → Evaluator(s) -``` - -**Nodes:** -- 1 Testset node -- N Application nodes -- M Evaluator nodes - -**Execution:** -``` -For each testcase in testset: - scenario = create_scenario() - - For each application: - outputs = invoke_application(testcase.inputs) - log_result(scenario, application, outputs) - - For each evaluator: - metrics = invoke_evaluator(testcase, application.outputs) - log_result(scenario, evaluator, metrics) - - compute_metrics(scenario) -``` - -**Flags:** -- `is_live = false` (offline) -- `application_required = true` (implicit) -- `concurrent_execution = false` (implicit) - ---- - -### Structure 2: Batch Testset Evaluation (API Legacy) - -**Current implementation:** `api/oss/src/core/evaluations/tasks/legacy.py` - -**Graph:** -``` -Testset → [Applications already invoked] → Evaluator(s) -``` - -**Nodes:** -- 1 Testset node -- N Application nodes (invoked separately, before this task) -- M Evaluator nodes - -**Execution:** -``` -# Applications already invoked, results available as "invocations" - -For each testcase: - scenario = scenarios[idx] # Already created - invocation = invocations[idx] # Already executed - - If invocation has error: - skip evaluators - - trace = fetch_trace(invocation.trace_id) - - For each evaluator: - metrics = invoke_evaluator(testcase, invocation.outputs) - log_result(scenario, evaluator, metrics) - -# Metrics computed later via separate task -``` - -**Flags:** -- `is_live = false` (offline) -- `application_required = true` (but invoked separately) -- `concurrent_execution = false` (implicit) -- `metrics_computation_mode = "deferred"` (implicit) - ---- - -### Structure 3: Live Query Evaluation (API) - -**Current implementation:** `api/oss/src/core/evaluations/tasks/live.py` - -**Graph:** -``` -Query → Evaluator(s) -``` - -**Nodes:** -- 1 Query node -- M Evaluator nodes -- **No application nodes** (evaluates existing traces) - -**Execution:** -``` -For each interval (e.g., hourly): - # Overwrite query time window with current interval - traces = query_traces( - filters=query_spec.filters, - start_time=interval_start, # Overwritten - end_time=interval_end, # Overwritten - ) - - scenarios = create_scenarios( - nof_scenarios=len(traces), - timestamp=interval_start, - interval=interval_duration, - ) - - For each trace: - scenario = scenarios[idx] - - For each evaluator: - metrics = invoke_evaluator(trace, trace.outputs) - log_result(scenario, evaluator, metrics) - -# Metrics computed later via separate task -``` - -**Flags:** -- `is_live = true` (online) -- `application_required = false` (implicit - no apps invoked) -- `concurrent_execution = false` (implicit) -- `metrics_computation_mode = "deferred"` (implicit) - ---- - -## Future Graph Structures - -### Structure 4: Multi-Application Comparison (Future) - -**Use case:** Compare multiple application variants on the same testset - -**Graph:** -``` -Testset → Application A → Evaluator 1 - → Application B → Evaluator 1 - → Application C → Evaluator 1 -``` - -**Challenge:** -- Current implementation evaluates all evaluators against each application -- Need to support: All applications evaluated by the same evaluators - -**Desired execution:** -``` -For each testcase: - scenario = create_scenario() - - outputs_by_app = {} - For each application: - outputs = invoke_application(testcase.inputs) - outputs_by_app[application.id] = outputs - log_result(scenario, application, outputs) - - For each evaluator: - For each application: - metrics = invoke_evaluator(testcase, outputs_by_app[application.id]) - log_result(scenario, f"{evaluator.id}-{application.id}", metrics) -``` - -**New requirement:** -- Evaluators need access to ALL application outputs for comparison -- Graph edges must specify which app outputs go to which evaluators - ---- - -### Structure 5: Chained Applications (Future) - -**Use case:** Multi-step workflows (e.g., retrieval → generation → verification) - -**Graph:** -``` -Testset → Retrieval App → Generation App → Verification App → Evaluator -``` - -**Challenge:** -- Current implementation assumes applications are independent -- Need to support: Application B takes Application A's outputs as inputs - -**Desired execution:** -``` -For each testcase: - scenario = create_scenario() - - # Step 1: Retrieval - retrieval_outputs = invoke_application(retrieval_app, testcase.inputs) - - # Step 2: Generation (uses retrieval outputs) - generation_inputs = { - **testcase.inputs, - "context": retrieval_outputs["documents"], - } - generation_outputs = invoke_application(generation_app, generation_inputs) - - # Step 3: Verification (uses generation outputs) - verification_inputs = { - **testcase.inputs, - "generated_text": generation_outputs["text"], - } - verification_outputs = invoke_application(verification_app, verification_inputs) - - # Step 4: Evaluate final output - metrics = invoke_evaluator(testcase, verification_outputs) -``` - -**New requirements:** -- Graph edges must specify input/output mappings -- Topological sort to determine execution order -- Support for partial failures (if step 2 fails, skip step 3) - ---- - -### Structure 6: Conditional Evaluation (Future) - -**Use case:** Run different evaluators based on outputs - -**Graph:** -``` -Testset → Application → [if output.type == "chat"] → Chat Evaluator - → [if output.type == "completion"] → Completion Evaluator -``` - -**Challenge:** -- Current implementation runs all evaluators for all scenarios -- Need to support: Conditional evaluator invocation based on outputs - -**Desired execution:** -``` -For each testcase: - scenario = create_scenario() - - outputs = invoke_application(testcase.inputs) - - # Conditional evaluator selection - if outputs.get("type") == "chat": - evaluators = [chat_evaluator_1, chat_evaluator_2] - elif outputs.get("type") == "completion": - evaluators = [completion_evaluator_1] - else: - evaluators = [generic_evaluator] - - For each evaluator in evaluators: - metrics = invoke_evaluator(testcase, outputs) -``` - -**New requirements:** -- Graph edges can have conditions -- Evaluator selection based on runtime data -- Support for dynamic graph execution - ---- - -### Structure 7: Human-in-the-Loop Evaluation (Future) - -**Use case:** Mix automated and human evaluation - -**Graph:** -``` -Testset → Application → Auto Evaluator 1 - → Auto Evaluator 2 - → Human Evaluator (async) -``` - -**Challenge:** -- Current implementation expects synchronous evaluator responses -- Human evaluation is asynchronous (can take hours/days) - -**Desired execution:** -``` -For each testcase: - scenario = create_scenario() - - outputs = invoke_application(testcase.inputs) - - # Automated evaluators (synchronous) - For each auto_evaluator: - metrics = invoke_evaluator(testcase, outputs) - log_result(scenario, auto_evaluator, metrics) - - # Human evaluator (asynchronous) - task = create_human_evaluation_task(testcase, outputs) - log_result(scenario, human_evaluator, status="pending") - -# Later, when human completes evaluation: -update_result(scenario, human_evaluator, metrics=human_metrics) -``` - -**New requirements:** -- Support for async/pending results -- Ability to update results after initial execution -- Separate "automated complete" vs "fully complete" states - ---- - -### Structure 8: Iterative Refinement (Future) - -**Use case:** Evaluate, refine, re-evaluate in a loop - -**Graph:** -``` -Testset → Application v1 → Evaluator → [if score < threshold] → Application v2 → Evaluator -``` - -**Challenge:** -- Current implementation is single-pass -- Need to support: Conditional re-execution based on results - -**Desired execution:** -``` -For each testcase: - scenario = create_scenario() - - version = 1 - max_iterations = 3 - - While version <= max_iterations: - outputs = invoke_application(app[version], testcase.inputs) - metrics = invoke_evaluator(testcase, outputs) - - if metrics["score"] >= threshold: - break # Success! - - version += 1 - - log_final_result(scenario, version, metrics) -``` - -**New requirements:** -- Support for loops in graph -- Conditional loop exit -- Tracking iterations in results - ---- - -## Graph Representation - -### Current Representation (Implicit) - -**Location:** Embedded in execution logic, not explicit data structure - -**Example (SDK):** -```python -# Implicit graph in loop structure -for testset_revision in testsets: - for testcase in testcases: - for application in applications: - # Edge: testset → application - invoke_application(testcase.inputs) - - for evaluator in evaluators: - # Edge: application → evaluator - invoke_evaluator(application.outputs) -``` - -**Problems:** -- Graph structure is code, not data -- Can't serialize/visualize graph -- Hard to validate before execution -- No way to query "what will this evaluation do?" - ---- - -### Desired Representation (Explicit) - -**Goal:** Represent graph as data structure - -**Example:** -```python -class EvaluationGraph: - """Explicit graph representation.""" - - nodes: list[GraphNode] - edges: list[GraphEdge] - -class GraphNode: - id: str - type: Literal["testset", "query", "application", "evaluator"] - config: dict[str, Any] - -class GraphEdge: - from_node_id: str - to_node_id: str - port_mapping: dict[str, str] # Maps output ports to input ports - condition: Optional[str] # Optional condition (future) - -# Example graph for batch testset evaluation -graph = EvaluationGraph( - nodes=[ - GraphNode( - id="testset-1", - type="testset", - config={ - "testset_id": "ts-123", - "testset_revision_id": "rev-456", - } - ), - GraphNode( - id="app-1", - type="application", - config={ - "application_id": "app-789", - "application_revision_id": "rev-abc", - } - ), - GraphNode( - id="eval-1", - type="evaluator", - config={ - "evaluator_id": "eval-xyz", - "evaluator_revision_id": "rev-def", - } - ), - ], - edges=[ - GraphEdge( - from_node_id="testset-1", - to_node_id="app-1", - port_mapping={ - "testcase.data": "inputs", # testcase data → app inputs - } - ), - GraphEdge( - from_node_id="app-1", - to_node_id="eval-1", - port_mapping={ - "outputs": "outputs", # app outputs → eval outputs - "trace": "trace", # app trace → eval trace - } - ), - GraphEdge( - from_node_id="testset-1", - to_node_id="eval-1", - port_mapping={ - "testcase": "testcase", # testcase → eval testcase - } - ), - ], -) -``` - -**Benefits:** -- Serializable (can save/load) -- Validatable (check before execution) -- Visualizable (render as diagram) -- Queryable (analyze without executing) - ---- - -## Validation Rules - -### Rule 1: Data Source Compatibility -**Constraint:** `is_live = true` → Data source MUST be Query - -**Validation:** -```python -def validate_live_data_source(graph: EvaluationGraph, is_live: bool) -> None: - """Validate data source is compatible with is_live flag.""" - data_source_node = graph.get_data_source_node() - - if is_live and data_source_node.type == "testset": - raise ValidationError( - "Live evaluation (is_live=true) requires Query data source, " - "not Testset. Testsets yield the same data every interval." - ) -``` - ---- - -### Rule 2: Graph Connectivity -**Constraint:** All non-source nodes must be reachable from a source node - -**Validation:** -```python -def validate_graph_connectivity(graph: EvaluationGraph) -> None: - """Validate all nodes are reachable from data source.""" - source_nodes = [n for n in graph.nodes if n.type in ["testset", "query"]] - - if not source_nodes: - raise ValidationError("Graph must have at least one data source node") - - reachable = set() - to_visit = [n.id for n in source_nodes] - - while to_visit: - current = to_visit.pop() - if current in reachable: - continue - reachable.add(current) - - # Add downstream nodes - for edge in graph.edges: - if edge.from_node_id == current: - to_visit.append(edge.to_node_id) - - unreachable = set(n.id for n in graph.nodes) - reachable - - if unreachable: - raise ValidationError( - f"Nodes {unreachable} are not reachable from data source" - ) -``` - ---- - -### Rule 3: No Cycles (Current - may relax in future) -**Constraint:** Graph must be a DAG (Directed Acyclic Graph) - -**Validation:** -```python -def validate_no_cycles(graph: EvaluationGraph) -> None: - """Validate graph has no cycles.""" - visited = set() - in_path = set() - - def has_cycle(node_id: str) -> bool: - if node_id in in_path: - return True # Cycle detected - if node_id in visited: - return False - - visited.add(node_id) - in_path.add(node_id) - - for edge in graph.edges: - if edge.from_node_id == node_id: - if has_cycle(edge.to_node_id): - return True - - in_path.remove(node_id) - return False - - for node in graph.nodes: - if has_cycle(node.id): - raise ValidationError( - f"Graph contains a cycle involving node {node.id}" - ) -``` - -**Note:** This rule may be relaxed in the future to support iterative refinement (Structure 8) - ---- - -### Rule 4: Valid Port Mappings -**Constraint:** Edge port mappings must reference valid ports - -**Validation:** -```python -def validate_port_mappings(graph: EvaluationGraph) -> None: - """Validate edge port mappings reference valid ports.""" - - # Define valid ports for each node type - PORT_DEFINITIONS = { - "testset": { - "outputs": ["testcase", "testcase.data", "testcase.id"] - }, - "query": { - "outputs": ["trace", "trace.id", "trace.inputs", "trace.outputs"] - }, - "application": { - "inputs": ["inputs"], - "outputs": ["outputs", "trace", "trace_id", "span_id"] - }, - "evaluator": { - "inputs": ["testcase", "inputs", "outputs", "trace"], - "outputs": ["metrics", "trace", "trace_id"] - }, - } - - for edge in graph.edges: - from_node = graph.get_node(edge.from_node_id) - to_node = graph.get_node(edge.to_node_id) - - from_ports = PORT_DEFINITIONS[from_node.type]["outputs"] - to_ports = PORT_DEFINITIONS[to_node.type]["inputs"] - - for from_port, to_port in edge.port_mapping.items(): - if from_port not in from_ports: - raise ValidationError( - f"Node {from_node.id} ({from_node.type}) " - f"has no output port '{from_port}'" - ) - - if to_port not in to_ports: - raise ValidationError( - f"Node {to_node.id} ({to_node.type}) " - f"has no input port '{to_port}'" - ) -``` - ---- - -## Summary - -### Current State -- **3 graph structures** currently supported (testset batch, testset batch API, query live) -- **1 explicit flag** (`is_live`) -- **5 implicit flags** (hardcoded behaviors) -- **Implicit graph** representation (embedded in code) -- **Tensor model:** 5 entities — EvaluationRun → EvaluationScenario → EvaluationResult (keyed by `step_key`, `repeat_idx`) -- **Results:** By reference only (testcase_id, trace_id, error — no inline values) -- **Metrics:** 3 types — Global (offline), Variational (both), Temporal (online); computed via SQL over trace IDs from results - -### Key Constraints -- **Live evaluation requires Query data source** (testsets yield same data) -- **Graphs must be DAGs** (no cycles, for now) -- **All nodes must be reachable** from data source - -### Future Evolution -- **Explicit graph representation** (nodes + edges as data) -- **More complex structures** (multi-app comparison, chained apps, conditionals) -- **Additional flags** (concurrency, error handling, metrics mode) -- **Relaxed constraints** (cycles for iterative refinement, async evaluators) - -### Next Steps -1. Define explicit graph data structure -2. Implement graph validation rules -3. Support serialization/deserialization -4. Add graph visualization -5. Extend to support future structures - ---- - -**Document Status:** Draft - covers graph structure, confirmed flags, tensor model, and metrics types -**Next Action:** Review flag taxonomy, tensor entity schemas, and metrics computation with team diff --git a/docs/designs/eval-loops/gap-analysis.md b/docs/designs/eval-loops/gap-analysis.md deleted file mode 100644 index 909418b2b7..0000000000 --- a/docs/designs/eval-loops/gap-analysis.md +++ /dev/null @@ -1,267 +0,0 @@ -# Evaluation Gap Analysis - -**Created:** 2026-02-17 -**Purpose:** Map the current codebase (SDK, backend API, frontend) to the desired operation model, identifying what exists, what is named differently, and what is missing -**Related:** -- [Evaluation Structure](./evaluation-structure.md) -- [Evaluation Operations](./evaluation-operations.md) -- [Desired Architecture](./desired-architecture.md) -- [Refactoring Analysis](./refactoring-analysis.md) - ---- - -## Table of Contents - -- [Document Coherence Notes](#document-coherence-notes) -- [Terminology Mapping](#terminology-mapping) -- [Current Implementation by Layer](#current-implementation-by-layer) - - [SDK](#sdk) - - [Backend API](#backend-api) - - [Frontend](#frontend) -- [Operation Model Coverage](#operation-model-coverage) -- [Flag Coverage](#flag-coverage) -- [Structural Gaps](#structural-gaps) - ---- - -## Document Coherence Notes - -The five design documents are broadly coherent. Minor inconsistencies to be aware of: - -| Document | Note | -|----------|------| -| `iteration-patterns.md` | Uses "applications/evaluators" terminology; predates the `type: input/invocation/annotation` vocabulary from `evaluation-operations.md` | -| `desired-architecture.md` | Uses "ports & adapters / RemoteAPIPersistence" framing; SDK analysis shows no such abstraction exists yet — it is all direct HTTP | -| `refactoring-analysis.md` | Identifies 4-level SDK loop vs 2/3-level API loops — still accurate; the `TensorSlice` / `process` vocabulary in `evaluation-operations.md` is the intended reconciliation | -| `evaluation-structure.md` | Tensor entity model (composite key, by-reference results) is consistent with the DB schema (`unique on (project_id, run_id, scenario_id, step_key, repeat_idx)`) | -| `evaluation-operations.md` | `repeat_target`, `reuse_traces`, `allow_decrease_repeats` flags are **not yet in the codebase** — documented as desired state | - ---- - -## Terminology Mapping - -| Desired term | SDK term | Backend term | Frontend term | -|--------------|----------|--------------|---------------| -| `add_step` | Implicit in `acreate_run()` payload | Built inside `setup_evaluation()` | Implicit in creation wizard (evaluator + testset selection) | -| `remove_step` | — | — | — | -| `add_scenario` | `aadd_scenario()` | `create_scenario()` / `POST /scenarios/` | — (backend-driven) | -| `remove_scenario` | — | `delete_scenario()` / `DELETE /scenarios/{id}` | — | -| `populate` | `alog_result()` | `create_result()` / `POST /results/` | `upsertStepResultWithAnnotation()` | -| `prune` | — | `delete_result[s]()` / `DELETE /results/` | — | -| `probe` | — | `query_results()` / `POST /results/query` | `queryStepResults()` | -| `process` | `aevaluate()` (the whole loop) | `SimpleEvaluationsService.start()` → Taskiq worker | "Run" button → triggers backend | -| `refresh_metrics` | `acompute_metrics()` | `refresh_metrics()` / `POST /metrics/refresh` | — (backend-driven) | -| `get_flags` | — | `GET /runs/{run_id}` (returns full run) | — | -| `set_flag` | — | `PATCH /runs/{run_id}` with `flags` in body | Implicit (is_live via online eval creation) | -| `TensorSlice` | Not implemented | Not implemented | Not implemented | -| `step.type = "input"` | Implicit (testset data) | `type: "input"` in `EvaluationRunDataStep` | `type: "input"` in step meta | -| `step.type = "invocation"` | `"application-{slug}"` step_key | `type: "invocation"` | `kind: "invocation"` | -| `step.type = "annotation"` | `"evaluator-{slug}"` step_key | `type: "annotation"` | `kind: "annotation"` | -| `step.origin = "auto"` | Origin enum exists in models | `origin: "auto"` in step | `origin: "automated"` ⚠️ (inconsistent spelling) | -| `step.origin = "human"` | Origin enum exists in models | `origin: "human"` | `origin: "human"` | -| `step.origin = "custom"` | Origin enum exists in models | `origin: "custom"` | Not explicitly exposed | - -**Note:** Frontend uses `"automated"` where backend/SDK use `"auto"`. This is a naming inconsistency to fix. - ---- - -## Current Implementation by Layer - -### SDK - -**File:** `sdk/agenta/sdk/evaluations/preview/evaluate.py` - -The SDK implements the `process` role in a single `aevaluate()` function. All operations are direct HTTP calls to the backend — no abstraction layer. - -#### What exists - -| Operation | SDK function | Endpoint | -|-----------|-------------|---------| -| Create run | `acreate_run()` | `POST /preview/simple/evaluations/` | -| Add scenario | `aadd_scenario()` | `POST /preview/evaluations/scenarios/` | -| Populate (log result) | `alog_result()` | `POST /preview/evaluations/results/` | -| Refresh metrics | `acompute_metrics()` | `POST /preview/evaluations/metrics/refresh` | -| Close run | `aclose_run()` | `POST /preview/evaluations/runs/{id}/close/{status}` | - -#### What is missing / gaps - -- **No `probe`** — SDK does not query existing results before writing. It always writes (no idempotency check / skip-if-success). -- **No `prune`** — SDK cannot delete results or scenarios. -- **No `TensorSlice`** — SDK always processes the full tensor (all scenarios, all steps, all repeats). -- **No `process(slice)` API** — `aevaluate()` is monolithic; there is no way to re-run a subset. -- **No ports & adapters** — `desired-architecture.md` describes `RemoteAPIPersistence`; the SDK currently has direct HTTP with no abstraction boundary. -- **No `repeat_target` or `reuse_traces`** — repeats not implemented in SDK loop. -- **Step keys use short slugs** (`"application-{uuid[:8]}"`) rather than full revision IDs — may cause collisions. -- **4-level nesting** (testset → testcase → application → evaluator) is tightly coupled; cannot be decomposed into slice-based execution. - -#### Desired SDK state - -The SDK loop becomes canonical `process(slice)`, with: -- Injected `RemoteAPIPersistence` implementing `populate`, `prune`, `probe` -- `probe` to skip already-successful cells -- `TensorSlice` to target re-runs or partial execution - ---- - -### Backend API - -**Files:** -- `api/oss/src/core/evaluations/service.py` -- `api/oss/src/core/evaluations/tasks/legacy.py` -- `api/oss/src/core/evaluations/tasks/live.py` -- `api/oss/src/apis/fastapi/evaluations/router.py` -- `api/oss/src/tasks/taskiq/evaluations/worker.py` - -#### What exists - -| Operation | Backend function | HTTP endpoint | -|-----------|----------------|--------------| -| Create run | `create_run()` | `POST /runs/` | -| Add scenario | `create_scenario[s]()` | `POST /scenarios/` | -| Remove scenario | `delete_scenario()` | `DELETE /scenarios/{id}` | -| Populate | `create_result[s]()` | `POST /results/` | -| Probe | `query_results()` | `POST /results/query` | -| Prune | `delete_result[s]()` | `DELETE /results/` | -| Refresh metrics | `refresh_metrics()` | `POST /metrics/refresh` | -| Close run | `close_run()` | `POST /runs/{id}/close` | -| Open run | `open_run()` | `POST /runs/{id}/open` | -| Start processing | `start()` → Taskiq worker | (internal dispatch, no HTTP endpoint) | -| Stop processing | `stop()` | (internal, no HTTP endpoint) | - -**DB unique constraint:** `(project_id, run_id, scenario_id, step_key, repeat_idx)` — matches the composite key in our tensor model. ✅ - -#### What is missing / gaps - -**Steps (critical gap):** -- **No `add_step` endpoint.** Steps are constructed inside `setup_evaluation()` (legacy.py) — the graph is assembled as a single blob and stored. There is no API to add or remove steps after creation. -- **No `remove_step` endpoint.** -- Steps are in `run.data.steps` (a list in a JSON column), not a normalized table. This makes adding/removing expensive (read-modify-write the whole blob). - -**Processing control:** -- **No `POST /runs/{id}/process` endpoint.** Processing is triggered by `start()` internally, which dispatches a Taskiq task. There is no slice parameter. -- **No slice-aware re-run.** `evaluate_batch_testset` and `evaluate_live_query` tasks always re-process the full run. - -**Flags:** -- **`repeat_target` not in `EvaluationRunFlags`.** The current flags model has `is_live`, `is_active`, `is_closed`, `has_queries`, `has_testsets`, `has_evaluators`, `has_custom`, `has_human`, `has_auto`. Missing: `repeat_target`, `reuse_traces`, `allow_decrease_repeats`. -- **No dedicated `set_flag` endpoint.** Flags are set via `PATCH /runs/{id}` with the whole flags object. The desired `set_flag(name, value)` (read-modify-write) is not yet a first-class operation. - -**TensorSlice:** -- **Not implemented anywhere.** All CRUD operations use individual IDs or full-table queries. No slice-based multi-dimension targeting. - -**`process` dispatch:** -- `SimpleEvaluationsService.start()` dispatches either `evaluate_batch_testset` or `evaluate_live_query` based on whether the run has `query_steps` or `testset_steps`. This is the current equivalent of `process`, but: - - It's hardcoded logic in the service, not a general slice executor - - No HTTP endpoint to trigger it externally (used via internal dispatch only) - - No idempotency (no `probe`-before-write to skip successful cells) - ---- - -### Frontend - -**Key files:** -- `web/oss/src/services/evaluations/api/index.ts` — core evaluation CRUD -- `web/oss/src/services/evaluations/results/api.ts` — step results + annotation linkage -- `web/oss/src/services/onlineEvaluations/api.ts` — live evaluation + queries -- `web/oss/src/services/annotations/api/index.ts` — annotation CRUD -- `web/oss/src/components/pages/evaluations/NewEvaluation/` — creation wizard -- `web/oss/src/components/EvalRunDetails/` — legacy results display - -#### What exists - -| Feature | Status | Notes | -|---------|--------|-------| -| Step `type` tracking | ✅ | `type: "invocation" \| "annotation" \| "input"` in step meta | -| Step `origin` tracking | ⚠️ | Uses `"automated"` instead of `"auto"` | -| Human annotation workflow | ✅ | Dedicated Annotate drawer; `upsertStepResultWithAnnotation()` | -| Populate (frontend) | ✅ | `upsertStepResultWithAnnotation()` → `POST /preview/evaluations/results/` | -| Probe (frontend) | ✅ | `queryStepResults()` → `POST /results/query` | -| Live evaluations | ✅ | Online eval drawer with `is_live` flag, query definition, sampling rate | -| Result display | ✅ | Metrics table, spider chart, temporal chart, scenario focus drawer | -| Evaluation creation | ✅ | 5-step wizard: app → variant → testset → evaluators → settings | - -#### What is missing / gaps - -- **No graph builder.** The graph (which steps, in what order) is assembled implicitly by the creation wizard (testset + evaluators = the graph). There is no DAG visualization or explicit step management UI. -- **`origin = "automated"` vs `"auto"`.** The frontend uses `"automated"` in step metadata filtering (`ActionCell.tsx`) while backend uses `"auto"`. This must be aligned. -- **No TensorSlice UI.** No way to select a subset of scenarios/steps/repeats to re-run or prune. -- **No `prune` operation in UI.** Users cannot delete specific results or scenarios. -- **No flag management UI** (beyond implicit toggles like start/stop for `is_active`, close for `is_closed`). -- **No `repeat_target`, `reuse_traces` in UI.** These flags don't exist in the codebase yet. - ---- - -## Operation Model Coverage - -Summary of coverage across all three layers: - -| Operation | SDK | Backend | Frontend | TensorSlice-aware | -|-----------|-----|---------|----------|-------------------| -| `add_step` | ⚠️ Implicit at creation | ❌ No endpoint | ⚠️ Implicit in wizard | — | -| `remove_step` | ❌ | ❌ | ❌ | — | -| `add_scenario` | ✅ `aadd_scenario` | ✅ `POST /scenarios/` | — | ❌ | -| `remove_scenario` | ❌ | ✅ `DELETE /scenarios/{id}` | — | ❌ | -| `increase_repeats` | ❌ | ❌ | ❌ | — | -| `decrease_repeats` | ❌ | ❌ | ❌ | — | -| `populate` | ✅ `alog_result` | ✅ `POST /results/` | ✅ `upsertStepResult...` | ❌ | -| `prune` | ❌ | ✅ `DELETE /results/` | ❌ | ❌ | -| `probe` | ❌ | ✅ `POST /results/query` | ✅ `queryStepResults` | ❌ | -| `refresh_metrics` | ✅ `acompute_metrics` | ✅ `POST /metrics/refresh` | — | — | -| `get_flags` | ❌ | ⚠️ Via `GET /runs/{id}` | — | — | -| `set_flag` | ❌ | ⚠️ Via `PATCH /runs/{id}` | ⚠️ Implicit only | — | -| `process` | ✅ `aevaluate()` (monolithic) | ⚠️ Internal only (Taskiq) | ⚠️ "Run" button triggers it | ❌ | - -Legend: ✅ Exists · ⚠️ Partial/implicit · ❌ Missing - ---- - -## Flag Coverage - -| Flag | In `EvaluationRunFlags` | Set by whom | Gaps | -|------|------------------------|-------------|------| -| `is_live` | ✅ | Frontend (online eval creation) | — | -| `is_active` | ✅ | Backend (`start()` / `stop()`) | No HTTP endpoint to toggle directly | -| `is_closed` | ✅ | Backend (`close_run()` / `open_run()`) | — | -| `has_queries` | ✅ | Backend (derived at creation) | Derived, not user-set | -| `has_testsets` | ✅ | Backend (derived at creation) | Derived, not user-set | -| `has_evaluators` | ✅ | Backend (derived at creation) | Derived, not user-set | -| `has_custom` | ✅ | Backend (derived at creation) | Derived, not user-set | -| `has_human` | ✅ | Backend (derived at creation) | Derived, not user-set | -| `has_auto` | ✅ | Backend (derived at creation) | Derived, not user-set | -| `repeat_target` | ❌ | — | Not implemented anywhere | -| `reuse_traces` | ❌ | — | Not implemented anywhere | -| `allow_decrease_repeats` | ❌ | — | Not implemented anywhere | - ---- - -## Structural Gaps - -### Priority 1 — Critical for the operation model - -1. **`add_step` / `remove_step` endpoints.** Steps need to be first-class entities with their own lifecycle, not a blob inside `run.data`. This is the foundation for the graph mutation model. - -2. **`repeat_target`, `reuse_traces`, `allow_decrease_repeats` flags.** Not in the data model at all. Needed before implementing repeat and trace-reuse behaviors. - -3. **`increase_repeats` / `decrease_repeats`.** No concept of repeat count as a mutable run property. Currently hardcoded per-run at creation. - -### Priority 2 — Important for operational correctness - -4. **`process(slice)` HTTP endpoint.** Processing is currently internal-only (Taskiq dispatch). To support re-runs of failed/missing results, process needs to be externally triggerable with a slice parameter. - -5. **`TensorSlice` in `probe` / `prune` / `populate`.** Current operations are per-entity (single ID) or full-table. Slice-based multi-dimensional targeting is not implemented. - -6. **Idempotent `process` (probe-before-write).** SDK and backend tasks always write results without checking if a successful result already exists. This means re-running overwrites correct data. - -### Priority 3 — Naming and consistency - -7. **Frontend `"automated"` → `"auto"`.** One-line fix in `ActionCell.tsx` and any other frontend files that filter on origin. - -8. **SDK step key format.** Currently `"application-{uuid[:8]}"` — short slugs risk collision and don't match the design's `step_key = revision_id`-based format. - -9. **`set_flag` as a first-class endpoint.** Currently flags are set via `PATCH /runs/{id}` with an arbitrary body. A dedicated `POST /runs/{id}/flags/{flag_name}` (or similar) would match the operation model and make constraints explicit. - -10. **`RemoteAPIPersistence` abstraction in SDK.** The `desired-architecture.md` calls for injecting a persistence adapter. Currently all HTTP calls are inlined in `evaluate.py`. Extracting them into an adapter class would decouple orchestration from transport. - ---- - -**Document Status:** Draft — reflects codebase state as of 2026-02-17 -**Next Action:** Prioritize gaps and assign to refactoring phases diff --git a/docs/designs/scope-only-routers-plan.md b/docs/designs/scope-only-routers-plan.md new file mode 100644 index 0000000000..f6eda2a0fb --- /dev/null +++ b/docs/designs/scope-only-routers-plan.md @@ -0,0 +1,106 @@ +# Scope-Only Routers + Access Router Migration — Plan + +Status: PROPOSED (no edits applied). For review before execution. + +## Goal (per user) + +Make `oss/src/routers/` and `ee/src/routers/` (and the matching `services/`) +**scope-only**: organizations, workspaces, projects, users, api_keys. Everything +else moves out or is removed. + +Non-scope things currently in `routers/`: +1. **health** (`oss/src/routers/health_router.py`) — inline into entrypoints. +2. **permissions** (`oss/src/routers/permissions_router.py`) — move to the new + `oss/src/apis/fastapi/access/` structure, mirroring EE's `access` router, and + change the path `/permissions/verify` → `/access/permissions/check`. + +Decisions locked: +- **Path: new only, no alias.** Update all in-repo callers in the same change. +- **Fern client + API reference regeneration: handled by the user**, not here. + So we do NOT touch `web/packages/agenta-api-client` generated files. + +## Scope of THIS change (API side) + +### 1. Health → inline into entrypoints +- Add `@app.get("/health/", ...)` (or `/health`) directly in + `entrypoints/routers.py` returning `{"status": "ok"}` (status 200, + operation_id `health_check`, tag `Status`). +- Remove the `health_router` import + its `include_router` mount + (entrypoints/routers.py:1139). +- Delete `oss/src/routers/health_router.py`. +- (9-line file, zero deps — trivial.) + +### 2. Permissions → OSS `apis/fastapi/access/` + new path +- Create `oss/src/apis/fastapi/access/router.py` with an `AccessRouter` class + (mirror EE's `AccessRouter`: `self.router = APIRouter()`, + `add_api_route(...)`, `@intercept_exceptions()`). + - Route: `GET /permissions/check` (operation_id `check_permissions`), so + mounted under the `/access` prefix it becomes **`/access/permissions/check`**. + - Move the `Allow`/`Deny` classes + the `verify_permissions` handler body + (rename handler → `check_permissions`) verbatim; keep the `is_oss()` → + Allow / `is_ee()` → real-check behavior unchanged. + - Keep the conditional EE imports (`check_action_access`, `check_entitlements`, + `Permission`, `Counter`) exactly as today — handler works OSS-standalone. + - Add `__init__.py` to `oss/src/apis/fastapi/access/`. +- Mount in `entrypoints/routers.py`: instantiate `AccessRouter()` and + `app.include_router(access_router.router, prefix="/access", tags=["Access"])`. + - **Coexistence:** EE's `extend_main` also mounts its `access_router` at + `/access` (`/access/plans`, `/access/roles`). FastAPI merges routers on the + same prefix, so OSS `/access/permissions/check` + EE `/access/plans|roles` + all coexist when EE runs; OSS-only has just `/access/permissions/check`. + (No collision: distinct subpaths.) +- Remove the old `permissions_router` import + mount + (entrypoints/routers.py:1145, prefix `/permissions`). +- Delete `oss/src/routers/permissions_router.py`. + +### 3. In-repo callers of the old path (update to new path) +- **Throttle map** `ee/src/core/access/entitlements/types.py:174`: + `(Method.ANY, "/permissions/verify")` → `(Method.ANY, "/access/permissions/check")` + (keep it in the SERVICES_FAST category, same as today). +- **Python SDK** (in-repo): + - `sdks/python/agenta/sdk/middlewares/running/vault.py:130,173` — the + `{host}/api/permissions/verify` call + its docstring → + `{host}/api/access/permissions/check`. + - `sdks/python/agenta/sdk/middlewares/routing/auth.py:141` — same path update. + - NOTE: confirm the SDK sends the same query params (`action`, + `resource_type`, …) and reads `effect`/`credentials` from the response — the + handler contract is unchanged, only the path moves. +- **Do NOT touch** `web/packages/agenta-api-client/*` (Fern-generated) or + `web/_reference/*` — user regenerates these. + +### 4. Verify +- ruff on touched api files. +- Runtime: OSS app import (`/access/permissions/check` route present); EE + extend mounts both. Confirm OSS-only path returns Allow. +- SDK: `sdks/python` — ruff format + check on the two middleware files; run any + SDK unit tests that touch these middlewares. +- Throttle: the ENDPOINTS entry resolves (no test asserts the old literal? check + `test_*throttl*`). + +## Deferred (NOT in this change — flagged for later) + +- **Services `admin_manager.py` + `commoners.py`** are non-scope (platform/ + onboarding helpers) but are genuinely service-layer; moving them to a `core/` + home is a separate, lower-priority cleanup. Leave for now. +- **EE routers** (`ee/src/routers/organization_router.py`, + `workspace_router.py`) are already scope-only — no change. +- The 6 pre-existing eval-loop test failures (UEL-017/021/022/023) are unrelated. + +## Open questions for sign-off + +- **Q1 — route path/op:** `GET /access/permissions/check`, operation_id + `check_permissions`? (EE uses `fetch_access_plans`/`fetch_access_roles`; this + would be `check_permissions` under the same `/access` group.) OK? +- **Q2 — health path:** keep trailing slash `/health/` (current) or `/health`? + Inline vs a tiny `entrypoints/health.py`? Recommend: inline `@app.get("/health/")` + in routers.py (matches current path exactly, no new file). +- **Q3 — SDK path:** the SDK calls `{host}/api/permissions/verify` (note the + `/api` prefix). New path `{host}/api/access/permissions/check`. Confirm the + `/api` mount prefix is unchanged (only the sub-path moves). + +## Blast radius +- Delete 2 router files; edit `entrypoints/routers.py` (imports + mounts + inline + health); new `apis/fastapi/access/` package (2 files). +- Update throttle map (1 line) + 2 SDK middleware files (2-3 lines). +- No EE router changes; EE `access` router untouched (keeps adding plans/roles). +- Generated client + API ref: user-handled. diff --git a/docs/designs/third-party-subsystem-access.md b/docs/designs/third-party-subsystem-access.md new file mode 100644 index 0000000000..878ba6268c --- /dev/null +++ b/docs/designs/third-party-subsystem-access.md @@ -0,0 +1,125 @@ +# Third-Party Subsystem Access Patterns + +How `api/` code should reach external subsystems (Postgres, Redis, Stripe, PostHog, +SuperTokens, SendGrid, …), why the patterns differ, and what the current state is. + +Authored 2026-05-21 from a read-only audit of the `api/` tree plus the decisions +made while fixing UEL-024 and UEL-027 (see +`docs/designs/unified-eval-loops/findings.md`). + +## Intent + +There is **no single DI story**, and that is deliberate. Forcing one uniform +pattern across every subsystem fights both the libraries (some are module +singletons, some are instantiated clients) and the call shape (some deps are +required on every request, some are optional one-liners). Instead, the access +pattern is chosen by classifying each subsystem along **two axes**: + +1. **Required vs optional** — is the dependency needed for the app to function on + every request, or is it an optional integration that may be disabled/absent? +2. **Per-request client vs boot-time framework vs shared-orchestration** — how is + the dependency used at the call site? + +The goal: testability (a swappable seam), no eager work at import time for +optional deps, and not duplicating orchestration that several callers share. + +## The model (decision table) + +| Subsystem | Required? | Call shape | Pattern | Rationale | +|-----------|-----------|------------|---------|-----------| +| **Postgres / SQLAlchemy** | required | per-request client | **Lazy singleton factory + constructor injection** | Modern DAOs take `engine=None` and fall back to `get_transactions_engine()`. The factory is a lazy singleton; the constructor param is the test seam. | +| **Redis / cache** | required-ish | per-request, but always behind helpers | **Lazy singleton factory + util wrapper** | Callers never touch the Redis client — they call `get_cache`/`set_cache` in `caching.py`, backed by a module-global lazy engine. | +| **Stripe** | optional | one-liner direct calls (`stripe.Customer.create`) | **Lazy loader, direct use** | `_load_stripe()` returns the module (with `api_key` set), or `None`. No shared orchestration → no util wrapper; callers use it directly and null-check. | +| **PostHog** | optional | one-liner direct calls (`posthog.capture`) | **Lazy loader, direct use** | Same as Stripe. `_load_posthog()` returns the module or `None`. | +| **SendGrid (email)** | optional | shared orchestration (load template → format → validate sender → send) | **Lazy loader + util wrapper** | Email callers share real glue, so it belongs behind a util like Redis behind `caching.py`. `_load_sendgrid()` returns a client instance; `emailing.send_email(...)` is the public entry. | +| **Loops (marketing contacts)** | optional | single HTTP POST | **Util wrapper, direct HTTP (no client to load)** | Not an SDK — a raw `httpx` call to the Loops API. Nothing to lazy-load; lives in `emailing.py` as `add_contact(...)` (same outbound surface as email), enabled-guarded on `env.loops.enabled`. | +| **SuperTokens** | required | boot-time framework | **Eager conditional init at startup** | Not a per-request client — a framework that registers global middleware/recipes. Initialized once via `init_supertokens()` at app boot. DI/lazy would add nothing. | + +### The three loader/access shapes, concretely + +- **Constructor injection (Postgres):** + ```python + class MetersDAO: + def __init__(self, engine: TransactionsEngine = None): + self.engine = engine or get_transactions_engine() + # tests: MetersDAO(engine=mock_engine) + ``` +- **Lazy loader, direct use (Stripe / PostHog):** + ```python + stripe = _load_stripe() # None if disabled/unavailable + if stripe is None: return + stripe.Customer.create(...) + # tests: monkeypatch the loader, e.g. _load_posthog + ``` +- **Lazy loader + util wrapper (Redis, SendGrid):** + ```python + # callers never see the client: + await get_cache(...) ; await set_cache(...) # caching.py + await emailing.send_email(to_email=..., subject=...) # emailing.py + ``` + +## Cross-cutting rules (decided) + +1. **Each `lazy.py` loader owns its enabled-check** and returns `None` when the + subsystem is disabled/unavailable; callers null-check the result. (SendGrid + does this now. Stripe/PostHog still gate `if env.X.enabled` at call sites — see + "Known gaps" — but the intended direction is loader-owns-it.) +2. **Loaders return whatever the library is designed for.** `stripe`/`posthog` + return the module (global `api_key`); `sendgrid` returns a client instance. Do + **not** force uniformity against the library's own shape. +3. **Optional deps must not do work at import time.** No module-global client + construction (the UEL-027 anti-pattern). Build on first use via the loader. +4. **Wrap in a util only when callers share orchestration.** One-liner deps + (Stripe/PostHog) use the loader directly. Deps with shared glue (Redis, email) + get a util so the glue isn't duplicated across call sites. +5. **Don't add production "proxy globals" just to satisfy tests.** Patch the real + seam (constructor, or the loader/factory function). This is the UEL-024 lesson. + +## Current state (2026-05-21) + +- **Postgres:** consistent. Modern DAOs (`MetersDAO`, `SecretsDAO`, `ToolsDAO`, + `EvaluationsDAO`, `GitDAO`, `IdentitiesDAO`, EE `EventsDAO`/`OrganizationsDAO`/ + `SubscriptionsDAO`, `TracingDAO` on `AnalyticsEngine`) all use constructor + injection with factory fallback. Engines wired once at + `api/entrypoints/routers.py` (~lines 398-401). Legacy `db_manager`/ + `db_manager_ee` call `get_transactions_engine()` inline per-function (factory + lookup, no injectable seam) — acceptable for legacy that is being replaced. +- **Redis:** `caching.py` module-global lazy engine + helper functions. Clean + isolation; no DAO takes Redis as a constructor param. +- **Stripe / PostHog:** `_load_stripe` / `_load_posthog` in + `oss/src/utils/lazy.py`; used directly in services (`subscriptions/service.py`, + `meters/service.py`, `auth/helper.py`, `auth/supertokens/overrides.py`, + `analytics_service.py`). +- **SendGrid:** fixed in UEL-027. `_load_sendgrid()` added to `lazy.py`; + `oss/src/utils/emailing.py` is the util (public `send_email`, private + `_read_email_template` / `_render_email_template`); template at + `oss/src/utils/templates/send_email.html`. The old + `oss/src/services/email_service.py` and the dead eager `sg` in `db_manager_ee.py` + were removed. Four call sites migrated to `emailing.send_email(...)`. +- **SuperTokens:** `init_supertokens()` at startup (`routers.py` ~line 189). + +## Known gaps / follow-ups (not yet done) + +- **Stripe/PostHog enabled-check still at call sites.** Per rule 1 it should move + into the loaders (return `None` when disabled). ~5 call sites; not migrated to + avoid unprompted churn. +- **Email render-per-recipient.** The EE `notify_org_admin_invitation` loop calls + `emailing.send_email` per admin, re-rendering identical HTML each time. Cheap; + could split render-once if it ever matters. +- **Legacy `db_manager_ee` has no injectable seam.** Fine while it is slated for + replacement; if it survives, class-ify it or thread `engine` through. + +## Tradeoffs considered (and rejected) + +- **One uniform DI everywhere.** Rejected: fights library shapes (module vs + client) and over-engineers optional one-liner deps. +- **Fully unpacking `email_service` to inline `sg.send` at every call site (to + match Stripe/PostHog).** Rejected: the four email callers share template + + format + sender-validation glue; inlining would duplicate ~10 lines ×4. Email is + a caching-style boundary, not a one-liner dep. +- **Forcing Stripe/PostHog loaders to return wrapped client objects** for surface + uniformity. Rejected: works against how those libraries are built (module + singletons with a global `api_key`). +- **Adding module-level proxy globals so existing tests' `monkeypatch` targets + keep working (UEL-024).** Rejected: hides the real seam and degrades production + code; tests were updated to patch the actual seam instead. diff --git a/docs/designs/unified-eval-loops/breakdown.md b/docs/designs/unified-eval-loops/breakdown.md new file mode 100644 index 0000000000..d908a9628f --- /dev/null +++ b/docs/designs/unified-eval-loops/breakdown.md @@ -0,0 +1,341 @@ +# Unified Eval Loops — Branch Breakdown + +Diff base: `release/v0.100.9` + +This branch is not a clean PR stack. It is a feature branch with several rounds +of implementation, refactors, follow-up fixes, and test backfills. The earlier +`breakdown.md` tried to impose a speculative split that does not line up well +with the actual diff. + +This version is grounded in the current branch contents and groups changes by +reviewable change-cluster rather than by an idealized incremental history. + +## Scope excluded from this breakdown + +These changed files are auto-generated and should not drive the review split: + +- Fern / generated API clients +- `docs/docs/reference/api/**` + +Those files should be regenerated after the code-facing API is settled. + +## Size, excluding generated churn + +The full branch touches 595 files, but the biggest bucket is generated output: + +- `docs/docs/reference/api/**`: 291 files +- Fern / generated clients: large follow-on churn across `clients/` and `web/packages/agenta-api-client/` + +What remains is still a substantial branch centered on four real code streams: + +1. Unified eval loops core: OSS runtime, evaluation API/persistence, and Python SDK parity +2. Access-control refactor and scope/auth wiring +3. EE admin/org/events follow-ons +4. Small web UI and local infra adjustments + +## Chunk 1 — Unified eval loops core: OSS runtime, evaluation API, persistence, and SDK parity + +This is the core of the branch. + +The old evaluation execution path is replaced with a new runtime-oriented +structure in the OSS API, the evaluation domain model and queue semantics are +reworked around that runtime, and the Python SDK gains a parallel runtime so +local `evaluate()` and server-side execution use the same concepts. + +What actually changed: + +- OSS runtime package: + - `api/oss/src/core/evaluations/runtime/models.py` + - `api/oss/src/core/evaluations/runtime/topology.py` + - `api/oss/src/core/evaluations/runtime/planner.py` + - `api/oss/src/core/evaluations/runtime/tensor.py` + - `api/oss/src/core/evaluations/runtime/sources.py` + - `api/oss/src/core/evaluations/runtime/adapters.py` + - `api/oss/src/core/evaluations/runtime/executor.py` + - `api/oss/src/core/evaluations/runtime/cache.py` + - `api/oss/src/core/evaluations/runtime/runner.py` + - `api/oss/src/core/evaluations/runtime/locks.py` +- Task orchestration rewritten around: + - `api/oss/src/core/evaluations/tasks/run.py` + - `api/oss/src/core/evaluations/tasks/processor.py` + - `api/oss/src/core/evaluations/tasks/query.py` +- Legacy task paths removed or hollowed out: + - `api/oss/src/core/evaluations/tasks/legacy.py` deleted + - `api/oss/src/core/evaluations/tasks/live.py` largely replaced + - `api/oss/src/core/evaluations/tasks/batch.py` reduced +- Core service contracts updated: + - `api/oss/src/core/evaluations/interfaces.py` + - `api/oss/src/core/evaluations/types.py` + - `api/oss/src/core/evaluations/utils.py` + - `api/oss/src/core/evaluations/service.py` +- Worker wiring updated: + - `api/oss/src/tasks/taskiq/evaluations/worker.py` + - `api/entrypoints/worker_evaluations.py` +- Evaluation API layer: + - `api/oss/src/apis/fastapi/evaluations/models.py` + - `api/oss/src/apis/fastapi/evaluations/router.py` + - `api/oss/src/apis/fastapi/evaluations/utils.py` +- Evaluation persistence: + - `api/oss/src/dbs/postgres/evaluations/dbes.py` + - `api/oss/src/dbs/postgres/evaluations/dao.py` + - `api/oss/src/dbs/postgres/evaluations/utils.py` +- Default queue migrations: + - `api/oss/databases/postgres/migrations/core/versions/a1d2e3f4a5b6_add_default_evaluation_queues.py` + - `api/oss/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py` + - EE mirror migrations under `api/ee/databases/postgres/migrations/core/versions/` +- Service-level behavior changes visible from commits and tests: + - default queue creation/backfill + - one active default queue invariant + - archive/unarchive rules + - closed-run conflict handling + - run/queue query flags + - refresh-metrics dispatch behavior + - batch query to evaluator finalization fixes + - concurrency data on runs + +The branch history shows this was not a single clean implementation. It was +implemented, then corrected by several follow-up fixes: + +- default queue lifecycle/policy +- closed-run 409 behavior +- evaluator-run finalization +- refresh metrics dispatch +- exact source-family resolution + +Tests backing this cluster: + +- `api/oss/tests/pytest/acceptance/evaluations/_flow_helpers.py` +- `api/oss/tests/pytest/acceptance/evaluations/test_closed_run_guard.py` +- `api/oss/tests/pytest/acceptance/evaluations/test_default_queue_lifecycle.py` +- `api/oss/tests/pytest/acceptance/evaluations/test_default_queue_policy.py` +- `api/oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_modify.py` +- `api/oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_run.py` +- `api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_flow.py` +- `api/oss/tests/pytest/acceptance/evaluations/test_evaluation_metrics_refresh.py` +- `api/oss/tests/pytest/acceptance/evaluations/test_simple_evaluations_workflows.py` +- `api/oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py` +- `api/oss/tests/pytest/unit/evaluations/test_cache_split_utils.py` +- `api/oss/tests/pytest/unit/evaluations/test_query_eval_loops.py` +- `api/oss/tests/pytest/unit/evaluations/test_queue_dao_serialization.py` +- `api/oss/tests/pytest/unit/evaluations/test_run_flag_matrix.py` +- `api/oss/tests/pytest/unit/evaluations/test_run_flags.py` +- New SDK runtime package: + - `sdks/python/agenta/sdk/evaluations/runtime/__init__.py` + - `sdks/python/agenta/sdk/evaluations/runtime/models.py` + - `sdks/python/agenta/sdk/evaluations/runtime/topology.py` + - `sdks/python/agenta/sdk/evaluations/runtime/planner.py` + - `sdks/python/agenta/sdk/evaluations/runtime/adapters.py` + - `sdks/python/agenta/sdk/evaluations/runtime/executor.py` + - `sdks/python/agenta/sdk/evaluations/runtime/processor.py` +- Preview evaluation path updated: + - `sdks/python/agenta/sdk/evaluations/preview/evaluate.py` + - `sdks/python/agenta/sdk/evaluations/results.py` + - `sdks/python/agenta/sdk/models/evaluations.py` +- Supporting runtime/engine adjustments: + - `sdks/python/agenta/sdk/engines/running/errors.py` + - `sdks/python/agenta/sdk/engines/running/handlers.py` + - `sdks/python/agenta/sdk/engines/running/interfaces.py` + - `sdks/python/agenta/sdk/engines/running/utils.py` + - `sdks/python/agenta/sdk/middlewares/running/vault.py` + - `sdks/python/agenta/sdk/middlewares/routing/auth.py` + - `sdks/python/agenta/sdk/managers/applications.py` + - `sdks/python/agenta/sdk/managers/evaluators.py` + - `sdks/python/agenta/sdk/litellm/mocks/__init__.py` + +Tests added or expanded: + +- `sdks/python/oss/tests/pytest/acceptance/evaluations/test_evaluate_flow.py` +- `sdks/python/oss/tests/pytest/integration/test_evaluate_orchestration.py` +- `sdks/python/oss/tests/pytest/unit/test_evaluate_specs.py` +- `sdks/python/oss/tests/pytest/unit/test_evaluations_runtime.py` +- `sdks/python/oss/tests/pytest/utils/test_mock_v0.py` +- `sdks/python/oss/tests/pytest/acceptance/integrations/test_vault_secrets.py` + +Review note: +This is not three independent chunks. The OSS runtime, evaluation API/queue +behavior, and SDK runtime are one feature stream and should be reviewed +together. If anyone ever needs to split it further, the only sensible seam is: + +- runtime internals and task orchestration +- product-facing evaluation API/persistence semantics +- SDK parity and test coverage + +## Chunk 2 — Access-control refactor and scope/auth wiring + +Separate from evaluations, the branch also refactors access control and pushes +more explicit scope/auth wiring through OSS and EE. + +This stream is real, but it is not the main story of the branch. + +What actually changed: + +- New EE access package: + - `api/ee/src/core/access/controls.py` + - `api/ee/src/core/access/entitlements/*` + - `api/ee/src/core/access/permissions/*` +- Old EE entitlement code removed or reduced: + - `api/ee/src/core/entitlements/controls.py` + - `api/ee/src/core/entitlements/service.py` +- OSS access endpoint added: + - `api/oss/src/apis/fastapi/access/router.py` +- EE access endpoint override: + - `api/ee/src/apis/fastapi/access/router.py` +- Auth/helper wiring updates: + - `api/oss/src/core/auth/helper.py` + - `api/oss/src/core/auth/service.py` + - `api/oss/src/core/auth/supertokens/overrides.py` + - `api/oss/src/core/auth/turnstile.py` +- Broad router touch-ups consistent with scope/auth propagation: + - multiple files in `api/oss/src/apis/fastapi/*/router.py` +- Related DAO/service touch-ups: + - `api/oss/src/dbs/postgres/blobs/dao.py` + - `api/oss/src/dbs/postgres/events/dao.py` + - `api/oss/src/dbs/postgres/folders/dao.py` + - `api/oss/src/dbs/postgres/git/dao.py` + - `api/oss/src/dbs/postgres/secrets/*` + - `api/oss/src/dbs/postgres/tools/dao.py` + - `api/oss/src/dbs/postgres/tracing/dao.py` + - `api/oss/src/dbs/postgres/users/dao.py` + - `api/oss/src/dbs/postgres/webhooks/dao.py` + - `api/oss/src/dbs/postgres/shared/dbas.py` + +EE follow-on files tied to the new access package: + +- `api/ee/src/core/meters/*` +- `api/ee/src/core/subscriptions/*` +- `api/ee/src/dbs/postgres/meters/dao.py` +- `api/ee/src/dbs/postgres/subscriptions/dao.py` +- `api/ee/src/apis/fastapi/billing/router.py` +- `api/ee/src/middlewares/throttling.py` + +Tests: + +- `api/ee/tests/pytest/unit/test_access_controls.py` +- `api/ee/tests/pytest/unit/test_controls_env_override.py` +- `api/ee/tests/pytest/unit/test_billing_router.py` +- `api/ee/tests/pytest/unit/test_billing_settings.py` +- `api/ee/tests/pytest/unit/test_compute_meter_id.py` +- `api/ee/tests/pytest/unit/test_meters_dao_fetch.py` +- `api/ee/tests/pytest/unit/test_meters_dao_strict_soft.py` +- `api/ee/tests/pytest/unit/test_meters_types.py` +- `api/ee/tests/pytest/unit/test_period_from.py` +- `api/ee/tests/manual/test_billing_period.py` + +Review note: +This stream can be reviewed independently from unified eval loops, but the branch +interleaves it with the evaluation work. + +## Chunk 3 — EE org/events/admin cleanup and service-layer drift correction + +There is a smaller EE-focused stream that cleans up admin/event/organization +paths and updates some legacy service code while the branch was open. + +What actually changed: + +- EE API/service/DAO changes: + - `api/ee/src/apis/fastapi/events/router.py` + - `api/ee/src/apis/fastapi/organizations/router.py` + - `api/ee/src/apis/fastapi/spans/router.py` + - `api/ee/src/core/events/service.py` + - `api/ee/src/core/tracing/service.py` + - `api/ee/src/core/workspaces/types.py` + - `api/ee/src/dbs/postgres/events/dao.py` + - `api/ee/src/dbs/postgres/organizations/dao.py` + - `api/ee/src/dbs/postgres/tracing/dao.py` + - `api/ee/src/main.py` +- Legacy EE service churn: + - `api/ee/src/services/admin_manager.py` + - `api/ee/src/services/commoners.py` + - `api/ee/src/services/db_manager_ee.py` + - `api/ee/src/services/organization_service.py` + - `api/ee/src/services/workspace_manager.py` + - `api/ee/src/services/converters.py` deleted + - `api/ee/src/services/db_manager.py` deleted + - `api/ee/src/services/email_helper.py` deleted + - `api/ee/src/services/selectors.py` deleted + - `api/ee/src/services/templates/send_email.html` removed + +This cluster also has a small OSS-parity trail in old routers/services: + +- `api/oss/src/routers/*` +- `api/oss/src/services/*` +- `api/oss/src/models/api/workspace_models.py` + +Review note: +This is the noisiest non-evaluation chunk because it mixes cleanup, naming, +deletions, and parity fixes. It is better described as branch hygiene plus EE +admin follow-ons than as a single intentional feature. + +## Chunk 4 — Local infra, worker, and web follow-ons + +The branch also includes a set of supporting changes that are real but smaller. + +Infra and worker support: + +- `api/oss/src/utils/lazy.py` +- `api/oss/src/utils/caching.py` +- `api/oss/src/utils/emailing.py` +- `api/oss/src/utils/env.py` +- `api/oss/src/dbs/postgres/shared/engine.py` +- `api/oss/src/dbs/redis/shared/engine.py` +- `api/oss/src/middlewares/analytics.py` +- `api/oss/src/middlewares/auth.py` +- `api/oss/src/tasks/asyncio/events/worker.py` +- `api/oss/src/tasks/asyncio/tracing/worker.py` +- `api/oss/src/tasks/asyncio/webhooks/dispatcher.py` +- `api/entrypoints/routers.py` +- `api/entrypoints/worker_events.py` +- `api/entrypoints/worker_tracing.py` +- `api/entrypoints/worker_webhooks.py` +- `services/entrypoints/main.py` +- `services/oss/src/managed.py` +- `hosting/docker-compose/oss/docker-compose.dev.yml` +- `hosting/docker-compose/ee/docker-compose.dev.yml` + +Frontend follow-ons: + +- `web/oss/src/components/pages/evaluations/NewEvaluation/Components/AdvancedSettings.tsx` +- `web/oss/src/components/pages/evaluations/NewEvaluation/Components/NewEvaluationModalInner.tsx` +- `web/oss/src/components/pages/evaluations/NewEvaluation/assets/constants.ts` +- `web/oss/src/components/pages/evaluations/NewEvaluation/types.ts` +- `web/oss/src/services/evaluations/api/index.ts` +- `web/packages/agenta-entities/src/secret/api/api.ts` +- package/lockfile churn under `web/` + +The frontend part is small and clearly downstream of the evaluation API changes: +it exposes advanced evaluation settings, including concurrency-related fields, +and updates the request payload shape. + +## Docs in this branch + +Non-generated design docs are a separate stream and should be reviewed as docs, +not as evidence of the implementation split: + +- `docs/designs/unified-eval-loops/**` +- `docs/designs/unify-evals-and-queues/**` +- `docs/designs/eval-loops/**` +- `docs/designs/access-controls-refactor-plan.md` +- `docs/designs/scope-only-routers-plan.md` +- `docs/designs/third-party-subsystem-access.md` + +## Recommended review order + +If someone needs to review this branch as it exists today, this is the least +confusing order: + +1. Chunk 1: Unified eval loops core +2. Chunk 2: Access-control refactor and scope/auth wiring +3. Chunk 3: EE org/events/admin cleanup and service-layer drift correction +4. Chunk 4: Local infra, worker, and web follow-ons +5. Design docs + +## Bottom line + +The branch is mostly not “many small independent features.” It is: + +- one large unified-eval-loops stream spanning OSS runtime, evaluation API/persistence, and SDK parity +- one medium access-control/scope stream +- one smaller EE/admin cleanup stream +- one thin frontend/infra follow-on stream + +That is the shape the breakdown should reflect. diff --git a/docs/designs/unified-eval-loops/findings.md b/docs/designs/unified-eval-loops/findings.md new file mode 100644 index 0000000000..440cfab458 --- /dev/null +++ b/docs/designs/unified-eval-loops/findings.md @@ -0,0 +1,924 @@ +# Unified Eval Loops Findings + +Review scope: full `feat/unified-eval-loops` branch diff against `main`, with emphasis on the most recent `evals<>queues implementation` commit (`603820f5a`). + +Sources: + +- Fresh deep scan of code, docs, tests, and migrations on the active checkout. +- 2026-06-01 deep scan of the current `feat/unified-eval-loops` checkout, focused on the new `TensorSlice` execution path (`runtime/tensor.py` -> `tasks/processor.py` -> SDK runtime planner/logger). +- User-provided full pytest failure output from the API suite: 41 failed, 1344 passed, 8 skipped. +- Full reread of every document in `docs/designs/unified-eval-loops/` during the current applicability audit. +- Targeted local checks: + - `pytest -q ee/tests/pytest/unit/test_controls_env_override.py::TestNoOverride::test_billing_pricing_accepts_legacy_agenta_pricing_alias ee/tests/pytest/unit/test_controls_env_override.py::TestNoOverride::test_billing_pricing_accepts_legacy_stripe_pricing_alias` passed locally. + - Targeted auth/meters/db-manager tests reproduced the stale monkeypatch failures from UEL-024. + - Targeted evaluation-runtime tests could not collect in the local shell because `agenta.sdk.evaluations.runtime` was not importable from that invocation; the user-provided full-suite output remains the validation source for those failures. +- Design references: `docs/designs/unified-eval-loops/{proposal,plan,gap,research,step-removal-semantics}.md`. +- Extension references: `docs/designs/unify-evals-and-queues/{proposal,plan,gap,unify-evals-extension-synthesis,unify-evals-extension-verbatim,research}.md`. +- Existing closed findings (UEL-001..UEL-006) retained below for history; reviewed only after the independent pass. + +## Summary + +- Status: no open findings from the 2026-06-01 tensor-slice scan remain after the 2026-06-02 slice-processing fix. +- Focus area: the newly wired `TensorSlice` re-execution path for existing scenarios. +- Closed in the current pass: + - `UEL-032`: result persistence now has explicit slice execution modes via `TensorSlice.process_mode` (`fill-missing` default, `force` opt-in), so existing addressed cells no longer force the old collision behavior. + - `UEL-033`: requested `scenario_ids` and `repeat_idxs` now shape execution, and missing addressed cells on an explicit scenario no longer silently no-op. + +## Notes + +- 2026-05-20 re-audit: re-scanned current code against every OPEN finding without applying fixes. Status/diagnosis corrections recorded inline under each finding's `Re-audit (2026-05-20)` block. Net result: UEL-021 and UEL-022 share a single confirmed root cause (source-backed evaluator origin default) that is more precise than the original "kind conflation" diagnosis; UEL-021's claimed expected `kind` values are wrong; UEL-017's failing assertion is a flag-gate bug, not the multi-batch race the finding describes; UEL-023's `interface`/`configuration` sub-issue is stale (closed via UEL-003) and the actual failing assertion is unfiltered `inputs`. +- No local full-suite test execution was performed while auditing these findings; findings marked `reproduced` either come from the user-provided full-suite output or from the targeted local checks listed above. +- Findings below were re-checked against current code. Where runtime confirmation is still missing, the finding flags it and may need handoff to `test-codebase`. +- The branch carries two intertwined design tracks: unified evaluation loops (planner / source resolvers / tensor slice / runnable executor) and the evals×queues unification (default queue lifecycle + flag redefinitions). Findings cover both. +- Do not fix test-hook drift by adding broad production proxy objects. Prefer direct test injection/patching where the code already supports it, or add narrow compatibility helpers with a clear production purpose. + +## Open Questions + +- ~~Should the legacy migration that mass-creates default queues for all existing runs gate on `has_human`/policy in a single pass, or is the runtime "first-edit reconciles" lag acceptable?~~ **Resolved (2026-05-21):** the migration now mirrors the runtime create policy in a single pass — it creates default queues only for `has_human=true` runs, archives stale active default queues for runs that no longer qualify, and carries the run's own status instead of a hardcoded `running`. No reconcile lag. See UEL-010 (closed). +- ~~Is destructive `remove_step` + `prune` still the chosen lifecycle per `step-removal-semantics.md`? If so, when is the missing implementation expected?~~ **Resolved (2026-05-21):** destructive remove + prune is confirmed and implemented. Rather than a separate `remove_step` endpoint, the prune cascade is folded into the shared create/edit reconcile path (`EvaluationsService._reconcile_run`): `create_run` is "edit from an empty graph" (`prior_step_keys=set()`, prune is a no-op), and `edit_run` diffs the prior graph and prunes cells + input-only orphan scenarios + flushes metrics for any dropped step. See UEL-014 (closed). +- ~~For source-backed queues, should the public `SimpleQueueData.kind` report the source family (`queries` / `testsets`) or the executable scenario family (`traces` / `testcases`)?~~ **Resolved by 2026-05-20 re-audit:** the failing tests assert the *source* family (`kind="queries"` / `kind="testsets"`), so the current mapping is correct and the real bug is the `is_queue=False` / evaluator-origin default. See UEL-021 re-audit. +- ~~Should legacy unit tests keep monkeypatching module-level `posthog` / `engine` symbols, or should they be updated to patch dependency factories / constructor injection?~~ **Resolved by 2026-05-20 re-audit:** update the tests to patch the current seam (`_load_posthog`, `get_transactions_engine`, or constructor-injected engine) — production proxy globals are explicitly disallowed by the Notes guidance. See UEL-024 re-audit. + +## Open Findings + +None. + +## Closed Findings + +### [CLOSED] UEL-032: Existing-scenario `process(slice)` needed explicit fill-missing vs force rerun semantics + +- ID: `UEL-032` +- Origin: `scan` +- Lens: `verification` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Category: `Correctness` +- Summary: Existing-scenario slice processing now has an explicit execution mode. `TensorSlice.process_mode` defaults to `fill-missing`, which skips already-populated addressed cells and counts them as reused, and `force` reruns addressed cells through the result setter path. +- Resolution (2026-06-02): + - Added `process_mode` to `TensorSlice` with `fill-missing` as the default and `force` as the opt-in overwrite mode. + - `BackendSliceProcessor` now computes the addressed cell set up front and only sends those coordinates into the SDK loop. + - Fill-missing skips addressed existing cells (`summary.reused`), while force reruns them. + - Result persistence remains keyed by `(run_id, scenario_id, step_key, repeat_idx)` through the setter path, so reruns no longer depend on the removed edit-by-id surface. + - Unit coverage now asserts the distinction between fill-missing and force. + +### [CLOSED] UEL-033: `BackendSliceProcessor` ignored explicit scenario/repeat targeting for sparse slices + +- ID: `UEL-033` +- Origin: `scan` +- Lens: `verification` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Category: `Completeness` +- Summary: Sparse tensor slices are now honored. Explicit `scenario_ids` remain authoritative even when the addressed cells are missing, and repeat targeting is enforced before execution rather than being lost inside the full-run planner surface. +- Resolution (2026-06-02): + - `BackendSliceProcessor` now uses caller-provided `tensor_slice.scenario_ids` directly instead of deriving scenario scope from already-existing addressed cells. + - Added a planning/filter seam to the SDK slice processor so the backend can constrain execution to the addressed cell coordinates, including `repeat_idxs`. + - Added seeded invocation context from existing scenario cells so evaluator-only reruns can reuse upstream invocation outputs without rerunning the whole scenario. + - Unit coverage now asserts that a missing addressed cell on an explicit scenario still executes, and that repeat-specific targeting is enforced. + +### [CLOSED] UEL-015: `TensorSliceOperations.process` only refreshes metrics; the documented `process(slice)` contract is unimplemented + +- ID: `UEL-015` +- Origin: `scan` +- Lens: `verification` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Category: `Completeness` +- Summary: `TensorSliceOperations.process` is the only in-process implementation of the design's central `process(slice)` operation. It does not plan, execute, or populate cells — it only calls `evaluations_service.refresh_metrics(...)` and returns an empty `ProcessSummary`. The actual planner/executor pipeline runs only through `tasks/source_slice.py` and `tasks/run.py`, which are full-batch entrypoints rather than slice-aware. +- Evidence: + - `api/oss/src/core/evaluations/runtime/tensor.py:151-169`: `process` body is `await refresh_metrics(...)` then `return ProcessSummary()`. No planner/runner invocation. + - `docs/designs/unified-eval-loops/proposal.md:160-208` describes `process(run, slice)` as the full plan-and-execute loop. + - `tasks/source_slice.py:174-566` implements something close to the contract but is keyed on `run_id`/`source_items`/`testcase_ids`/`trace_ids`, not on the canonical `TensorSlice` shape. +- Impact: + - Slice-aware retries, partial re-execution by `(scenario_ids, step_keys, repeat_idxs)`, and the "probe-before-write" workflow described in `gap.md` §"Execution Gaps" are not yet possible. + - The current `process` reads like a complete implementation (it does mutate metrics) but silently does almost nothing. Callers may believe they have executed a slice when they have only refreshed metrics. +- Files: + - `api/oss/src/core/evaluations/runtime/tensor.py` + - `api/oss/src/core/evaluations/tasks/source_slice.py` +- Cause: The slice operation surface was scaffolded ahead of the planner/runner wiring needed to make `process(slice)` operational. +- Suggested Fix: + - Either route `process(slice)` through the source-slice processor by translating `TensorSlice` into the parameters that `process_evaluation_source_slice` already accepts, or rename `TensorSliceOperations.process` to clarify that it is only a metrics refresh. + - Add a unit test that asserts the documented behavior (execute auto cells in the slice, populate result cells, return a non-empty `ProcessSummary`). +- Alternatives: + - Mark this method explicitly as `metrics-refresh-only` and provide a separate `execute(slice)` method when planner integration lands. Keep the contract honest. +- Re-audit (2026-05-20): **Still reproduces.** `runtime/tensor.py` `process` (def at line 151) early-returns `ProcessSummary()` (line 159), calls `refresh_metrics(...)` (line 161), and returns an empty `ProcessSummary()` (line 169). No planner/runner invocation. Diagnosis and severity unchanged. +- Resolution (2026-05-22): made `process(slice)` an honest plan->execute->populate->refresh op via the design's "same executor, different adapters" seam, rather than faking it or renaming it down. Two parts: + - **Seam (`runtime/tensor.py`).** Added a `SliceProcessor` protocol and an optional `slice_processor` dep on `TensorSliceOperations`. `process` now short-circuits empty slices, then delegates to the injected processor; with no processor wired it raises `NotImplementedError` instead of silently refreshing metrics and returning an empty summary (the actual bug — it "silently does almost nothing"). The seam is adapter-free so `runtime/` does not depend on `tasks/`; the concrete impl is injected at the composition root. + - **Backend executor (`tasks/processor.py`).** Added `BackendSliceProcessor`, the real slice re-executor for the canonical `TensorSlice` (existing scenarios x steps x repeats — the retry / fill-missing / re-run-one-evaluator axis). For each scenario it rebuilds the source binding from the stored input result cell (`trace_id` for trace/query sources, `testcase_id` for testcase/testset sources), re-hydrates trace/testcase context via `resolve_direct_source_items`, plans from the run's CURRENT graph (so modified steps re-run with freshly resolved revisions), and reuses the SDK `process_evaluation_source_slice` engine with an existing-scenario `create_scenario` adapter — so cells populate against the addressed scenario, not a new one. Hashed-trace handling is correct because the runners are `BackendCachedRunner`s (cache lookup by step references/links before invoking), shared with the ingest path via the extracted `_resolve_runners_and_revisions` helper. + - This is the design distinction the finding's "Suggested Fix" missed: `process_evaluation_source_slice` is an INGEST loop (one source item -> one freshly CREATED scenario), so it could not be a thin translation target for a `TensorSlice` that addresses EXISTING cells. The re-executor bridges the two by reconstructing bindings from stored cells. + - Tests (`api/oss/tests/pytest/unit/evaluations/test_runtime_topology_planner.py`): existing tensor-ops test now asserts `process` raises without a processor (was: returns empty summary + refreshes metrics); added `test_tensor_slice_process_delegates_to_injected_processor` and `test_backend_slice_processor_reexecutes_existing_scenario` (rebuilds source from input cell, reuses the existing scenario, wires the auto evaluator runner). Full api suite green via `py-run-tests`. + +### [CLOSED] UEL-009: Inferred-flag derivation is shared between migration and runtime, with brittle heuristics + +- ID: `UEL-009` +- Origin: `scan` +- Lens: `verification` +- Severity: `P2` +- Confidence: `high` +- Status: `fixed` +- Category: `Soundness` +- Summary: The DAO-side helper `_make_run_flags` (`api/oss/src/dbs/postgres/evaluations/utils.py:73-138`) is the canonical place that derives the eight `has_*` flags from `run.data.steps`. It is invoked from `create_run_flags` / `edit_run_flags`, which the DAO calls inside `create_run` / `create_runs` / `edit_run` / `edit_runs`. The data migration (`a2b3c4d5e6f8`) duplicates the same heuristic in SQL. Both rely on two fragile rules: (1) substring matching of reference keys for `has_queries` / `has_testsets`, and (2) hardcoded step.key literals `{"traces", "query-direct", "testcases", "testset-direct"}` with empty references for `has_traces` / `has_testcases`. +- Evidence: + - `api/oss/src/dbs/postgres/evaluations/utils.py:104-136` walks `run.data.steps`, resets the eight `has_*` flags, and recomputes them from step shape. + - Lines 113-116 set `has_traces=True` if `step.key.lower() in {"traces", "query-direct"}` and refs are empty; same idiom for `has_testcases` with `{"testcases", "testset-direct"}`. + - Lines 118-124 set `has_queries=True` if any reference key contains the substring `"query"`; `has_testsets=True` if it contains `"testset"`. A reference key like `query_anchor`, `subquery`, or `testset_metadata` would falsely trigger either flag. + - `api/oss/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py:24-39` uses the same literals to backfill `has_traces` / `has_testcases` on existing rows. + - `docs/designs/unified-eval-loops/research.md:316-323` and `docs/designs/unify-evals-and-queues/unify-evals-extension-synthesis.md:30-33` explicitly identify "synthetic step-key inspection" as a target for removal under the new model. + - `api/oss/tests/pytest/unit/evaluations/test_run_flags.py:61-111` locks in the current heuristic (steps named `"traces"` / `"testcases"` with empty refs are expected to produce `has_traces=True` / `has_testcases=True`). +- Impact: + - Runtime and migration agree on the heuristic, so the backfill is internally consistent with new inserts. The risk is structural, not immediate. + - Any caller that constructs a direct-source input step with a non-matching `step.key` (e.g. `"my_traces_source"`) will silently produce `has_traces=False`. Downstream dispatch checks (`SimpleQueuesService._get_kind`, `dispatch_trace_slice`, `dispatch_testcase_slice`) will then misclassify or refuse the run. + - Any future reference key containing the substring `"query"` or `"testset"` (e.g. a hypothetical `query_anchor` or `testset_metadata` ref on an unrelated step) would incorrectly flip `has_queries` / `has_testsets` to `True`. + - The two heuristics live in two languages (Python and SQL). A future change to the rule must be applied in both places to keep backfilled and runtime rows consistent. +- Files: + - `api/oss/src/dbs/postgres/evaluations/utils.py` + - `api/oss/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py` + - `api/oss/tests/pytest/unit/evaluations/test_run_flags.py` +- Cause: There is no structural marker on a step that identifies its source family. Until one exists, both the migration and the DAO must infer from `step.key` / `step.references` shape. +- Suggested Fix: + - Introduce a structured marker on `EvaluationRunDataStep` for source family (e.g. a `source_kind: Literal["query","testset","trace","testcase"]` field, or a sentinel reference `{"direct_source": Reference(...)}`) and have both the DAO and the migration read it. + - Until then, harden the substring match by enforcing exact reference-key membership (`"query_revision" in references` rather than substring-on-key) and lock the direct-source step keys in a single shared constant referenced from both Python and the migration. + - Add a unit test that asserts `has_queries` does NOT trigger for a fake reference key like `"some_query_anchor"`. +- Alternatives: + - Accept the heuristic as the persistent rule and document it as a contract; gate any future step-key changes through a versioning check. +- Re-audit (2026-05-20): **Still reproduces.** Confirmed in `oss/src/dbs/postgres/evaluations/utils.py`: hardcoded direct-source step keys `{"traces", "query-direct"}` / `{"testcases", "testset-direct"}` at lines 113-116, and substring matching `"query" in step_key` / `"testset" in step_key` at lines 121-124 (refs in finding cited 104-136 / 118-124; logic unchanged, lines shifted to ~94-124). Diagnosis and severity unchanged. +- Resolution (2026-05-22): adopted the "harden to exact membership" path. `_make_run_flags` now keys `has_queries` / `has_testsets` on EXACT reference-key presence (`query_revision` / `testset_revision`) instead of substring matching, with the key sets pulled into module constants; direct-source keys stay in `DIRECT_TRACE_STEP_KEYS` / `DIRECT_TESTCASE_STEP_KEYS`. The backfill in `a2b3c4d5e6f8` was updated to match (JSONB `?` key-presence, not substring). Unit tests assert `query_anchor` / `testset_metadata` do NOT trigger the flags and that the exact key still triggers among other refs. The duplication across Python + SQL remains (two languages), and the heuristic-vs-structural-marker tradeoff is unchanged — a `source_kind` marker on the step is still the longer-term option but was not needed to remove the substring fragility. + +### [CLOSED] UEL-011: No tests cover default-queue reconciliation, archive/unarchive, or `_validate_default_queue_data` + +- ID: `UEL-011` +- Origin: `scan` +- Lens: `verification` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Category: `Testing` +- Summary: The evals×queues unification adds a lot of new behavior — `_reconcile_default_queue`, `archive_queue`, `unarchive_queue`, `_validate_default_queue_data`, `_sync_run_queue_flag_for_default_queue`, default-queue lookup, partial unique index — but pytest coverage is zero. +- Evidence: + - `grep -rn "default queue\|default_queue\|is_default" api/oss/tests/pytest/` returns no relevant hits (matches are all unrelated workspace/project `is_default`). + - `grep -rn "reconcile_default_queue\|archive_queue\|unarchive_queue\|fetch_default_queue\|EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS" api/oss/tests/` returns nothing. + - Existing acceptance tests in `test_simple_queues_basics.py` cover queue creation and source-family resolution but do not exercise default-queue lifecycle. +- Impact: + - The entire unified-queues lifecycle is unverified. UEL-008/UEL-009/UEL-010/UEL-013 in particular would have been caught by basic coverage. + - Future refactors of `_reconcile_default_queue` have no safety net. +- Files: + - `api/oss/tests/pytest/` (no relevant tests) + - `api/oss/tests/pytest/unit/evaluations/test_run_flags.py` (covers other flag aspects but not queue reconciliation) +- Cause: The implementation was prioritized over tests; the design (`unify-evals-and-queues/gap.md` §Tests) lists exactly these as required test cases. +- Suggested Fix: + - Add unit tests for `_reconcile_default_queue` under all four state transitions (`required+missing`, `required+archived`, `required+active`, `not required+active`). + - Add tests for `archive_queue`/`unarchive_queue` returning `None` when the queue is missing, raising for closed runs, and syncing `run.flags.is_queue`. + - Add tests for `_validate_default_queue_data` rejecting `user_ids`, `scenario_ids`, `step_keys`, `batch_size`, `batch_offset` on `is_default=True` queues, both in create and edit paths. + - Add a DAO/integration test asserting that the partial unique index `ux_evaluation_queues_default_per_run` prevents two default queues per run (including archived rows). + - Add a regression test for `delete_queue`/`delete_queues` raising "default queues must be archived, not hard deleted". +- Alternatives: + - Cover the same surface through acceptance tests over the HTTP routes (`/queues/{id}/archive`, `/runs/{id}/default-queue`). Reduces unit-level reach but covers HTTP wiring at the same time. +- Resolution (2026-05-22): coverage added via the HTTP-route alternative. `test_default_queue_lifecycle.py` exercises all four reconcile transitions (required+missing → create, not-required+active → archive, required+archived → unarchive, required+active → no-op) plus the non-human (no default queue) case, asserting `is_queue` sync each time; `test_default_queue_policy.py` covers `_validate_default_queue_data` rejection of `user_ids`/`scenario_ids`/`step_keys`/`batch_size`/`batch_offset` (create + edit), demotion/deletion-forbidden, and the partial-unique-index uniqueness (reject second active default, recreate after archive, allow across runs — see UEL-030). The planted-`is_queue` reconcile-back regression (UEL-020) is also here. Default-queue lifecycle is no longer untested. + +### [CLOSED] UEL-020: `is_queue` is recomputed only at the service layer; the DAO neither resets nor derives it + +- ID: `UEL-020` +- Origin: `scan` +- Lens: `verification` +- Severity: `P2` +- Confidence: `high` +- Status: `fixed` +- Category: `Soundness` +- Summary: The DAO helper `_make_run_flags` resets the eight `has_*` flags on every `create_run` / `edit_run` and re-derives them from `run.data.steps`. It does not touch `is_queue`, which is computed only by `_reconcile_default_queue` and `_sync_run_queue_flag_for_default_queue` at the service layer. Any caller that bypasses the service and writes to the DAO directly (or any future caller that constructs `EvaluationRunFlags(is_queue=True)` and calls `edit_run`) can plant a stale `is_queue` value that survives until reconciliation runs again. The same asymmetry means the DAO has no invariant to detect or correct a desynced `is_queue` on read. +- Evidence: + - `api/oss/src/dbs/postgres/evaluations/utils.py:94-102` resets only `has_queries`, `has_testsets`, `has_traces`, `has_testcases`, `has_evaluators`, `has_custom`, `has_human`, `has_auto`. `is_queue` is left to whatever the caller passed (or the row already had). + - `api/oss/src/core/evaluations/service.py:440-447` writes `is_queue` based on `has_human AND default_queue.deleted_at IS NULL`, only inside `_reconcile_default_queue`. + - `api/oss/src/core/evaluations/service.py:1818-1849` writes `is_queue` only when the changed queue is `is_default=True`. + - `api/oss/src/core/evaluations/service.py:511-533` and `587-635` are the only paths that chain reconcile after a run write. A caller using `evaluations_dao.edit_run` directly will skip reconcile entirely. + - Design contract: `is_queue` is "active default queue exists AND active human evaluator work exists" (`docs/designs/unify-evals-and-queues/unify-evals-extension-synthesis.md:40-55`). This is a derived fact that should not be persistable by the DAO without re-derivation. +- Impact: + - In the current codebase, every write path goes through the service, so reconcile fires. The persisted value is correct in practice. + - Any new code that writes via the DAO (background jobs, EE extensions, future operations like `add_step`/`remove_step`) must remember to call reconcile or it will leave `is_queue` stale. + - There is no DAO-layer invariant test that flags a desynced row, so a regression can go unnoticed. +- Files: + - `api/oss/src/dbs/postgres/evaluations/utils.py` + - `api/oss/src/core/evaluations/service.py` +- Cause: `is_queue` depends on facts outside `run.data.steps` (the existence and lifecycle of a default queue), which the DAO does not load. Service-layer reconciliation was added later as the single source of truth. +- Suggested Fix: + - Document the invariant in `utils.py` (`# is_queue is service-derived; do not write directly`) and add a service-layer regression test that calls `edit_run` with `is_queue=True` on a run that should not be queue-eligible and asserts reconciliation flips it back. + - Alternatively, move `is_queue` out of the persisted flag and compute it on read in the service layer, so it cannot be wrong. +- Alternatives: + - Keep `is_queue` persisted but enforce reconciliation as part of every `edit_run` call (e.g. wrap the DAO call inside a service decorator that always runs reconcile afterward). +- Resolution (2026-05-22): adopted the Suggested Fix. The invariant is now documented in `_make_run_flags` (`is_queue` is service-derived, owned by `_reconcile_default_queue`; do not write via the DAO without running reconcile). A regression test (`test_default_queue_lifecycle.py::test_planted_is_queue_flag_is_reconciled_back`) PATCHes `is_queue=True` onto a non-human (non-eligible) run and asserts the edit path reconciles it back to `False` with no default queue. Kept `is_queue` persisted (the compute-on-read alternative was not needed); every write still routes through the service reconcile. + +### [CLOSED] UEL-017: Source-aware queue runs may transiently flip run status during multi-batch dispatch + +- ID: `UEL-017` +- Origin: `scan` +- Lens: `verification` +- Severity: `P2` +- Confidence: `medium` +- Status: `not-a-bug` +- Category: `Correctness` +- Summary: `process_evaluation_source_slice` writes the run status (`RUNNING`/`SUCCESS`/`ERRORS`/`FAILURE`) at the end of each invocation when `update_run_status=True`. For source-aware queues that dispatch multiple batches (`_dispatch_source_batches` triggers several slice runs), each batch can flip the status before subsequent batches arrive, producing a `SUCCESS` -> `RUNNING` -> `SUCCESS` thrash. +- Evidence: + - `api/oss/src/core/evaluations/tasks/source_slice.py:527-563` writes `run.status = run_status` after the slice, gated only by `severity` ordering against the **current** persisted value (not against in-flight slice arrivals). + - `api/oss/src/core/evaluations/tasks/run.py:109-148` and `tasks/source_slice.py:174-205` are invoked once per slice, not once per run. + - Source-aware queue creation in `SimpleQueuesService._dispatch_source_batches` (referenced at `service.py:3730-3743`) ships multiple slices for a single run. +- Impact: Operators observing a run during multi-batch dispatch may see status oscillate. Idempotent observers and frontend status displays may flicker. The severity rule at line 543-547 floors at the higher status, but only against the previously persisted status — not against still-pending parallel slices. +- Files: + - `api/oss/src/core/evaluations/tasks/source_slice.py` +- Cause: Status update logic was written for the single-slice case and re-used for multi-slice dispatch without coordination. +- Suggested Fix: + - Pass an explicit `expected_total_slices` / `slice_index` (or a "this is the last slice" flag) into `process_evaluation_source_slice`, and only update the run status on the final slice. + - Alternatively, move run-status reconciliation out of the slice loop into a separate run-finalize task that aggregates across slices. +- Alternatives: + - Document the thrash and accept it. Cheaper but degrades the UX promise of the unified loop. +- Re-audit (2026-05-20): **A concrete, test-failing sub-bug exists that this finding does not capture; promote to `reproduced`.** The failing test `test_source_slice_processor_preserves_higher_queue_status` (pytest dump) is not about the multi-batch race this finding describes — it fails because the severity-floor block in `process_evaluation_source_slice` is gated on `run.flags.has_traces or run.flags.has_testcases` (`tasks/source_slice.py:566-571`). The test's run is `EvaluationRunFlags(is_queue=True, has_queries=True)` (query-backed), so `has_traces`/`has_testcases` are both False, the floor is skipped, and `run_status` stays `SUCCESS` instead of being floored up to the persisted `ERRORS` — hence `assert ... == ERRORS` got `SUCCESS`. + - So there are now two distinct issues under the "status preservation" umbrella: (1) **this finding's** multi-slice thrash race (still `medium`/un-reproduced as a *race*), and (2) the severity-floor flag gate excluding `has_queries`/`has_testsets` runs (reproduced, test-failing). They share `source_slice.py:566-586`. + - Corrected suggested fix for (2): include `has_queries`/`has_testsets` (or simply gate on `run.flags.is_queue`) in the severity-floor condition at line 569 so source-backed queue runs preserve their higher persisted status. This is the same `source_slice.py:569` gate referenced in UEL-022's re-audit. +- Resolution (2026-05-21) — **partial; item (2) fixed, item (1) still OPEN.** The severity-floor flag gate (`tasks/processor.py`) was widened from `has_traces or has_testcases` to also include `has_queries or has_testsets`, so source-backed (query/testset) queue runs now preserve their higher persisted status. The test-failing sub-bug `test_source_slice_processor_preserves_higher_queue_status` passes (resolved together with UEL-022). +- Resolution (2026-05-22) — **item (1) is NOT a bug; closing.** Re-analysed the multi-slice (`_dispatch_source_batches` ships one slice per input step) case under the *current* severity order (`FAILURE:4 > ERRORS:3 > SUCCESS:2 > RUNNING:1 > PENDING:0`): + - Each slice computes its status from its OWN processed subset: all-done → a terminal status, only its own pending items → RUNNING. A slice that finishes never computes RUNNING for already-done work, and the floor only keeps a *more severe* stored status. So a later all-success slice can never floor the run back UP to RUNNING — the `SUCCESS -> RUNNING -> SUCCESS` thrash the finding described cannot occur with this ordering. (The thrash was an artifact of the OLD ordering where RUNNING outranked SUCCESS; that was the real UEL-028 single-slice pin, since fixed by the reorder.) + - The only residual is cosmetic: with concurrent slices the first to finish writes the run's terminal status (and clears `is_active`) slightly before its siblings finish. The final resting state is still correct (the last write lands the right terminal status), and **liveness is not read from `status`** — it lives in `is_active`, which only *acts* via `fetch_live_runs` (DAO), gated on `is_live=True`. Source-batch/queue runs are dispatched as `queue_traces` / `queue_testcases` topologies, never `live_query` (`classify_steps_topology` returns a single dispatch), so `is_active` never gates them and the early write cannot cut off a sibling slice (each slice is an already-queued, independent taskiq task). The glitch is a transient readout only. + - Therefore no last-slice-finalize flag is warranted (it would thread a signal through ~6 dispatch hops including the taskiq boundary for a self-correcting cosmetic glitch on a field nobody gates on for these runs). +- Architectural note (root cause of the whole confusion): `status` overloads two orthogonal axes — liveness (running vs done) and outcome (success/errors/failure). The severity-floor exists only to reconstruct the axis that the single enum throws away. The clean model splits them: liveness = `is_active` (already a flag), outcome = a separate terminal value; then there is no floor and no severity map. Tracked as a future cleanup, not required for correctness now. + +### [CLOSED] UEL-018: SDK runtime drops extra runner outputs silently when batch is longer than planned cells + +- ID: `UEL-018` +- Origin: `scan` +- Lens: `verification` +- Severity: `P3` +- Confidence: `medium` +- Status: `fixed` +- Category: `Correctness` +- Summary: `process_evaluation_source_slice` zips `batch_cells` with `executions` and only handles the case where there are fewer executions than cells (closed UEL-004). When the runner returns **more** executions than planned cells (a contract violation in the other direction), the trailing executions are silently discarded. +- Evidence: + - `sdks/python/agenta/sdk/evaluations/runtime/source_slice.py:182-204` iterates `zip(batch_cells, executions)`, naturally truncating to the shorter sequence. + - Lines 216-230 handle only `len(executions) < len(batch_cells)` (`batch_cells[len(executions):]`). + - The mismatch flag at line 216 still sets `scenario_has_errors=True`, but no per-extra-cell logging happens. +- Impact: Low — the contract violation is "extra outputs from the runner," which should be rare. But the silent drop means there is no audit trail for the extra outputs. Closed UEL-004 fixed only the under-return direction. +- Files: + - `sdks/python/agenta/sdk/evaluations/runtime/source_slice.py` +- Cause: The fix for UEL-004 covered the under-count case explicitly; the over-count case was not added. +- Suggested Fix: + - When `len(executions) > len(batch_cells)`, log a structured warning that includes the extra outputs' summaries. + - Optionally, persist a marker result row for the unused executions or surface them in the `ProcessedScenario.metrics`. +- Alternatives: + - Treat extra outputs as a hard error (raise). More aggressive but symmetric with UEL-004. +- Resolution (2026-05-22): the mismatch branch in `processor.py` (SDK runtime) now splits under-count vs over-count. Over-count logs a structured warning with the dropped executions' summaries (`trace_id` / `span_id` / `status` / `error`) and still flags the scenario as having errors; the planned cells are logged from the first executions. Chose the structured-warning path over hard-raise to stay consistent with the soft-worker stance. Unit test `test_sdk_source_slice_handles_over_count_runner_batch` covers it. + +### [CLOSED] UEL-019: Source-resolver `query_revision` lookup ignores resolver returning `None` and falls through to the next resolver + +- ID: `UEL-019` +- Origin: `scan` +- Lens: `verification` +- Severity: `P3` +- Confidence: `medium` +- Status: `fixed` +- Category: `Soundness` +- Summary: `resolve_queue_source_batches` iterates resolvers in order and stops as soon as one returns a non-empty batch. If a `query_revision` reference resolves to zero trace IDs, the function returns `None` and the loop tries the testset resolver next, which silently does the wrong thing if the same step (somehow) had both refs. +- Evidence: + - `api/oss/src/core/evaluations/runtime/sources.py:229-244` iterates `[QueryRevisionTraceResolver, TestsetRevisionTestcaseResolver]` and `break`s on the first truthy batch. + - A step has either a `query_revision` or a `testset_revision` reference under current design, so the fall-through is harmless in practice — but if a future step carries both refs (a possibility under mixed-source planning), the second resolver would win silently. + - The first resolver returns `None` whenever `trace_ids` is empty (line 106-107) instead of returning an empty batch, which conflates "no traces" with "wrong resolver". +- Impact: Low today (single-ref steps), but the resolver chain is brittle to future graph extensions and to legitimately empty query results. +- Files: + - `api/oss/src/core/evaluations/runtime/sources.py` +- Cause: The resolver chain was modeled as "first-match wins" rather than "first-applicable wins". +- Suggested Fix: + - Split the resolver protocol into `applies(step)` and `resolve(step)`; iterate until `applies` returns `True`, and treat empty results as a real empty batch (or a structured error). + - Return `ResolvedSourceBatch(kind=..., step_key=..., trace_ids=[])` for empty query results so the loop sees the resolver applied. +- Alternatives: + - Document the assumption "exactly one source reference per input step" and add a planner-level validator that rejects steps violating it. +- Resolution (2026-05-22): combined both directions of the Suggested Fix. Each resolver now declares its exact `source_reference_key` and an `applies(step)` check; `resolve_queue_source_batches` selects the resolver whose key is present (first-applicable, not first non-empty), so an empty query result is a real empty batch for that resolver and never falls through to the testset resolver. The "exactly one source reference per input step" rule is enforced: a step carrying both `query_revision` and `testset_revision` raises `SourceResolutionError`. Unit tests cover the no-fall-through and multi-ref-rejection cases. + + +### [CLOSED] UEL-030: "one default queue per run" is not enforced — unique index absent and no code-level guard + +- ID: `UEL-030` +- Origin: `test` +- Lens: `validation` +- Severity: `P2` +- Confidence: `high` +- Status: `fixed` +- Category: `Soundness` +- Summary: The design relies on the partial unique index `ux_evaluation_queues_default_per_run` to guarantee at most one default queue per run, and `_reconcile_default_queue` assumes `fetch_default_queue` returns a single row. But the index is **absent from the running dev DB** and `create_queue` has **no code-level guard**, so creating two `is_default=True` queues for the same run succeeds — both become active default queues. +- Evidence: + - `POST /evaluations/queues/` with `flags.is_default=True` twice for the same `run_id` both return `count=1` (test `test_default_queue_policy.py::test_second_default_queue_for_same_run_is_rejected`, currently xfail). + - `pg_indexes` for `evaluation_queues` lists only `pkey`, `run_id`, `project_id`, `flags`, `tags`, `user_ids` — **no** `ux_evaluation_queues_default_per_run`, even though `alembic_version` is at `b2c3d4e5f7a8` (past `a1b2c3d4e5f6`, which creates it) and no later migration drops it. + - Model declares the index in `api/oss/src/dbs/postgres/evaluations/dbes.py:290-296` with `postgresql_where=text("(flags ->> 'is_default')::boolean = true")` — note this counts **archived** rows (no `deleted_at IS NULL`), which conflicts with the reconcile flow that archives then later unarchives/recreates a default queue. + - Migration `a1b2c3d4e5f6` creates the index with the same `WHERE (flags ->> 'is_default')::boolean = true` (no `deleted_at` predicate). +- Impact: + - Duplicate active default queues per run are possible, breaking the `fetch_default_queue` "single row" assumption used throughout `_reconcile_default_queue` / `_sync_run_queue_flag_for_default_queue`. + - If the index *were* applied as written (`is_default=true` without `deleted_at IS NULL`), the archive→recreate path in reconcile would hit a unique violation against the archived row. So the index predicate is also likely wrong. +- Files: + - `api/oss/src/dbs/postgres/evaluations/dbes.py` + - `api/oss/databases/postgres/migrations/core/versions/a1b2c3d4e5f6_add_default_evaluation_queues.py` + - `api/oss/src/core/evaluations/service.py` (create_queue path) +- Cause: The uniqueness guarantee was specified as a DB index but (a) the index is not present in the environment, and (b) its predicate omits `deleted_at IS NULL`, so it cannot both enforce one *active* default and allow archive→recreate. +- Suggested Fix: + - Correct the index predicate to `(flags ->> 'is_default')::boolean = true AND deleted_at IS NULL` (one *active* default per run, archived rows excluded) in both the model and a new migration; confirm it actually applies in the dev/prod DBs. + - Add a code-level guard in `create_queue` (reject/▸no-op a second active default for a run) so the invariant holds even if the index is missing, and surface it as `EntityCreationConflict`. + - Flip `test_second_default_queue_for_same_run_is_rejected` from xfail to a passing assertion. +- Notes: + - Surfaced while writing default-queue policy coverage (UEL-011). +- Resolution (2026-05-21): + - **Root cause was a duplicate alembic revision id.** `add_default_evaluation_queues` shared the revision id `a1b2c3d4e5f6` with `drop_corrupted_metrics_for_some_runs`, so alembic resolved to one file and silently skipped the index migration — which is why the index was absent from the DB despite `alembic_version` being past `a1b2c3d4e5f6`. + - Renamed the index migration to revision `a1d2e3f4a5b6` and chained both new branch migrations (`a1d2e3f4a5b6` add-index, `a2b3c4d5e6f8` backfill) linearly after each environment's head — OSS after `e6f7a8b9c0d1`, EE after `b2c3d4e5f7a8` (EE carries three extra meter/role migrations past the shared OSS head). Both graphs now resolve to a single head `a2b3c4d5e6f8` (verified with `find_head.py core`). Migrations are mirrored in both `api/oss/.../core/versions/` and `api/ee/.../core/versions/`. + - Corrected the index predicate to `(flags ->> 'is_default')::boolean = true AND deleted_at IS NULL` in both the model (`dbes.py`) and the migration, so it enforces one *active* default per run while allowing archive→recreate. + - Fixed two SQL type bugs in the backfill (`evaluation_runs.data` / `evaluation_queues.data` are `json`, not `jsonb`): cast `data::jsonb` before `jsonb_array_elements`, and insert `'{}'::json` into the `data` column. + - Verified end to end on a `--nuke` rebuild: fresh DB lands at head `a2b3c4d5e6f8` and `ux_evaluation_queues_default_per_run` exists with the corrected predicate. + - **No separate code-level guard was added.** A pre-emptive SELECT guard was prototyped while the index was missing, but it never worked (it didn't block the second insert) and was not race-safe (separate SELECT/INSERT sessions). It was removed: the partial unique index is the real enforcement, and the existing `check_entity_creation_conflict` in `create_queue`/`create_queues` already translates the index's unique-violation `IntegrityError` into `EntityCreationConflict` (HTTP 409). + - `test_second_default_queue_for_same_run_is_rejected` is now a passing assertion (xfail removed). Full `test_default_queue_policy.py` suite: 16 passed, including the three `TestDefaultQueueUniqueness` cases (reject second active default, recreate after archive, allow across different runs). + +### [CLOSED] UEL-031: Closed-run lock was silently ineffective — `EvaluationClosedConflict` swallowed by `@suppress_exceptions` on 21 DAO mutations + +- ID: `UEL-031` +- Origin: `test` +- Lens: `validation` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Category: `Correctness` +- Summary: Closing a run (`POST /runs/{id}/close`) is meant to lock it: subsequent content mutations should fail with 409. 21 DAO methods raise `EvaluationClosedConflict` when `flags.is_closed`, and 20 user-facing routes carry `@handle_evaluation_closed_exception()` (which converts it to 409) — but every DAO method's `@suppress_exceptions(...)` swallowed the conflict **before** it reached the router, so closed-run mutations silently returned 200/empty instead of 409. The lock was effectively a no-op at the HTTP layer. +- Evidence: + - All 21 raising methods used `@suppress_exceptions()` (or `exclude=[EntityCreationConflict]`) — **none** excluded `EvaluationClosedConflict` (`api/oss/src/dbs/postgres/evaluations/dao.py`). + - `test_closed_run_guard.py`: editing a closed run / creating a scenario / creating a result returned 200 (count 0), not 409. + - The 3 run routes (`create_runs`, `edit_runs`, `edit_run`) additionally lacked `@handle_evaluation_closed_exception()`, so even once the DAO raised, `edit_run` returned a generic 500. +- Worker hazard (why a blanket fix was unsafe): `process_evaluation_source_slice` calls `edit_run`/`edit_scenario` at finalization **without** checking `is_closed`. If a user closes a run mid-flight, making those raise would turn benign finalization into a crash/FAILURE. +- Resolution (2026-05-21) — harden user-facing, keep worker soft: + - Added `EvaluationClosedConflict` to the `exclude` of the 20 user-facing mutation DAO methods that raise it (edit_run/edit_runs, create/edit/delete scenario(s)/result(s), edit/delete metrics, create/edit queue(s)); the metric setter path already had no suppression so it propagates. Now the decorated routes return 409. + - Added `@handle_evaluation_closed_exception()` to the `create_runs`/`edit_runs`/`edit_run` routes (`api/oss/src/apis/fastapi/evaluations/router.py`). + - Made the worker tolerant: `process_evaluation_source_slice` wraps its finalization `edit_run` and per-item `edit_scenario` in `try/except EvaluationClosedConflict` (log + skip) — closing is a lock, not a failure. +- Files: + - `api/oss/src/dbs/postgres/evaluations/dao.py` + - `api/oss/src/apis/fastapi/evaluations/router.py` + - `api/oss/src/core/evaluations/tasks/source_slice.py` +- Regression coverage: `test_closed_run_guard.py` (5 tests) — close→is_closed, edit/scenario/result blocked with 409, open→edits allowed again. Existing flow tests confirm non-closed runs still finalize. +- Notes: + - Distinct from UEL-012 (which deliberately makes queue archive/unarchive NOT raise on a closed run — those paths were left unguarded on purpose). Surfaced while writing closed-run coverage. + +### [CLOSED] UEL-028: Batch (non-queue) source runs never finalize run status — stuck `running` after all scenarios succeed + +- ID: `UEL-028` +- Origin: `test` +- Lens: `validation` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Category: `Correctness` +- Summary: A batch (non-queue) source-backed run (testset → application → auto evaluator) processed every scenario to `success` and refreshed metrics, but the run-level `status` never rolled up — it stayed `running` with `flags.is_active=true` indefinitely. Reproduced end-to-end against the dev stack with the new `mock_v0` (LLM-free, sandbox-free) workflow. +- Evidence: + - Worker logs: `[SLICE] Complete processed=2 has_errors=False`, `scenarios_with_pending=0`, `scenarios_with_auto_results=2`. Both scenarios reached `success` in the DB. + - Targeted diagnostic logging proved the chain: the per-item computation correctly produced `run_status=SUCCESS` (`has_errors=[False,False]`, `has_pending=[False,False]`), but the value entering the terminal `edit_run` was `RUNNING`. + - Root cause: the severity-floor block in `process_evaluation_source_slice` (`api/oss/src/core/evaluations/tasks/source_slice.py`) ranked `RUNNING` (2) **above** `SUCCESS` (1). Across slices it floors the persisted status up to the more-severe value — so a run whose stored status was `RUNNING` could never transition to `SUCCESS`; it pinned at `running` forever. The transient `RUNNING`/`PENDING` states were incorrectly treated as outranking the terminal `SUCCESS`. + - Prevalence on the dev DB before the fix (batch/`is_live=false` runs): `success=35` vs `running=114`, `pending=211`. + - Not a caching issue: there is no cache layer on the evaluation-run service/DAO, and the stale value was confirmed directly in Postgres. +- Resolution (2026-05-21) — Option B from `docs/designs/unified-eval-loops/run-status-finalization.md` (no run-wide scenario aggregation): + - **Corrected the severity floor** so terminal statuses outrank the transient ones: `FAILURE(4) > ERRORS(3) > SUCCESS(2) > RUNNING(1) > PENDING(0)`. A freshly computed terminal status (incl. `SUCCESS`) now replaces a stale `RUNNING`, while a prior `FAILURE`/`ERRORS` still floors over a later SUCCESS-only slice (UEL-017's intent). `test_source_slice_processor_preserves_higher_queue_status` still passes. + - **Reset `status=RUNNING` on every (re)dispatch** in the activation flow (`service.py` `_activate_evaluation_run`), not just on creation. This makes the **extended-finished** case correct: extending a `success` run flips it back to `running` while the new work executes, then the slice re-finalizes it. (Previously it kept `run.status` on re-activation, so an extended finished run stayed `success`.) + - Hardening: the terminal `edit_run` clears `flags.is_active` for terminal statuses; `dao.edit_run` writes `status` via `status.value` + `flag_modified` (mirroring `close_run`), since `edit_dbe_from_dto` dumped the DTO without `mode="json"` and left an enum that did not persist reliably. + - **Scope note:** only the `batch_testset` / `batch_invocation` dispatch (`update_run_status=True`) finalizes; `live_query` / `batch_query` pass `update_run_status=False` and are untouched. Batch testset/invocation runs are **single-slice** today (`process_testset_source_run` issues exactly one `process_evaluation_source_slice` call), so the multi-slice early-finalize race (Option C / UEL-017 item 1) does not apply here yet. + - Regression coverage: `api/oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_run.py` runs a full testset → `mock_v0` app → `mock_v0` auto-evaluator evaluation end-to-end through the real worker and asserts `status=success`. Passes (~4s). DB row after the fix: `status=success`, `is_active=false`. +- Files: + - `api/oss/src/core/evaluations/tasks/source_slice.py` + - `api/oss/src/core/evaluations/service.py` + - `api/oss/src/dbs/postgres/evaluations/dao.py` +- Notes: + - Surfaced by the new `agenta:custom:mock:v0` test workflow (deterministic, no LLM, no code sandbox), which lets evaluation runs execute end-to-end in acceptance tests. + - Full analysis + rejected alternatives: `docs/designs/unified-eval-loops/run-status-finalization.md`. + +### [CLOSED] UEL-029: Batch query→evaluator runs never finalize run status (dispatched with `update_run_status=False`) + +- ID: `UEL-029` +- Origin: `test` +- Lens: `validation` +- Severity: `P2` +- Confidence: `high` +- Status: `fixed` +- Category: `Correctness` +- Summary: The `batch_query` topology (query → evaluator, no application, not live) dispatched through `process_query_source_run` with `update_run_status=False`, so — like UEL-028 on the testset path — it never rolled its status up to a terminal value. The `live_query` topology shares the same function and *must* stay running, so the two cases needed to be distinguished. +- Resolution (2026-05-21): + - `process_query_source_run` already receives `use_windowing` (True for `batch_query`, False for `live_query`). Set `update_run_status = use_windowing`, so batch query runs finalize while live runs keep polling. + - Zero-traces batch case: `process_evaluation_source_slice` rejects empty input (raises "no source items"), so an empty batch query is finalized **directly** via `evaluations_service.edit_run(status=SUCCESS, is_active=False)` rather than through the slice. (A batch query with no matching traces is complete, not failed.) + - With UEL-028's finalization in place, non-empty batch query slices finalize via the same severity-floor path. +- Files: + - `api/oss/src/core/evaluations/tasks/query.py` +- Regression coverage: `test_evaluation_flows_run.py::test_batch_query_to_evaluator_runs_to_success` now asserts `status=success` + `is_active=false` (previously xfail). E1 suite: 4 passed. +- Notes: + - Surfaced while building the run-to-completion flow suite with the `mock_v0` harness. + +### [CLOSED] UEL-012: archive/unarchive a queue must be allowed on a closed run (was: `@suppress_exceptions()` swallows `EvaluationClosedConflict`) + +- ID: `UEL-012` +- Origin: `scan` +- Lens: `verification` +- Severity: `P2` +- Confidence: `high` +- Status: `fixed` +- Category: `Correctness` +- Summary: `archive_queue`/`unarchive_queue` raised `EvaluationClosedConflict` when the parent run was closed; the bare `@suppress_exceptions()` swallowed it, returning `count=0` instead of a 409. The original finding proposed surfacing the 409. +- Resolution (2026-05-21) — **policy decision: allow both.** Archiving/unarchiving a queue is a worklist/lifecycle action, not a content edit of the run, so it must succeed even on a closed (locked) run. The `is_closed` guard was **removed** from both `archive_queue` and `unarchive_queue` in `api/oss/src/dbs/postgres/evaluations/dao.py` (no `EvaluationClosedConflict` is raised on these paths anymore, so there is nothing left to swallow). This supersedes the "surface a 409" suggestion. +- Files: + - `api/oss/src/dbs/postgres/evaluations/dao.py` +- Regression coverage: `api/oss/tests/pytest/acceptance/evaluations/test_evaluation_flows_modify.py::test_unarchive_and_archive_default_queue_on_closed_run` — closes a human-queue run, then asserts archive + unarchive do **not** return 409. + +### [CLOSED] UEL-016: Service raises bare `ValueError` for default-queue policy violations instead of typed domain exceptions + +- ID: `UEL-016` +- Origin: `scan` +- Lens: `verification` +- Severity: `P3` +- Confidence: `high` +- Status: `fixed` +- Category: `Consistency` +- Summary: `_validate_default_queue_data`, the demotion guards, and `delete_queue`/`delete_queues` raised bare `ValueError` for default-queue policy violations, against the `AGENTS.md` rule that domain exceptions be typed in the core layer and converted at the API boundary. +- Resolution (2026-05-21): + - Added a `DefaultQueueError` base + `DefaultQueueDataInvalid`, `DefaultQueueDemotionForbidden`, `DefaultQueueDeletionForbidden` (with structured `queue_id` context) in `api/oss/src/core/evaluations/types.py`. + - Replaced the bare `ValueError` raises in `service.py` (`_validate_default_queue_data`, the two demotion guards, both delete paths) with these typed exceptions. + - Added HTTP exception classes (`DefaultQueueDataInvalidException` → 422, `DefaultQueueEditingForbiddenException` → 409) in `apis/fastapi/evaluations/models.py`, and extended the `handle_evaluation_closed_exception` decorator (`apis/fastapi/evaluations/utils.py`) to convert the domain exceptions on the queue routes. +- Files: + - `api/oss/src/core/evaluations/types.py` + - `api/oss/src/core/evaluations/service.py` + - `api/oss/src/apis/fastapi/evaluations/models.py` + - `api/oss/src/apis/fastapi/evaluations/utils.py` + +### [CLOSED] UEL-021: Source-backed simple queues are classified as `queries` / `testsets`, but runtime and tests expect `traces` / `testcases` + +- ID: `UEL-021` +- Origin: `test` +- Lens: `validation` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Area: `Evaluations / Simple Queues` +- Summary: Query-backed and testset-backed simple queues currently keep `SimpleQueueKind.QUERIES` / `SimpleQueueKind.TESTSETS` as their queue kind, while the runtime dispatch works on resolved trace/testcase scenario items. The provided test run shows both unit and acceptance failures where query-backed queues are expected to expose `kind="traces"` and dispatch trace slices, and testset-backed queues are expected to expose `kind="testcases"` and dispatch testcase slices. +- Evidence: + - `oss/tests/pytest/unit/evaluations/test_query_eval_loops.py::test_simple_queue_create_dispatches_each_query_source_with_step_key` observed `created_queue.data.kind == "queries"` but expected `"traces"`. + - `oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py::test_create_source_backed_queue_preserves_repeats_and_assignments` observed `"testsets"` but expected `"testcases"`. + - Acceptance tests for creating source-backed queues returned zero queued items where one was expected. + - `api/oss/src/core/evaluations/service.py` maps queue data with `_get_source_kind()` returning `SimpleQueueKind.QUERIES` for `queue.data.queries` and `SimpleQueueKind.TESTSETS` for `queue.data.testsets`. + - `SimpleQueuesService._parse_queue()` reports `kind=self._get_kind(run)`, so the source family leaks into the public simple-queue response. +- Files: + - `api/oss/src/core/evaluations/service.py` + - `api/oss/src/core/evaluations/types.py` + - `api/oss/tests/pytest/unit/evaluations/test_query_eval_loops.py` + - `api/oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py` +- Cause: The model conflates two different concepts: source declaration family (`queries` / `testsets`) and resolved executable item family (`traces` / `testcases`). Source-backed queue creation preserves source revision IDs correctly, but public queue kind and downstream dispatch checks are using the source family where tests expect the resolved family. +- Explanation: A query-backed queue is created from query revisions, but the work assigned to reviewers/evaluators is trace scenarios. Similarly, a testset-backed queue resolves testset revisions into testcase scenarios. The API needs to preserve both facts without using one field for both. +- Suggested Fix: + - Keep `queries` / `testsets` fields as source references. + - Return `kind="traces"` for query-backed queues and `kind="testcases"` for testset-backed queues, or introduce an explicit `source_kind` field if the source family must be exposed. + - Ensure source-backed queue runs are marked/queryable consistently so `query(kind="traces")` and `query(kind="testcases")` include them. + - Add regression tests for query-backed and testset-backed queue creation, fetch, query, and add-source rejection behavior. +- Alternatives: + - If product wants `kind` to mean source family, update the tests and API docs. That would be a deliberate contract change and should not be inferred from the current implementation. +- Sources: + - Provided pytest output. + - Local inspection of `SimpleQueuesService`. +- Re-audit (2026-05-20): **Diagnosis corrected; severity unchanged (P1); confidence raised to high.** The "kind conflation" framing is inaccurate, and two evidence claims are wrong: + - The acceptance tests do NOT expect `kind="traces"`/`kind="testcases"`. `test_create_simple_queue_from_queries` asserts `queue["data"]["kind"] == "queries"` (`test_simple_queues_basics.py:173`) and `test_create_simple_queue_from_testsets` asserts `"testsets"` (line 199). They expect the source family preserved, plus `queries`/`testsets` arrays echoed back. The current `_get_source_kind` mapping to `QUERIES`/`TESTSETS` is therefore *correct*, not the bug. + - The real failure is upstream: the `KeyError: 'queue'` / `count == 0` happen because `_parse_queue` returns `None`. Trace: `SimpleQueuesService.create` source-backed branch calls `simple_evaluations_service._make_evaluation_run_data(...)` (`service.py:3636`). That builder defaults list-shaped evaluators to `DEFAULT_ORIGIN_EVALUATORS = "custom"` (`service.py:124`, applied at `3045-3049`), so `has_human=False` → `_reconcile_default_queue` leaves `is_queue=False` → `_get_kind` short-circuits to `None` at `service.py:4224` (`not run.flags.is_queue`) → `_parse_queue` returns `None` at `4271-4272` → router emits empty envelope. + - Contrast: the non-source `_make_run_data` (`service.py:4087`) defaults list evaluators to `origin="human"` (`4093-4099`), so direct trace/testcase queues get `has_human=True` → `is_queue=True` and parse correctly. This asymmetry between the two builders is the single root cause shared with UEL-022. + - Corrected suggested fix: align the source-backed evaluator-origin default with the non-source builder (default list-shaped evaluators to `"human"` in `_make_evaluation_run_data`), OR decouple `_get_kind`/`_parse_queue` from `is_queue` so a created queue is always parseable. Do NOT change `_get_source_kind`'s family mapping. Keep `kind` reporting the source family per the tests. +- Resolution (2026-05-21): **Fixed via the evaluator-origin default, scoped to the queue path only.** Confirmed the re-audit's root cause: the source builder defaulted bare-list evaluators to `custom` → `has_human=False` → `is_queue=False` → `_parse_queue` returned `None` → empty envelope. + - Added a keyword-only `default_evaluator_origin: Origin = DEFAULT_ORIGIN_EVALUATORS` param to `_make_evaluation_run_data` and used it in the list-coercion. Only `SimpleQueuesService.create` passes `"human"`; the two `SimpleEvaluationsService` callers (create/edit) keep the `custom` default, so simple-evaluation behavior is unchanged. The shared run builder stays general — this is a queue-path default, not a run-layer rule. + - **Explicit origins are always honored:** the `"human"` default applies only to a bare list (origin-less). A dict like `{id: "auto"}` is passed through verbatim — the default never overrides it. + - **New simple-queue constraint** (per user): a queue must resolve to **at least one human evaluator**. A human evaluator is one that is origin-less (defaults to human) or explicitly `"human"`. So a bare list is always valid; an explicit dict is valid only if at least one value is `"human"` — a dict whose values are all non-human (all `auto`, all `custom`, or any `auto`/`custom` mix) is rejected. Enforced as a `SimpleQueueData.validate_sources` model-validator rule ("simple queues must have at least one human evaluator", 422) at request parse — before any run/default-queue is created. The underlying evaluation run has no such restriction. + - `_get_source_kind` family mapping unchanged; `kind` still reports the source family per the tests. + - Tests: `test_simple_queues_basics.py` adds `test_source_backed_queue_with_bare_evaluator_list_is_human_queue` (bare list → `has_human`/`is_queue` true), `test_simple_queue_rejects_evaluator_dicts_with_no_human` (all-auto, all-custom, and auto+custom mix → all 422), `test_simple_queue_allows_human_mixed_with_non_human_evaluators` (human+auto and human+custom → valid queue, both origins honored). Full file 20 passed. + +### [CLOSED] UEL-022: Source-backed queue dispatch enters the source-slice processor with `has_queries` / `has_testsets`, but the processor accepts only direct-source flags + +- ID: `UEL-022` +- Origin: `test` +- Lens: `validation` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Area: `Evaluations / Simple Queues` +- Summary: Source-backed queue creation resolves query revisions into trace batches and testset revisions into testcase batches, then dispatches those batches through the same source-slice processor used by direct trace/testcase queues. The dispatch wrapper accepts source-backed flags (`has_queries` / `has_testsets`), but `process_evaluation_source_slice()` still treats `require_queue=True` as requiring direct-source flags only (`has_traces` / `has_testcases`). Query-backed and testset-backed queues can therefore pass the first dispatch gate and fail inside the slice processor. +- Evidence: + - `oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py::test_create_simple_queue_from_queries` failed with `assert 0 == 1`. + - `oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py::test_create_simple_queue_from_testsets` failed with `assert 0 == 1`. + - `oss/tests/pytest/unit/evaluations/test_runtime_topology_planner.py::test_simple_evaluation_queue_batches_dispatch_through_slice_processor` failed because a queue-shaped run fixture with source-backed input was rejected by dispatch. + - `SimpleEvaluationsService.dispatch_trace_slice()` permits `run.flags.has_traces or run.flags.has_queries`. + - `SimpleEvaluationsService.dispatch_testcase_slice()` permits `run.flags.has_testcases or run.flags.has_testsets`. + - `api/oss/src/core/evaluations/tasks/source_slice.py` rejects `require_queue=True` unless `run.flags.has_traces or run.flags.has_testcases`; it does not accept `has_queries` / `has_testsets`. + - `SimpleQueuesService._dispatch_source_batches()` resolves query/testset-backed source batches and calls `dispatch_trace_slice()` / `dispatch_testcase_slice()` with `input_step_key=batch.step_key`, so those batches eventually reach the stricter source-slice guard. +- Files: + - `api/oss/src/core/evaluations/service.py` + - `api/oss/src/core/evaluations/tasks/source_slice.py` + - `api/oss/tests/pytest/unit/evaluations/test_runtime_topology_planner.py` + - `api/oss/tests/pytest/acceptance/evaluations/test_simple_queues_basics.py` +- Cause: The queue dispatch layer was partially updated for source-backed queues, but the source-slice processor still equates "queue batch" with direct trace/testcase source flags. +- Explanation: A query-backed source batch is executable as trace items, but the run still carries `has_queries` because the input step references a query revision. A testset-backed source batch is executable as testcase items, but the run still carries `has_testsets`. The source-slice processor needs to validate the actual batch request plus input step, not only the direct-source family flags. +- Suggested Fix: + - Update `process_evaluation_source_slice()` queue validation so source-backed queue batches are allowed when `input_step_key` points to a `query_revision` or `testset_revision` input step and the concrete `trace_ids` / `testcase_ids` are present. + - Alternatively, pass `require_queue=False` for source-backed queue dispatches after validating the source batch in `SimpleQueuesService._dispatch_source_batches()`. + - Keep direct ad-hoc trace/testcase queues guarded by `has_traces` / `has_testcases`. + - Add tests for query-backed and testset-backed source batch dispatch through the actual source-slice processor. +- Alternatives: + - Persist additional resolved-source flags (`has_traces` for query-backed queues and `has_testcases` for testset-backed queues) alongside source-family flags. This would make the current guard pass, but it blurs the design's separation between source family and resolved executable item family. +- Sources: + - Provided pytest output. + - Local inspection of `SimpleEvaluationsService.dispatch_*`, `SimpleQueuesService._dispatch_source_batches()`, and `tasks/source_slice.py`. +- Re-audit (2026-05-20): **Partially confirmed; shares root cause with UEL-021.** The source-slice processor's `has_traces`/`has_testcases`-only gate is real and reproduced (see UEL-017 re-audit for the exact line — `tasks/source_slice.py:569` gates on `run.flags.has_traces or run.flags.has_testcases`, excluding `has_queries`/`has_testsets`). However, the *primary* reason the acceptance tests see `assert 0 == 1` is the same `is_queue=False` / evaluator-origin-default issue documented in UEL-021's re-audit — the queue never reaches a parseable state, so no scenarios are reported. The processor-gate gap (this finding) is the *secondary* defect that surfaces once UEL-021's origin default is fixed. Fix UEL-021 first, then re-run the source-backed dispatch tests to isolate whether the `source_slice.py:569` gate still blocks query/testset-backed batches. Keep this finding OPEN as the second-order fix. +- Resolution (2026-05-21): **Fixed.** Two parts, as the re-audit predicted: + - The `require_queue` dispatch gate in `process_evaluation_source_slice` (`tasks/source_slice.py:283-326`) had already been updated on the branch to accept `has_queries`+`query_revision` and `has_testsets`+`testset_revision` source batches — verified, no change needed there. + - The remaining offender was the run-status severity-floor block (`source_slice.py:566`), gated on `has_traces or has_testcases` only. Widened it to also accept `has_queries`/`has_testsets` so source-backed queue runs are covered. (This is the same gate as UEL-017 item #2.) + - With UEL-021's origin default fixed (queues now reach a parseable state), the source-backed dispatch tests pass: `test_simple_queues_basics.py` 20 passed, `test_query_eval_loops.py` + `test_runtime_topology_planner.py` all green (including `test_source_slice_processor_preserves_higher_queue_status`). + +### [CLOSED] UEL-023: Backend runtime adapter does not project source inputs onto the revision's declared input schema + +- ID: `UEL-023` +- Origin: `test` +- Lens: `validation` +- Severity: `P2` +- Confidence: `high` +- Status: `fixed` +- Area: `Evaluations / Runtime Adapters` +- Summary: Two adapter tests fail for separate compatibility reasons. `BackendWorkflowRunner` now puts `interface` and `configuration` on the outer `WorkflowServiceRequest`, while the unit test expects them on `workflow_request.data`. `BackendCachedRunner` always forwards `semaphore=...` to the wrapped runner, but simple runner implementations may expose `execute_batch(requests)` only. +- Evidence: + - `test_backend_workflow_runner_invokes_application_through_workflow_service` fails with `AttributeError: 'WorkflowRequestData' object has no attribute 'interface'`. + - `test_backend_cached_runner_preserves_partial_hit_order` fails with `TypeError: BatchRunner.execute_batch() got an unexpected keyword argument 'semaphore'`. + - Local inspection confirms `WorkflowServiceRequestData` is constructed with only `revision`, `parameters`, `testcase`, `inputs`, `trace`, and `outputs`. + - Local inspection confirms `BackendCachedRunner.execute_batch()` calls `self.runner.execute_batch(missing, semaphore=semaphore)` unconditionally. +- Files: + - `api/oss/src/core/evaluations/runtime/adapters.py` + - `api/oss/tests/pytest/unit/evaluations/test_runtime_topology_planner.py` +- Cause: The adapter boundary is not using a single explicit protocol. Some callers/tests still expect the older request data shape and a simpler batch runner signature. +- Explanation: These are not the same bug. The request-shape issue needs a contract decision with the workflow service DTOs. The `semaphore` issue can be fixed either by standardizing the runner protocol or by making `BackendCachedRunner` only wrap runners that implement the full protocol. +- Suggested Fix: + - Define the expected `WorkflowServiceRequest` shape once and update either the test or the adapter to match it. Avoid mutating Pydantic DTOs ad hoc to add fields that the model does not declare. + - Define a runner protocol for `execute_batch(requests, semaphore=None)` and update test runners to implement it, or make `BackendCachedRunner` tolerant of wrapped runners that do not accept `semaphore`. + - Add a small protocol-focused unit test for cached partial-hit order and semaphore forwarding. +- Alternatives: + - If the outer `interface` / `configuration` fields are canonical, close the data-shape assertion as stale and update the test to assert `workflow_request.interface` and `workflow_request.configuration`. +- Sources: + - Provided pytest output. + - Local inspection of `BackendWorkflowRunner` and `BackendCachedRunner`. +- Re-audit (2026-05-20): **Evidence stale; real failing assertion is different.** The pytest dump's actual failure for `test_backend_workflow_runner_invokes_application_through_workflow_service` is `assert workflow_request.data.inputs == {"input": "hello"}` — got all four source keys (`correct_answer`, `testcase_id`, `testcase_dedup_id`, `input`). It is NOT the `AttributeError: 'WorkflowRequestData' object has no attribute 'interface'` this finding's Evidence cites; that `interface`/`configuration` assertion was already removed when UEL-003 was closed (the test now asserts via `workflow_request.data.revision[...]`/`parameters`). So this finding's part (a) is stale. + - Confirmed real defect: `BackendWorkflowRunner.execute` sets `inputs=request.source.inputs` verbatim (`adapters.py:309`) with no projection onto the revision's declared input schema (`revision["data"]["schemas"]["inputs"]["properties"]`). The test supplies a schema declaring only `input`, so the adapter should project `request.source.inputs` down to declared properties and drop `correct_answer`/`testcase_id`/`testcase_dedup_id`. + - The `semaphore` sub-issue (`BackendCachedRunner.execute_batch` forwarding `semaphore=` unconditionally) was not re-verified in this pass; left as-is pending a targeted check. + - Corrected suggested fix: in `BackendWorkflowRunner.execute`, filter `request.source.inputs` to the keys present in `revision["data"]["schemas"]["inputs"]["properties"]` before assigning to `WorkflowServiceRequestData.inputs`. Update the finding title to drop the `WorkflowServiceRequestData.interface` shape sub-issue (closed via UEL-003) and add the input-projection sub-issue. +- Resolution (2026-05-21): **Fixed; both sub-issues confirmed closed.** Reproduced the suite: `test_backend_workflow_runner_invokes_application_through_workflow_service` failed exactly per the re-audit (`workflow_request.data.inputs` carried `correct_answer`/`testcase_id`/`testcase_dedup_id`); `test_backend_cached_runner_preserves_partial_hit_order` already **passed** (the `semaphore` sub-issue is stale, no change needed). + - Added `_project_inputs(inputs, data)` in `api/oss/src/core/evaluations/runtime/adapters.py` and applied it at the `inputs=` assignment in `BackendWorkflowRunner.execute`. It filters `request.source.inputs` to the keys declared in `data.schemas.inputs.properties`; revisions with no declared input schema pass through unchanged so untyped/legacy revisions are not broken. + - Verified: the failing test passes; the full `test_runtime_topology_planner.py` file is now 39 passed / 1 failed, where the remaining failure is the separate `test_source_slice_processor_preserves_higher_queue_status` (UEL-017/UEL-022 severity-floor gate), not this finding. + +### [CLOSED] UEL-024: Unit tests patch legacy module globals that no longer exist after dependency lookup refactors + +- ID: `UEL-024` +- Origin: `test` +- Lens: `validation` +- Severity: `P2` +- Confidence: `high` +- Status: `fixed` +- Area: `Compatibility / Tests` +- Summary: Several unit tests fail before exercising behavior because they patch module-level globals (`auth_helper.posthog`, `meters.dao.engine`, `db_manager_ee.engine`) that no longer exist. The production code now lazy-loads or constructor-loads dependencies through `_load_posthog()` / `get_transactions_engine()`. +- Evidence: + - Auth helper tests fail with `AttributeError: module 'oss.src.core.auth.helper' has no attribute 'posthog'`. + - Meter DAO tests fail with `AttributeError: module 'ee.src.dbs.postgres.meters.dao' has no attribute 'engine'`. + - Workspace invitation removal test fails with `AttributeError: module 'ee.src.services.db_manager_ee' has no attribute 'engine'`. + - Local inspection shows `auth.helper` imports `_load_posthog()` and assigns no module-level `posthog`. + - Local inspection shows `MetersDAO.__init__()` accepts `engine: TransactionsEngine = None` and defaults to `get_transactions_engine()`, so tests can inject a mock engine directly. + - Local inspection shows `remove_user_from_workspace()` calls `get_transactions_engine()` locally, and the same test file already has helper coverage that patches that function in earlier tests. +- Files: + - `api/oss/src/core/auth/helper.py` + - `api/ee/src/dbs/postgres/meters/dao.py` + - `api/ee/src/services/db_manager_ee.py` + - `api/oss/tests/pytest/unit/auth/test_helper.py` + - `api/ee/tests/pytest/unit/test_meters_dao_strict_soft.py` + - `api/ee/tests/pytest/unit/services/test_db_manager_ee.py` +- Cause: Test monkeypatch targets drifted after implementation changed dependency lookup style. The tests still patch old module-level handles instead of the current seam. +- Explanation: This is a test compatibility problem unless the module-level globals are part of an intentional public contract. Adding broad production proxy globals just to satisfy monkeypatches makes the code worse and hides the real dependency seam. +- Suggested Fix: + - Update auth tests to patch `_load_posthog()` or a small `_get_posthog_client()` helper if one is introduced. + - Update meter DAO tests to pass `MetersDAO(engine=mock_engine)` where `mock_engine.session()` returns the fake context. + - Update `db_manager_ee` pending-invite test to patch `get_transactions_engine()` consistently with the existing `_patch_core_session()` helper in the same file. + - Do not add broad module-level engine proxy objects to production DAO/service code. +- Alternatives: + - Add narrow compatibility aliases only if external code, not just tests, imports those module globals. No evidence of that exists in the provided failures. +- Sources: + - Provided pytest output. + - Local inspection of the affected modules and tests. +- Re-audit (2026-05-20): **Fully confirmed, all three module seams reproduce.** Verified directly: + - `oss/src/core/auth/helper.py` has no module-level `posthog`; it calls `_load_posthog()` inside `_get_posthog_string_entries` (line 63, imported at line 8). Tests patching `auth_helper.posthog` raise `AttributeError`. + - `ee/src/dbs/postgres/meters/dao.py` injects via `MetersDAO.__init__(self, engine: TransactionsEngine = None)` defaulting to `get_transactions_engine()` (lines 96-99); no module-level `engine`. Tests patching `dao_module.engine` raise `AttributeError`. Fix: construct `MetersDAO(engine=mock_engine)`. + - `ee/src/services/db_manager_ee.py` calls `get_transactions_engine()` per-function (lines 82, 103, 125, 147, …); no module-level `engine`. Tests patching `db_manager_ee.engine` raise `AttributeError`. Fix: patch `db_manager_ee.get_transactions_engine`. + - This is the cleanest fix candidate of the failing-test findings — test-only changes, no production code touched, consistent with the "do not add proxy globals" note. Open Question on monkeypatch strategy can be resolved as "update tests to patch the current seam." +- Resolution (2026-05-21): **Already fixed on this branch; verified by running the suite (30 passed, 0 failed).** All three test files now patch the current seam exactly as prescribed: + - `oss/tests/pytest/unit/auth/test_helper.py` patches `_load_posthog` (3 sites), not `auth_helper.posthog`. + - `ee/tests/pytest/unit/test_meters_dao_strict_soft.py` constructs `MetersDAO(engine=_mock_engine(session))`, not module-global `engine`. + - `ee/tests/pytest/unit/services/test_db_manager_ee.py` patches `get_transactions_engine` where it is called, not `db_manager_ee.engine`. + - No production code touched; consistent with the "do not add proxy globals" note. + +### [CLOSED] UEL-025: Legacy billing pricing alias tests are environment-sensitive because the subprocess helper inherits canonical pricing env + +- ID: `UEL-025` +- Origin: `test` +- Lens: `validation` +- Severity: `P3` +- Confidence: `medium` +- Status: `fixed` +- Area: `Environment / Billing Settings` +- Summary: Tests that set legacy `AGENTA_PRICING` or `STRIPE_PRICING` expected those values to populate `env.billing.pricing`, but the user-provided suite output observed a canonical/default Stripe price instead. Local inspection shows `BillingSettings.pricing` already honors legacy aliases after `AGENTA_BILLING_PRICING`, and targeted local execution of both legacy alias tests passed when no canonical pricing env was present. The remaining issue is test isolation: the subprocess helper starts from `dict(os.environ)`, so inherited `AGENTA_BILLING_PRICING` can legitimately mask legacy aliases. +- Evidence: + - `test_billing_pricing_accepts_legacy_agenta_pricing_alias` observed `price_1QmIwGB54aDbaYx3xE5J7WHA` instead of `price_agenta`. + - `test_billing_pricing_accepts_legacy_stripe_pricing_alias` observed the same default/canonical price instead of `price_stripe`. + - `api/oss/src/utils/env.py` uses `_load_json_env_dict_first("AGENTA_BILLING_PRICING", "AGENTA_PRICING", "STRIPE_PRICING")`. + - `api/ee/tests/pytest/unit/test_controls_env_override.py::_run()` copies the full parent environment before applying `env_extra`. + - Targeted local run of the two alias tests passed with `2 passed`, confirming the production alias order works when canonical env is absent. +- Files: + - `api/oss/src/utils/env.py` + - `api/ee/tests/pytest/unit/test_controls_env_override.py` +- Cause: The code path gives canonical env precedence by design, and the test helper does not scrub canonical env when validating legacy fallback. +- Explanation: The production precedence appears correct: canonical `AGENTA_BILLING_PRICING` should beat legacy aliases. The test helper should isolate env vars if it wants to validate legacy alias fallback. +- Suggested Fix: + - In `_run()` / `_ok()` test helpers, explicitly remove `AGENTA_BILLING_PRICING` when a legacy-alias test is running, or construct the subprocess env from a controlled minimal baseline. + - Add one explicit test that canonical env wins when both canonical and legacy aliases are present. +- Alternatives: + - If product wants legacy aliases to override canonical pricing, change `_load_json_env_dict_first()` order. This would contradict the existing canonical precedence test. +- Sources: + - Provided pytest output. + - Local inspection of `BillingSettings`. + - Targeted local pytest run. +- Resolution (2026-05-21): **Already fixed on this branch; verified (13 pricing tests pass).** `ee/tests/pytest/unit/test_controls_env_override.py` now isolates env for the legacy-alias tests exactly as prescribed: `test_billing_pricing_accepts_legacy_agenta_pricing_alias` and `..._stripe_pricing_alias` pass `"AGENTA_BILLING_PRICING": ""` in `env_extra` to clear the inherited canonical var so the legacy alias is consulted. A new `test_billing_pricing_prefers_canonical_env_over_legacy_aliases` locks in canonical-wins precedence. Production precedence (`_load_json_env_dict_first("AGENTA_BILLING_PRICING", "AGENTA_PRICING", "STRIPE_PRICING")`) is unchanged and correct. + +### [CLOSED] UEL-026: Events acceptance tests hit the EE audit permission/entitlement gate and need fixture alignment + +- ID: `UEL-026` +- Origin: `test` +- Lens: `validation` +- Severity: `P2` +- Confidence: `medium` +- Status: `fixed` +- Area: `Events / Acceptance` +- Summary: Four events acceptance tests expected HTTP 200 but received 403. Current code shows `POST /events/query` is gated in EE by both `Permission.VIEW_EVENTS` and the `Flag.AUDIT` entitlement. The acceptance fixture uses `cls_account["credentials"]` without asserting that account has event-view permission and audit entitlement, so the failures are likely fixture/plan setup unless response bodies or logs show a different 403 source. +- Evidence: + - `oss/tests/pytest/acceptance/events/test_events_basics.py::TestEventsBasics::test_query_events_returns_valid_response` failed with `assert 403 == 200`. + - The same 403 pattern appears for event type, unknown event type, and windowing-limit query tests. + - `api/oss/src/apis/fastapi/events/router.py` calls `check_action_access(... permission=Permission.VIEW_EVENTS)` and raises `FORBIDDEN_EXCEPTION` when false. + - The same route calls `check_entitlements(key=Flag.AUDIT)` and returns `NOT_ENTITLED_RESPONSE(Tracker.FLAGS)` when false. + - `api/oss/tests/pytest/unit/events/test_events_router_audit.py` already encodes allow/deny behavior for this gate. +- Files: + - `api/oss/tests/pytest/acceptance/events/test_events_basics.py` + - `api/oss/src/apis/fastapi/events/router.py` + - `api/oss/tests/pytest/unit/events/test_events_router_audit.py` +- Cause: Acceptance account setup likely does not guarantee the event-view permission and audit entitlement required by the route. +- Explanation: This is not evidence of an events DAO/service bug. It is a mismatch between acceptance expectations and the route's current EE access policy. +- Suggested Fix: + - Update the acceptance fixture to use an account/plan with `VIEW_EVENTS` and `AUDIT`, or assert 403 when the fixture lacks audit access. + - Log or assert the response body in the acceptance tests so permission denial and entitlement denial are distinguishable. + - Keep the existing unit coverage for audit allow/deny behavior. +- Alternatives: + - If OSS acceptance runs should bypass EE audit gating, ensure the test environment is actually OSS or conditionally skip/adjust the events acceptance tests under EE without audit entitlement. +- Sources: + - Provided pytest output. + - Local inspection of events router and unit tests. +- Resolution (2026-05-21): **Already fixed on this branch; verified.** The four OSS acceptance tests in `oss/tests/pytest/acceptance/events/test_events_basics.py` are now skipped with reason "Endpoint is plan/role-gated under EE; covered by the EE events suite" (5 skipped), matching the Alternatives fix. The plan/role-gated path is covered by `ee/tests/pytest/acceptance/events/test_events_basics.py` (5 passed). No 403-vs-200 mismatch remains. + +### [CLOSED] UEL-010: Backfill creates default queues for every existing run, bypassing the conditional policy + +- ID: `UEL-010` +- Origin: `scan` +- Lens: `verification` +- Severity: `P2` +- Confidence: `high` +- Status: `fixed` +- Category: `Migration` +- Summary: The backfill migration inserts a `flags.is_default=true` queue for every existing run that does not already have one, regardless of `has_human` or `EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS`. The runtime policy in `_reconcile_default_queue` archives default queues for runs that should not have them, but it only fires on the next `create_run`/`edit_run`. Until then the database contains stale active default queues. +- Evidence: + - `api/oss/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py:43-71` inserts unconditionally, only checking the absence of an existing default queue. + - `api/oss/src/core/evaluations/service.py:408` policy is `should_exist = EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS or has_human`. With the env constant hardcoded `False` (UEL-007), the steady-state policy requires `has_human=True` for auto-only runs to retain a default queue. + - `api/oss/src/core/evaluations/service.py:433-438` archives default queues that should not exist, but only on run edits. +- Impact: + - For environments where `has_human=False` is the majority, the migration creates many auto-only default queues that will only get archived on the next edit. Until then, `fetch_default_queue` returns active queues that the runtime would not have created. + - Queue analytics, "queues with no work" views, and simple-queue eligibility checks will see inconsistent state across the fleet during the lag window. + - The backfilled rows also default to `status='running'` regardless of run status, which is misleading for `success`/`failure` runs. +- Files: + - `api/oss/databases/postgres/migrations/core/versions/a2b3c4d5e6f8_backfill_default_evaluation_queues.py` +- Cause: The migration favors symmetry (every run gets a queue) over conformance with the runtime policy. +- Suggested Fix: + - Mirror the runtime policy in the migration: only insert default queues for runs that satisfy `has_human OR `. Archive existing default queues for runs that no longer qualify. + - Alternatively, run a one-shot data fix immediately after the migration that calls `_reconcile_default_queue` for each run. + - Set `status` to a more neutral value (e.g., `pending`) or carry over the run's status when creating queues during backfill. +- Alternatives: + - Keep the unconditional behavior and document the lag explicitly. This is acceptable if `EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS` is going to be flipped `True` in deployment, but the constant is currently `False`. +- Resolution (2026-05-21): **Fixed.** `a2b3c4d5e6f8_backfill_default_evaluation_queues.py` now mirrors the runtime create policy (`_reconcile_default_queue`) in a single pass: + - The INSERT is gated on `COALESCE((r.flags ->> 'has_human')::boolean, false) = true`, matching `should_exist = EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS or has_human` with the env toggle hardcoded `False`. + - A new reconcile-the-other-direction UPDATE archives (`deleted_at = now`, `deleted_by_id = r.created_by_id`) any stale active default queue on a run that no longer qualifies (`has_human = false`), so there is no "first-edit reconciles" lag. + - Created queues carry `COALESCE(r.status, 'running')` instead of a hardcoded `'running'`, so closed/success/failure runs are not misrepresented. + - The final `is_queue` recompute is unchanged (already `has_human AND active default queue exists`). + +### [CLOSED] UEL-014: Step lifecycle operations (`add_step`, `remove_step`, `prune`) are absent + +- ID: `UEL-014` +- Origin: `scan` +- Lens: `verification` +- Severity: `P2` +- Confidence: `high` +- Status: `fixed` +- Category: `Completeness` +- Summary: `docs/designs/unified-eval-loops/step-removal-semantics.md` chose **destructive** `remove_step` + `prune` as the lifecycle policy and listed `add_step`, `remove_step`, `add_scenario`, `remove_scenario`, `probe(slice)`, `populate(slice, results)`, `prune(slice)`, `process(slice)`, `refresh_metrics`, `set_flag` as the canonical operation surface (`proposal.md:367-381`). None of these exist in the API or service. +- Evidence: + - `api/oss/src/apis/fastapi/evaluations/router.py` (verified through subagent map) has no `add_step`, `remove_step`, `probe`, `prune`, `populate`, `process`, `set_flag` route. The closest in-process method is `TensorSliceOperations` (see UEL-015). + - `api/oss/src/core/evaluations/service.py` exposes `set_results`, `query_results`, `delete_results`, `refresh_metrics`, run/queue CRUD, but no graph-mutation API. + - `docs/designs/unified-eval-loops/step-removal-semantics.md:1-20` declares destructive remove+prune as the chosen model. +- Impact: Without the operation surface, graph evolution still requires recreating runs or in-place edits without prune cascades, which is the very fragmentation the design was meant to resolve. UI/API affordances for managing steps after creation cannot be built. +- Files: + - `api/oss/src/apis/fastapi/evaluations/router.py` + - `api/oss/src/core/evaluations/service.py` +- Cause: Implementation prioritized planner/runtime/queue plumbing; the operation API surface remains future work per the plan, but is not labeled as deferred in this branch. +- Suggested Fix: + - Track the operation API as an explicit follow-up. Either ship it in this branch or extend `gap.md` to mark it as a known-pending item. + - When implementing, follow the AGENTS.md domain conventions (`apis/fastapi//router.py` + `core//service.py` + `dbs/postgres//dao.py`). +- Alternatives: + - Resolve as `needs-user-decision` if the team prefers to land the unification without the operation surface and ship it as a follow-up PR. +- Resolution (2026-05-21): **Fixed for the `remove_step` + `prune` lifecycle (the part `step-removal-semantics.md` actually mandates).** Per user decision, graph mutation is not exposed as separate `add_step`/`remove_step` endpoints. Instead, since `create_run` is conceptually "edit from an empty graph", create and edit now funnel through one shared post-write reconciler, `EvaluationsService._reconcile_run`: + - `_reconcile_run(run, prior_step_keys)` runs `_prune_removed_steps(run, prior_step_keys - current_step_keys)` then `_reconcile_default_queue(run)`. + - `create_run` / `create_runs` pass `prior_step_keys=set()`, so prune is a guaranteed no-op (no prior cells) — preserving the "create = edit from scratch" property. + - `edit_run` / `edit_runs` fetch the prior run, capture its step keys, and after the DAO write prune the cells of any dropped step. Adding/keeping a step needs no special path: omitting a step from `data.steps` *is* a destructive removal. + - `_prune_removed_steps` implements the documented cascade: delete result cells for removed steps across scenarios/repeats; remove scenarios left with zero remaining cells (i.e. sourced only from a removed input step); flush metrics for surviving affected scenarios. Closed runs are rejected by the existing DAO `edit_run` guard. + - Files: `api/oss/src/core/evaluations/service.py` (`_reconcile_run`, `_prune_removed_steps`, `_step_keys`, rewired create/edit). Tests: `api/oss/tests/pytest/acceptance/evaluations/test_evaluation_step_removal.py` (non-input drop prunes only its cells; input drop prunes orphan scenarios; create does not prune; queue eligibility re-derived). 12/12 targeted acceptance tests pass; 52/54 evaluations unit tests pass (the 2 failures are pre-existing UEL-017/UEL-022/UEL-023, unrelated). + - Still open as separate findings: the slice-level `process(slice)` execution contract (UEL-015) and the other documented ops (`add_step`/`add_scenario`/`set_flag`/etc.) are not part of the remove+prune lifecycle and remain out of scope here. The full operation surface and per-op status (done / partial / deferred) is now catalogued in [`operations.md`](./operations.md); the deferred ops will be implemented later. + +### [CLOSED] UEL-027: SendGrid client is an eager import-time module-global, inconsistent with the lazy-loader pattern used for other optional third-party subsystems + +- ID: `UEL-027` +- Origin: `scan` +- Lens: `verification` +- Severity: `P3` +- Confidence: `high` +- Status: `fixed` +- Area: `Utils / Third-party subsystem access` +- Summary: A 2026-05-21 audit of how `api/` accesses third-party subsystems found that optional dependencies are otherwise reached through once-checked lazy loaders in `oss/src/utils/lazy.py` (`_load_stripe`, `_load_posthog`), but SendGrid was the odd one out: two separate module-level eager initializations at import time. This is the same drift trap as UEL-024 (module-globals are hard to swap in tests) and means SendGrid client construction runs at import regardless of whether email is ever sent. +- Evidence (pre-fix, files since changed/removed): + - `oss/src/services/email_service.py:13-22` (now deleted) constructed `sg = sendgrid.SendGridAPIClient(...)` at module import time, guarded only by `if env.sendgrid.enabled`, and referenced it directly in `send_email` (`sg.send(message)`). + - `ee/src/services/db_manager_ee.py:5,65-66` imported `sendgrid` and constructed `sg = sendgrid.SendGridAPIClient(api_key=env.sendgrid.api_key)` at import time **unconditionally** (no `enabled` guard). Grep confirmed `sg` was assigned and never read anywhere in that module — dead code. + - By contrast, Stripe and PostHog use `oss/src/utils/lazy.py` once-checked loaders that return `None` when disabled/unavailable, and callers null-check the result. + - No external module imports `email_service.sg` or `db_manager_ee.sg` directly; all email goes through `email_service.send_email()` / `read_email_template()` (consumers: `oss/src/services/user_service.py`, `oss/src/services/organization_service.py`, `ee/src/services/organization_service.py`). +- Cause: The audit's "two axes" model (required-vs-optional × per-request-client-vs-boot-time-framework) explains every other subsystem: required per-request deps (Postgres/Redis) get a lazy singleton factory plus constructor-injection on modern DAOs; optional per-call deps (Stripe/PostHog) get `lazy.py` loaders; the boot-time framework (SuperTokens) is initialized once at startup. SendGrid is an optional per-call client and should have been a `lazy.py` loader, but was instead two eager module-globals. +- Resolution (2026-05-21): + - Added `_load_sendgrid()` to `oss/src/utils/lazy.py`, mirroring `_load_stripe`/`_load_posthog` but returning a configured `SendGridAPIClient` instance (not a module — accepted per the "module-vs-client" decision below), or `None` when disabled/unavailable. Owns its own enabled-check and preserves the prior log lines (enabled / missing-sender / disabled). + - `ee/src/services/db_manager_ee.py`: removed the dead `sg = SendGridAPIClient(...)` global and its `import sendgrid` (the symbol was never used). + - **Boundary decision (per user):** email is a *caching-style* subsystem (callers share real orchestration: load template -> format placeholders -> validate sender -> send), unlike Stripe/PostHog which are one-liner direct calls (`stripe.Customer.create`, `posthog.capture`) with no shared glue. So email belongs behind a util like Redis behind `caching.py`, NOT inlined. Created `oss/src/utils/emailing.py` with a single public function `send_email(*, to_email, subject, username, action, workspace, call_to_action, from_email=None)` plus two private helpers (`_read_email_template`, `_render_email_template`). The loader's enabled-check + sender validation + `Mail` construction + `sg.send` all live inside the util; only `send_email` is part of the public surface. + - Deleted `oss/src/services/email_service.py` and moved its template `git mv oss/src/services/templates/send_email.html -> oss/src/utils/templates/send_email.html` (so path resolution relative to the module still works). + - Migrated all four call sites (`oss/src/services/organization_service.py`, `oss/src/services/user_service.py`, `ee/src/services/organization_service.py` x2) from the repeated `read_email_template().format(...)` + `email_service.send_email(...)` dance to a single `emailing.send_email(...)` call. Removed the now-duplicated `if not env.sendgrid.from_address: raise ValueError(...)` guard from call sites (the util owns it). Callers that short-circuit on a disabled mailer with a non-bool return (invite link / reset link) keep their own `if not env.sendgrid.enabled` early-return. + - No production behavior change: disabled SendGrid is still a logged no-op; enabled builds the client on first send instead of at import. + - Follow-up (same session): also folded the Loops marketing-contact helper into `emailing.py`. The former `ee/src/services/email_helper.py::add_contact_to_loops` was a stateless `httpx` POST to the Loops API (no SDK/client, so nothing to lazy-load) but had no enabled-guard. Moved it to `oss/src/utils/emailing.py::add_contact(email, ...)` (the method is edition-agnostic — just an HTTP call — so it lives in OSS utils; the `is_ee()` gate stays at the call site in `ee/src/services/commoners.py`), added an `if not env.loops.enabled: return None` no-op guard mirroring `send_email`, and deleted `email_helper.py`. `emailing.py` is now the single outbound email/contact surface: public `send_email` (transactional via SendGrid) + `add_contact` (Loops audience); helpers stay `_`-prefixed. + - ruff format + check clean; import smoke test passes; template loads from new location. +- Decisions captured during the audit (apply to future subsystem work): + - **Enabled-check placement:** each `lazy.py` loader owns its enabled-check and returns `None` when off; callers null-check. (Stripe/PostHog still gate at call sites today; not migrated in this pass but the intended direction.) + - **Module vs client:** loaders return whatever the library is designed for — `stripe`/`posthog` return the module (global `api_key`), `sendgrid` returns a client instance. Do not force uniformity against the library shape. + - **Boundary:** required per-request deps (Postgres/Redis) -> lazy singleton factory + constructor injection on modern DAOs; optional deps with shared orchestration (email) -> util wrapper (caching-style); optional one-liner deps (Stripe/PostHog) -> direct lazy use; boot-time framework (SuperTokens) -> eager conditional init at startup. +- Alternatives: + - Leave SendGrid eager and accept the inconsistency. Rejected: it was the only optional subsystem not using the `lazy.py` seam, and the EE copy was eager-unconditional dead code. + - Fully unpack `email_service` and inline `_load_sendgrid()` + `sg.send(Mail(...))` at every call site (to match Stripe/PostHog). Rejected: the four callers share template/format/validate orchestration, so inlining would duplicate ~10 lines x4; email is a caching-style boundary, not a one-liner. +- Sources: + - 2026-05-21 read-only subsystem-access audit (Explore agent) covering Postgres, Redis, Stripe, PostHog, SuperTokens, SendGrid. + - Local inspection of the (now-removed) `email_service.py`, `db_manager_ee.py`, `lazy.py`, and the four email call sites. + +### [CLOSED] UEL-008: `has_traces`/`has_testcases` flags are never set on run creation + +- ID: `UEL-008` +- Origin: `scan` +- Lens: `verification` +- Severity: `P1` +- Confidence: `high` +- Status: `stale` +- Category: `Correctness` +- Summary: Original claim was that the runtime never writes `has_traces`/`has_testcases`/`has_human`/etc. on run creation. +- Resolution: + - Retracted. A deeper trace of the DAO showed that `EvaluationsDAO.create_run`, `create_runs`, `edit_run`, and `edit_runs` all call `create_run_flags(_run)` / `edit_run_flags(run, base_flags=...)`, which delegate to `_make_run_flags` in `api/oss/src/dbs/postgres/evaluations/utils.py:73-138`. That helper resets the eight `has_*` flags and recomputes them from `run.data.steps` on every write that carries a step graph. + - The service-layer `_make_evaluation_run_flags` (3308-3337) does pass all eight `has_*` as explicit `False`, but the DAO overrides them with the derived values. The derived values are correct for the canonical step shapes used by `_make_evaluation_run_data` and `SimpleQueuesService._make_run_data`. + - The reported runtime symptom (`dispatch_trace_slice` short-circuit, `_get_kind` returning `None`) does not occur, because by the time those checks run, the DAO has already populated `has_traces` / `has_testcases` / `has_queries` / `has_testsets` correctly. + - Existing coverage in `api/oss/tests/pytest/unit/evaluations/test_run_flags.py:13-111` locks in the inference for the four families. + - The residual concerns about the inference's brittleness (synthetic step-key + substring matching) are tracked in the rewritten UEL-009. + +### [CLOSED] UEL-013: `SimpleQueuesService.create` forces `is_queue=False` and never re-derives via reconciliation + +- ID: `UEL-013` +- Origin: `scan` +- Lens: `verification` +- Severity: `P1` +- Confidence: `medium` +- Status: `stale` +- Category: `Correctness` +- Summary: Original claim was that `SimpleQueuesService.create` would leave `is_queue=False` because `has_human` was never set, so `_reconcile_default_queue` would never create a default queue. +- Resolution: + - Retracted. The DAO `create_run` runs `_make_run_flags`, which walks the constructed annotation steps and sets `has_human=True` when any annotation step has `origin="human"`. `SimpleQueuesService._make_run_data` defaults list-shaped evaluator inputs to `origin="human"` (service.py:4096-4099). For dict-shaped inputs, the explicit origin is honored. + - `EvaluationsService.create_run` then runs `_reconcile_default_queue(run=created_run)` with the DAO-derived `has_human`. When `has_human=True`, the default queue is created and `is_queue=True` is written via a follow-up `edit_run`. + - The explicit `EvaluationRunFlags(is_queue=False)` the service passes on create is consumed by `_make_run_flags`'s explicit-update merge and then reconciled, so the persisted value is correct. + - End-state: human-evaluator simple queues do reach `is_queue=True`; auto-only "queues" intentionally do not, per design (`unify-evals-extension-synthesis.md:174-181`). + - The layering question — that `is_queue` is service-only and the DAO does not enforce it — is split out as UEL-020. + +### [CLOSED] UEL-007: `EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS` ships as `False` for local-dev UX + +- ID: `UEL-007` +- Origin: `scan` +- Lens: `verification` +- Severity: `P2` +- Confidence: `high` +- Status: `wontfix` +- Category: `Consistency` +- Summary: The global default-queue policy toggle is a Python module-level constant (`False`) inside `service.py`, not an environment variable through `oss/src/utils/env.py`. +- Files: + - `api/oss/src/core/evaluations/service.py` +- Resolution: + - Wontfix per user decision: the constant is intentionally shipped as `False` so local development exercises the human-evaluator-conditional default-queue UX path. The `env` wiring (and the matching unit test) can be revisited when product flips the policy. + +### [CLOSED] UEL-004: Runnable batch length mismatches can silently drop planned cells + +- ID: `UEL-004` +- Origin: `scan` +- Lens: `verification` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Category: `Correctness` +- Summary: The shared source-slice loop zipped planned cells with runner results and did not verify that the runner returned one execution per requested cell. +- Files: + - `sdks/python/agenta/sdk/evaluations/runtime/source_slice.py` + - `sdk/tests/pytest/unit/test_evaluations_runtime.py` +- Resolution: + - Fixed by making `process_evaluation_source_slice` treat runner result-count mismatches as explicit scenario errors. + - Missing trailing planned cells are now logged as failed result cells with a contract-violation message instead of disappearing from persistence. + - Added focused SDK unit coverage for a two-repeat auto evaluator batch where the runner returns only one execution. +- Sources: + - The over-return direction is now tracked separately as UEL-018. + +### [CLOSED] UEL-005: Trace-backed queue slices do not load trace context before evaluator execution + +- ID: `UEL-005` +- Origin: `scan` +- Lens: `verification` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Category: `Correctness` +- Summary: Direct trace batches entered the unified runtime as `ResolvedSourceItem(trace_id=...)` only, so auto evaluators could receive no source trace, inputs, outputs, or span link. +- Files: + - `api/oss/src/core/evaluations/runtime/sources.py` + - `api/oss/src/core/evaluations/tasks/source_slice.py` + - `api/oss/tests/pytest/unit/evaluations/test_runtime_topology_planner.py` +- Resolution: + - Fixed by hydrating direct trace source items through `tracing_service` before converting them to SDK source items. + - The resolver now populates `trace`, root `span_id`, `inputs`, and `outputs` from `ag.data` when the source trace is available. + - Added focused unit coverage for direct source resolution and for `process_evaluation_source_slice(trace_ids=[...])` forwarding hydrated source context to the SDK runtime. + +### [CLOSED] UEL-006: Source-trace links are hard-coded as `invocation` + +- ID: `UEL-006` +- Origin: `scan` +- Lens: `verification` +- Severity: `P2` +- Confidence: `medium` +- Status: `wontfix` +- Category: `Consistency` +- Summary: The SDK runtime emits upstream links under the key `invocation`. +- Files: + - `sdks/python/agenta/sdk/evaluations/runtime/source_slice.py` + - `api/oss/src/core/evaluations/runtime/adapters.py` +- Resolution: + - Wontfix per user decision: the invocation link key is the workflow contract, and the key for the invocation step should be `invocation`. + +### [CLOSED] UEL-003: Dict-revision regression test asserts fields that the request model drops + +- ID: `UEL-003` +- Origin: `mixed` +- Lens: `verification` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Category: `Testing` +- Summary: The staged unit test for dict-shaped evaluator revisions fails because it asserts `workflow_request.interface` and `workflow_request.configuration`, but the active `WorkflowServiceRequest` alias is `WorkflowInvokeRequest`, whose declared payload surface is `data`. +- Resolution: + - Fixed by updating the regression test to assert preserved evaluator metadata through `workflow_request.data.revision["data"]` and `workflow_request.data.parameters`, matching the current SDK request model. + +### [CLOSED] UEL-001: Backend evaluator runner receives dumped revisions but reads them like DTOs + +- ID: `UEL-001` +- Origin: `scan` +- Lens: `verification` +- Severity: `P1` +- Confidence: `high` +- Status: `fixed` +- Category: `Correctness` +- Summary: Backend auto-annotation execution can invoke evaluators with an empty `interface` and `configuration` because the shared runtime dumps revisions to dictionaries before handing them to `BackendEvaluatorRunner`. +- Resolution: + - Fixed by making `BackendEvaluatorRunner` read revision, nested `data`, and `flags` from both dict-shaped and DTO-shaped objects. + - Added a focused unit case for dict-shaped evaluator revisions in `api/oss/tests/pytest/unit/evaluations/test_runtime_topology_planner.py`. + +### [CLOSED] UEL-002: Startup instrumentation uses raw prints in the FastAPI module + +- ID: `UEL-002` +- Origin: `scan` +- Lens: `verification` +- Severity: `P3` +- Confidence: `high` +- Status: `wontfix` +- Category: `Compatibility` +- Summary: `api/entrypoints/routers.py` now emits startup timing with top-level `print()` calls during module import. +- Resolution: + - Wontfix per user decision. diff --git a/docs/designs/unified-eval-loops/gap.md b/docs/designs/unified-eval-loops/gap.md new file mode 100644 index 0000000000..1e67168470 --- /dev/null +++ b/docs/designs/unified-eval-loops/gap.md @@ -0,0 +1,404 @@ +# Gap Analysis + +## Summary + +The current system has many pieces of a unified evaluation model, but they are split across setup surfaces, worker loops, SDK code, queue code, and frontend assumptions. + +The largest gaps are: + +- no first-class planner that turns run graph + sources + flags into tensor cells +- source resolution exists in several paths but not behind one resolver interface +- repeat-aware execution exists in current backend loops but not behind one execution planner +- pending/manual lifecycle exists in key loops but is still duplicated and topology-specific +- no slice-aware `process` operation +- no single slice-shaped operation boundary across process/probe/populate/prune +- source-family classification and simple-queue eligibility are still entangled through `is_queue` +- destructive step removal and archival step lifecycle are not yet separated + +## Already Implemented + +Current code already includes several capabilities that older design docs listed as missing or speculative: + +- `EvaluationRunFlags.is_cached` +- `EvaluationRunFlags.is_split` +- `EvaluationRunFlags.is_queue` +- `EvaluationRunData.repeats` +- `EvaluationResult.repeat_idx` +- `EvaluationResultQuery.repeat_idx` / `repeat_idxs` +- repeat helpers: `build_repeat_indices`, `required_traces_for_step`, `effective_is_split` +- cache helpers: `make_hash`, `fetch_traces_by_hash`, `select_traces_for_reuse`, `plan_missing_traces` +- source-aware queue creation from query/testset-backed sources +- source-backed queue dispatch to concrete trace/testcase batches +- human/custom evaluator pending behavior in live query and batch item paths +- repeat-aware input/evaluator result creation in live query +- repeat-aware input/application/evaluator result creation in batch testset +- repeat-aware input/application result creation in batch inference / batch invocation +- repeat-aware source/evaluator result creation in batch trace/testcase items + +## Setup Gaps + +Current setup is fragmented: + +- auto testset evaluation setup builds one specific graph shape +- human evaluation setup builds a related but separate testset shape +- live query setup has separate query semantics +- queue setup accepts direct trace/testcase IDs but not source revisions +- SDK/local setup has its own assumptions + +Still missing or incomplete: + +- canonical graph-oriented create request +- shared validation for input source combinations +- one canonical setup request model used by all wrappers +- wrapper-to-canonical translation for every existing setup API +- one place to enforce step origin semantics +- annotation queue convenience APIs that hide backing run/scenario/result setup while using the same canonical setup path + +## Source Resolution Gaps + +There is no shared abstraction for resolving source descriptors into concrete scenario items, even though source resolution now exists in several code paths. + +Resolver behavior that exists but should be extracted: + +- query revision -> trace refs for live windows +- query revision -> trace refs for source-backed queues +- testset revision -> testcase refs for source-backed queues +- testset revision -> testcase payloads for batch testset/invocation +- direct trace IDs -> trace refs +- direct testcase IDs -> testcase refs + +Current consequences: + +- scenario creation is repeated in each loop +- each loop owns part of source resolution itself +- live and batch query semantics are harder to compare +- unsupported mixed-source cases fail implicitly rather than through clear validation + + +## Source-Family Flag Gaps + +The current runtime still overloads `is_queue` in places where the concern is source family or queue-style ingestion. + +Missing from the target model: + +- explicit inferred `has_traces` +- explicit inferred `has_testcases` +- one place to prevent mixed source families using the source-family flags +- one topology contract that does not need synthetic step-name inspection to distinguish direct traces/testcases from query/testset sources + +## Default Queue Integration Gaps + +The queue layer should be a human-work view over the tensor, but the current runtime/docs still blur several concerns. + +Missing from the target model: + +- default queue as the canonical persisted view over active human work +- `queue.flags.is_default` +- queue lifecycle semantics that can drive simple-queue eligibility +- redefined persisted `run.flags.is_queue` as “active default queue + active human work” +- explicit separation between queue view semantics and source-family classification + +## Step Lifecycle Gaps + +The current mutation model still leans toward: + +```text +remove_step -> prune tensor cells +``` + +That is incomplete if product semantics require evaluator/step archival and retention of historical results. + +Needs explicit decision: + +- whether ordinary evaluator removal is archival/deactivation rather than destructive deletion +- whether active versus archived step state belongs in the graph model +- whether `process` defaults to active steps only +- whether queue eligibility is based on active human steps only +- when hard remove/prune is appropriate + +## Planner Gaps + +The system lacks an execution planner that can materialize these concepts once: + +- scenario cells +- input result cells +- auto executable cells +- human/custom pending cells +- repeat slots +- cache reuse plans +- upstream bindings between steps + +Current loops encode planning inline. This makes it difficult to support new combinations or change semantics consistently. + +- multiple input steps with consistent result slots +- repeat fan-out at different graph boundaries +- partial retries by tensor slice + +## Execution Gaps + +Current execution was specialized: + +- SDK preview evaluation had its own nested loop +- backend legacy batch testset had another loop +- backend live query had another loop +- queue batch evaluation had another loop + +Current backend implementation direction: + +- live query, batch query, direct trace queues, direct testcase queues, batch inference, and testset -> application -> evaluator resolve source items and call one backend source-slice processor +- batch inference is the application-only testset application graph shape +- API-internal task handlers have been collapsed to run and slice processors +- trace/testcase batch task helpers are no longer needed because the slice processor can call the source-slice processor directly +- specialized helper names may remain as wrappers while web/API compatibility is preserved + +Current SDK implementation direction: + +- SDK preview/local evaluation now routes through SDK-owned `process_evaluation_source_slice` +- SDK runner, result logging, trace loading, and metrics work are adapters around the shared SDK runtime contract +- backend execution now delegates to the SDK processor through backend-specific scenario, result, cache, status, trace, and workflow adapters + +Still missing: + +- unified `process(run, slice)` role exposed as a public API or service operation +- topological execution over planned cells +- idempotent probe-before-write behavior +- consistent error-as-result behavior +- shared metrics refresh policy after processing +- clear separation between execution and persistence adapters +- public API/service operation shape for invoking the SDK-owned source-slice processor by tensor slice + +## Runnable Execution Gaps + +The current worker loops still call step-specific invocation helpers directly. + +Known debt: + +- application execution still uses legacy batch invocation helper paths +- evaluator execution assembles workflow invocation requests separately +- cache lookup, trace validation, link/reference construction, and error-to-result conversion are repeated around those calls +- there is no single runnable-step executor that can handle application and evaluator steps through the same contract + +Needed: + +- `RunnableStepExecutor` interface for any auto runnable step +- application-step adapter that can initially wrap the current application invocation path +- evaluator-step adapter that can initially wrap workflow invocation +- shared request/context builder for references, links, inputs, trace, outputs, and parameters +- shared trace validation and result normalization +- migration path to deprecate legacy LLM app batch helper functions after parity is proven + +This should be treated as part of unification, not as a later cleanup. Otherwise the new loop would only centralize iteration while preserving the most brittle execution boundary. + +## Tensor Operation Gaps + +The intended tensor identity exists in storage and query models, but the operation model is incomplete. + +Missing or partial: + +- `TensorSlice` model across backend, SDK, and frontend +- slice-aware `probe` +- slice-aware `populate` +- slice-aware `prune` +- slice-aware `process` +- partial retry/fill-missing workflows +- repeat slot materialization as a shared planner primitive + +Existing APIs are mostly per-entity or full-run oriented. + +## Repeat And Fan-Out Gaps + +`repeat_idx` exists in result identity and current backend loops now expand it in the inspected paths. + +Current gaps: + +- repeat expansion is duplicated across loops +- queue repeats still also carry assignment semantics, so execution repeats and assignment lanes need an explicit shared contract +- `is_split` is enforced through helpers in some paths, but topology validation is still dispatch-specific +- no shared planner decides whether application or evaluator steps fan out + +Needed: + +- repeat-aware result-slot planner +- topology-specific fan-out validation +- deterministic repeat-slot binding for reused traces +- tests for full, partial, and zero cache hits under repeats + +## Cache Gaps + +Hash-based trace reuse is implemented in current backend worker paths, but not centralized. + +Still missing: + +- one shared cache-resolution stage used by every runnable step +- one explicit per-slot cache binding object +- parity tests proving all loops use the same cache rules +- a documented project-scoped cross-run reuse policy + +The current code uses `is_cached`; older docs that say `reuse_traces` should be treated as stale terminology unless an external compatibility need exists. + +## Origin Gaps + +`auto`, `human`, and `custom` origins exist in the current backend model, and human/custom pending behavior is present in several loops. + +Still missing or incomplete: + +- common pending/manual result lifecycle +- consistent frontend/backend origin naming +- external custom-populate contract for custom steps +- annotation queue progress/status semantics layered over evaluation results without duplicating task state + +Verify frontend/generated-client naming before changing UI code; backend type truth is `auto`. + +## Annotation Queue Layer Gaps + +Annotation Queue v2 identifies product/API gaps adjacent to unified loop execution: + +- convenience API for queue creation from traces and testsets +- annotator inbox/list view across queues +- per-item progress computed from evaluation results +- explicit export/write-back flow for testset-sourced annotation queues +- clear distinction between backing infrastructure status and consumer-facing task status +- UI that uses queue assignment instead of allowing annotators to annotate any scenario + +These should be built on top of the unified planner/source resolver/tensor result model, not as a separate runtime. + +## Graph Mutation Gaps + +Steps are stored in run data, but graph operations are not first-class enough. + +Missing: + +- `add_step` endpoint/service operation +- `remove_step` endpoint/service operation +- graph validation outside setup functions +- tensor pruning cascade when removing a step +- explicit immutable-reference policy in code paths +- UI/API affordances for managing steps after creation + +Without these, graph changes require specialized setup edits or recreation. + +## Flag Gaps + +Flags are consistently modeled in the current backend types, but old docs and possibly callers still use stale names. + +Current canonical backend flags include: + +- `is_live` +- `is_active` +- `is_cached` +- `is_split` +- `is_queue` +- `repeats` +- `is_closed` + +Compatibility names to reconcile: + +- `reuse_traces` vs `is_cached` +- `repeat_target` vs `is_split` +- `allow_decrease_repeats` if repeat count becomes mutable + +Still missing: + +- first-class constrained `set_flag` +- validation when flags conflict with topology +- end-to-end propagation through setup, run fetch, queue creation, SDK/local execution, and frontend state + +## Topology Gaps + +Supported topologies are implicit in dispatch logic. + +Missing: + +- explicit topology validation table in code +- structured error messages for unsupported combinations +- explicit rejection for not-planned shapes such as multiple application steps, mixed-source queues, and live testset runs +- future extension point for query -> application flows +- future extension point for testset -> evaluator flows + +Potentially useful future shapes: + +- `query -> application -> evaluator`, with query traces adapted as input data rather than application links +- `testset -> evaluator`, with an explicit evaluator testcase-only input contract + +Not planned for now: + +- multiple application steps in one worker-dispatched run +- mixed query/testset source families in one queue +- live testset evaluation + +The immediate goal should not be to support every theoretical graph. It should be to reject unsupported graphs through one planner, and make adding support localized. + +## API Gaps + +> The full operation surface and the implementation status of each op +> (done / partial / deferred) is catalogued in [`operations.md`](./operations.md). + +Missing or incomplete API surface: + +- graph-oriented create request +- `process(slice)` +- `probe(slice)` +- `prune(slice)` +- `populate(slice, results)` for bulk/slice writes +- `set_flag` +- response payloads that expose resolved source items and pending cells consistently + +Existing APIs should remain as compatibility wrappers while the canonical surface is introduced. + +## SDK Gaps + +The SDK should own the shared runtime contract. The preview loop can keep its +public setup API, but orchestration should move behind SDK runtime planning and +SDK-specific execution/persistence adapters. Backend workers should consume the +same SDK runtime models through backend-specific adapters. + +Missing: + +- remote API persistence adapter +- slice-aware processing +- probe-before-write +- cache parity with backend +- stable step key strategy aligned with backend graph steps +- removal of duplicate backend planner/topology logic once migration coverage is sufficient + +The desired state is not "SDK calls backend worker for everything." It is "SDK and backend share the same loop contract with different persistence/execution adapters." + +## Frontend Gaps + +Missing: + +- explicit graph builder/step management model +- TensorSlice UI concepts for retry, prune, and fill missing +- unified origin naming +- flag editing beyond current implicit flows +- display of pending human/custom cells across query, testset, and queue runs +- source-aware queue creation UI if that product path is enabled + +Frontend work can follow backend planner/API stabilization. + +## Testing Gaps + +Needed test coverage: + +- source resolver outputs for query, testset, trace IDs, testcase IDs +- topology validation success and failure cases +- repeat slot materialization for every supported topology +- `is_split=true` and `is_split=false` on testset -> application -> evaluator +- evaluator-only repeat fan-out for query and queue runs +- cache full hit, partial hit, and miss +- cross-run trace reuse +- human/custom pending cells in query-backed runs +- source-aware queues preserving query/testset revision references +- existing direct queue behavior remains unchanged +- SDK/backend parity for the same planned graph + +## Documentation Gaps + +Needed docs after implementation starts: + +- canonical source matrix +- topology validation matrix +- flag semantics and compatibility names +- manual/custom origin lifecycle +- cache and repeat behavior +- migration guide from specialized setup APIs to canonical graph creation diff --git a/docs/designs/eval-loops/iteration-patterns.md b/docs/designs/unified-eval-loops/legacy-iteration-patterns.md similarity index 97% rename from docs/designs/eval-loops/iteration-patterns.md rename to docs/designs/unified-eval-loops/legacy-iteration-patterns.md index c2991bf2ff..b6526a67f6 100644 --- a/docs/designs/eval-loops/iteration-patterns.md +++ b/docs/designs/unified-eval-loops/legacy-iteration-patterns.md @@ -1,5 +1,12 @@ # Evaluation Iteration Patterns +> **Status: Frozen legacy analysis (2026-02-16).** Moved here from the now-removed +> `docs/designs/eval-loops/`. The canonical unified design lives in +> [`proposal.md`](./proposal.md) and [`plan.md`](./plan.md); naming here (e.g. +> `repeat_target` enum, 8 `has_*` flags) predates and is superseded by those docs. +> Retained because the legacy loop-family nesting analysis below is not replicated +> elsewhere and is useful for assessing migration risk. + **Created:** 2026-02-16 **Purpose:** Document the iteration patterns used to execute evaluation graphs and populate evaluation tensors across API and SDK diff --git a/docs/designs/eval-loops/refactoring-analysis.md b/docs/designs/unified-eval-loops/legacy-refactoring-analysis.md similarity index 99% rename from docs/designs/eval-loops/refactoring-analysis.md rename to docs/designs/unified-eval-loops/legacy-refactoring-analysis.md index 8a53018076..c51e9d5a29 100644 --- a/docs/designs/eval-loops/refactoring-analysis.md +++ b/docs/designs/unified-eval-loops/legacy-refactoring-analysis.md @@ -1,10 +1,16 @@ # Evaluation System - Refactoring Analysis +> **Status: Frozen legacy analysis (2026-02-16).** Moved here from the now-removed +> `docs/designs/eval-loops/`. The canonical unified design lives in +> [`proposal.md`](./proposal.md) and [`plan.md`](./plan.md). Retained for the +> side-by-side loop comparison and the concrete "pure functions to extract" +> signatures, which are not replicated in the current design docs. + **Created:** 2026-02-16 **Purpose:** Detailed analysis of current code and concrete refactoring steps **Related:** -- [Current State - Iteration Patterns](./iteration-patterns.md) -- [Desired Architecture](./desired-architecture.md) +- [Current State - Iteration Patterns](./legacy-iteration-patterns.md) +- Desired Architecture — _removed; superseded by_ [`proposal.md`](./proposal.md) --- diff --git a/docs/designs/unified-eval-loops/operations.md b/docs/designs/unified-eval-loops/operations.md new file mode 100644 index 0000000000..32be681489 --- /dev/null +++ b/docs/designs/unified-eval-loops/operations.md @@ -0,0 +1,89 @@ +# Evaluation Operation Surface + +## Purpose + +This document is the single catalogue of the first-class evaluation operations the +unified-eval-loops design intends to expose, and the implementation status of each. + +It exists so the explicit operations (`add_step`, `remove_step`, `add_scenario`, +`add_result`, `refresh_metrics`, …) are tracked in one place. Most are **deferred** +and will be implemented later; this doc records what is done, what is not, and where +each is specified. + +It does not redefine semantics — see the canonical docs: + +- Operation list and direction: [`proposal.md`](./proposal.md) §"Operation API Direction" (lines ~366-383) +- Gaps and deferred surface: [`gap.md`](./gap.md) §"API Gaps", §"SDK Gaps" +- Destructive removal lifecycle: [`step-removal-semantics.md`](./step-removal-semantics.md) + +## Model + +The graph defines the tensor shape; operations mutate either the **graph** +(`add_step` / `remove_step` …) or the **tensor** (`populate` / `prune` …). The two +op families stay symmetric: + +```text +graph: add_step / remove_step / add_scenario / remove_scenario +tensor: probe / populate / prune / process +run: refresh_metrics / set_flag +``` + +A key invariant (per `step-removal-semantics.md`): the **stored graph and stored +tensor have the same shape**. Adding a step adds a tensor column dimension; removing +a step prunes that column's cells. `create_run` is conceptually "edit from an empty +graph", so create and edit share one reconciliation path. + +## Status legend + +- **done** — implemented and reachable through the service/router (or runtime). +- **partial** — exists but does not yet meet the documented contract. +- **deferred** — specified here/in the proposal, not yet implemented. To be done later. + +## Operations + +Every first-class op below is reachable over HTTP under `/api/simple/evaluations/{id}/…` +with an explicit `operation_id`, so the regenerated Fern client exposes a method per op. + +### Graph-shape operations + +| Operation | Status | Endpoint (`operation_id`) | Where | +| --- | --- | --- | --- | +| `add_steps` | **done** | `POST /{id}/steps/add` (`add_steps`) | `SimpleEvaluationsService.add_steps` — appends step columns to `run.data.steps` (idempotent on key); cells fill on the next `process`. The legacy folded-into-`edit_run` path still works too. proposal.md:370. | +| `remove_steps` | **done** | `POST /{id}/steps/remove` (`remove_steps`) | `SimpleEvaluationsService.remove_steps` — drops step columns by key. Destructive cell removal via the reconcile path (`_reconcile_run` / `_prune_removed_steps`) is also driven by omitting a step on `edit_run`; semantics in `step-removal-semantics.md`; closed UEL-014. | +| `add_scenarios` | **done** | `POST /{id}/scenarios/add` (`add_scenarios`) | `SimpleEvaluationsService.add_scenarios` — appends N skeleton rows (height). `populate` writes their input cells, `process` plans/executes (`process` never mints scenarios). proposal.md:372. | +| `remove_scenarios` | **done** | `POST /{id}/scenarios/remove` (`remove_scenarios`) | `SimpleEvaluationsService.remove_scenarios` — drops scenario rows and their cells (delegates to `delete_scenarios`). proposal.md:373. | +| `set_repeats` | **done** | `POST /{id}/repeats` (`set_repeats`) | `SimpleEvaluationsService.set_repeats` — sets the run's repeat (depth) dimension; existing cells untouched, new repeat slots fill on the next `process`. | + +### Tensor operations + +| Operation | Status | Endpoint (`operation_id`) | Where | +| --- | --- | --- | --- | +| `probe(slice)` | **done** | `POST /{id}/probe` (`probe_slice`) | `SimpleEvaluationsService.probe_slice` → `TensorSliceOperations.probe` / `probe_summary` in `runtime/tensor.py`. | +| `populate(slice, results)` | **done** | `POST /{id}/populate` (`populate_slice`) | `SimpleEvaluationsService.populate_slice` → `TensorSliceOperations.populate`. Low-level cell CRUD (`create_results` + `/evaluations/results/`) still underlies it. | +| `prune(slice)` | **done** | `POST /{id}/prune` (`prune_slice`) | `SimpleEvaluationsService.prune_slice` → `TensorSliceOperations.prune` (removes cells + refreshes metrics over the scope). Also driven by `remove_step` via `_prune_removed_steps`. | +| `process(slice)` | **done** | `POST /{id}/process` (`process_slice`) | `SimpleEvaluationsService.dispatch_tensor_slice` → taskiq → `TensorSliceOperations.process`, which delegates to the injected `SliceProcessor` (`APISliceProcessor` in `tasks/processor.py`). Re-executes the runnable cells for the EXISTING scenarios a `TensorSlice` addresses — rebuilds each scenario's source binding from its stored input cell, re-hydrates trace/testcase context, plans from the run's current graph (so modified steps re-run), runs the cache-aware runners (hashed-trace reuse), populates, refreshes metrics, and finalizes. With no processor wired it raises rather than silently refreshing. Closed UEL-015. proposal.md:160-208. | + +### Run operations + +| Operation | Status | Where / Tracking | +| --- | --- | --- | +| `refresh_metrics(scope)` | **done** | `EvaluationsService.refresh_metrics` + `/evaluations/metrics/refresh`. Also invoked by every tensor-write op (populate / process / prune) over the touched scope. | +| `set_flag` | deferred (this branch) | No first-class constrained `set_flag`. Flags are currently re-derived from the graph (`_make_run_flags`) and reconciled (`is_queue` via `_reconcile_default_queue`). A constrained setter lands later in this branch. proposal.md:379; gap.md:302, 340. | +| run start / stop / close / open | **done** | `create_run`/`close_run`/`open_run` + `/{id}/start`,`/stop`,`/close`,`/open` routes. | + +## Still deferred + +Only `set_flag` remains deferred on this branch (flags are re-derived from the graph +today; a constrained setter lands later — proposal.md:379; gap.md:302, 340). When +implemented it should: + +- follow the AGENTS.md domain conventions (`apis/fastapi/evaluations/router.py` + + `core/evaluations/service.py`), with typed domain exceptions in + `core/evaluations/types.py`; +- reuse the shared reconcile path so the mutation re-derives flags + default-queue + eligibility consistently with create/edit. + +The graph-shape and tensor ops are all wired end to end (service → HTTP → +Fern client). They are slice-shaped where applicable (`TensorSlice`) so setup, retry, +queue assignment, manual annotation, live ticks, and SDK/local runs share one tensor +contract (proposal.md:381). diff --git a/docs/designs/unified-eval-loops/origin-execution-model.md b/docs/designs/unified-eval-loops/origin-execution-model.md new file mode 100644 index 0000000000..92acaac6cc --- /dev/null +++ b/docs/designs/unified-eval-loops/origin-execution-model.md @@ -0,0 +1,171 @@ +# Origin execution model: today (`human`/`auto`/`custom`) and the future (`web`/`api`/`sdk`/`custom`) + +Status: design / forward-looking. The "today" section documents shipped +behavior; the "future" section is a proposal, not yet implemented. + +## What `origin` means + +Every evaluation step (`input`, `invocation`, `annotation`) carries an +`origin`. `origin` answers exactly one question: **who is responsible for +executing this step?** It is not "who created the run", not "who started it", +not "who dispatched the job" — only who *runs the work in a given slice*. + +- Type (SDK): `agenta/sdk/models/evaluations.py` — `Origin = Literal["custom", "human", "auto"]` +- Type (API): `api/oss/src/core/evaluations/types.py` — same literal. + +The evaluation runtime (`EvaluationPlanner` + the slice processor) is shared: +the **same** code runs inside the backend worker and inside the SDK +`evaluate()` loop. So `origin` is the only thing that lets one body of code +decide "is this step mine to run, or someone else's?". Two hosts read the same +plan and each runs only its own steps. + +## Today: `human` / `auto` / `custom` + +| origin | Who executes the step | Who reads it | +| -------- | ---------------------------------------------- | ----------------------------------------- | +| `human` | A person, via the web frontend | Web only | +| `auto` | The runtime host (backend worker, or the SDK) | Backend worker; SDK when it is the host | +| `custom` | The SDK / an external client | The SDK runtime, **only** when it is that client | + +Read carefully: `auto` and `custom` are **not** symmetric. + +- `auto` is read by *whatever runtime is currently hosting the run* — the + backend worker when the backend runs it, the SDK runtime when `evaluate()` + runs it. "auto" = "the runtime should run this". +- `custom` means "an external client runs this." The only situation where + `custom` resolves to "run it here" is when the SDK `evaluate()` loop **is** + that external client — i.e. the evaluation was both created and executed in + the SDK via `aevaluate()`. In every other context (a `custom` step on a run + the web created, or one the backend picked up), `custom` means "not mine" and + the step is left for whoever the external client is. + +### Where this is enforced in code + +- **Planner** — `agenta/sdk/evaluations/runtime/planner.py`, + `EvaluationPlanner._runnable_cells`: + + ```python + manual_origins = {"human"} if execute_custom else {"human", "custom"} + is_manual_annotation = step.type == "annotation" and step.origin in manual_origins + # should_execute = not is_manual_annotation + ``` + + `execute_custom` is the context flag that says "I am the external client for + custom steps." It is threaded `aevaluate() -> process_evaluation_source_slice + -> EvaluationPlanner.plan/plan_bindings -> _runnable_cells`. + +- **SDK host** — `agenta/sdk/evaluations/preview/evaluate.py`: wires a local + evaluator runner for `origin != "human"` (so auto **and** custom), and calls + the processor with `execute_custom=True`. The SDK is the custom client. + +- **Backend host** — `api/oss/src/core/evaluations/tasks/processor.py`: wires + runners only for annotation steps with `origin not in {"human", "custom"}` + (auto only), and never sets `execute_custom` (defaults `False`). The backend + leaves both human and custom alone. + + Confirmed in worker logs: a `custom` annotation processed by the backend + resolves to `runner_keys=[]` and the slice completes with the step left + pending — the backend does not run it. + +### How the web labels runs (`custom` = "SDK") + +The web does not store an authoritative "kind" — it **derives** it from the +run's step origins (`web/oss/src/lib/evaluations/utils/evaluationKind.ts`, +`deriveEvaluationKind`), priority order: + +1. online (live / source-backed) +2. `human` — any annotation step with `origin="human"` +3. `custom` — any step with `origin="custom"` +4. `auto` — default + +So in the UI a run containing `custom` steps is shown as the "SDK"/custom kind. +This is *load-bearing*: changing local-callable SDK evaluators from `custom` to +`auto` would silently reclassify every SDK evaluation as a backend `auto` run in +the UI. That is why local-callable evaluators created by `evaluate()` keep +`origin="custom"`, even though the SDK is the one running them. + +### The tension this resolves + +`custom` is overloaded: it means both "the SDK ran this" (when `evaluate()` is +the host) and "an external client owns this" (everywhere else). The +`execute_custom` flag is what disambiguates the two at runtime without changing +the stored origin — so the web's `custom`→"SDK" classification stays intact +while the backend correctly refuses to run custom steps. + +## Future direction (proposal): `web` / `api` / `sdk` / `custom` + +The cleaner long-term model replaces the *role-by-actor* origins with +*role-by-executor* origins. `origin` would name the executor that owns the step: + +| origin | Who executes the step | +| -------- | ---------------------------------------------- | +| `web` | The web frontend (a person, in the browser) | +| `api` | The backend behind the API | +| `sdk` | An Agenta SDK runtime (Python today, others later) | +| `custom` | Anything else — external scripts, third-party code; **no one in the platform picks it up** | + +Each executor runs exactly the steps stamped with its own name; everyone else +treats the rest as no-ops. `custom` becomes a true "nobody here runs this" — +unlike today, where `custom` sometimes means "the SDK runs this." + +This is strictly about **who runs a step in a slice**, independent of who +dispatched the job, who started the run, or who created it. + +### Why this is cleaner — and the open problem it raises + +- It removes the `auto`/`custom` overload. `auto` today secretly means "the + current runtime host", which is ambiguous once there is more than one runtime. +- It removes the need for the `execute_custom` context flag: an `sdk` step is + run by the SDK, full stop; an `api` step by the backend. No "am I the client + for this?" question. + +The open problem (and why we did **not** adopt this now): **the SDK runtime is +unified with the backend runtime.** The same planner/processor runs in both. If +both backend and SDK are "the runtime", what distinguishes an `api` step from an +`sdk` step beyond a label? And once there is a TypeScript SDK — a *different* +codebase also running evaluations — is that `sdk` too, or does it need its own +origin? A second candidate model ("web vs. runtime", only two values) collapses +under exactly this: it cannot tell a backend-run step from an SDK-run step, +because they share the runtime. The four-value `web/api/sdk/custom` model is the +one that survives multiple SDKs, at the cost of every executor having to know +its own identity. + +### Implications + +**Schema.** `origin` is currently `Literal["custom", "human", "auto"]` in both +`sdk/models/evaluations.py` and `core/evaluations/types.py`, stored inside the +run's `data.steps[].origin` (a `json` column, not `jsonb`). Moving to +`web/api/sdk/custom` is a value-domain change, not a column change — no DDL, but +every producer and reader of `origin` must agree on the new vocabulary +simultaneously, which is the hard part. + +**Data migration.** Existing rows carry `human`/`auto`/`custom`. A migration +would remap stored step origins: + +- `human` → `web` +- `auto` → `api` (the backend was the implied runtime host for stored auto runs) +- `custom` → split: SDK-run customs → `sdk`; everything else → `custom`. + This split is the lossy one — historically `custom` did not record *which* + external client ran it. A backfill can only infer "this was an SDK run" from + surrounding signal (e.g. the run was created via the SDK upsert path / has SDK + meta), and must fall back to `custom` when it cannot prove `sdk`. + +**Backward compatibility.** Run during the transition: + +- Keep accepting the legacy literals on write; normalize to the new domain at + the boundary (an adapter mapping `human/auto/custom` → `web/api/sdk/custom`), + the same shape as the existing legacy adapters in the API (cf. the + `__dedup_id__`/`testcase_dedup_id` normalization). +- The web's `deriveEvaluationKind` already special-cases `custom`; it would gain + `sdk` and treat `web`/`api` as the new human/auto. Keep reading the legacy + values for old runs. +- Dual-read for at least one deprecation window: planners must treat `api` and + legacy `auto` identically on the backend, `sdk` and "legacy custom that the + SDK runs" identically in the SDK, until all stored runs are migrated. + +**Net.** The future model is cleaner conceptually but requires (a) every +executor to self-identify, (b) a lossy `custom`→`sdk` backfill, and (c) a +dual-read compatibility window. The shipped `human/auto/custom` + `execute_custom` +model is the pragmatic interim: it fixes the actual bug (the SDK not running its +own custom evaluators) without a vocabulary migration, and keeps the web's +existing `custom`→"SDK" classification working unchanged. diff --git a/docs/designs/unified-eval-loops/plan.md b/docs/designs/unified-eval-loops/plan.md new file mode 100644 index 0000000000..2ccb00ab5e --- /dev/null +++ b/docs/designs/unified-eval-loops/plan.md @@ -0,0 +1,199 @@ +# Plan + +## Goal + +Move from multiple specialized setup and execution functions to unified evaluation loop(s) built around: + +- source resolvers +- run graph steps +- tensor slices +- repeat-aware planning +- origin-aware execution +- runnable-step execution +- adapter-based persistence + +This plan describes required work, not phases or timeline. + +## Baseline Inventory + +1. Lock down the current behavior with code references and tests before changing semantics. +2. Treat these as implemented baseline behavior: + - `is_cached` + - `is_split` + - current `is_queue` + - `repeats` + - repeat-indexed result creation in current backend workers + - hash-based cache helpers and worker integration + - source-aware queue creation and source batch dispatch + - human/custom pending behavior in query/queue-related paths +3. Maintain a parity matrix from current tests: + - `test_cache_split_utils.py` + - `test_query_eval_loops.py` + - `test_run_flags.py` + - queue assignment and queue DAO tests + - acceptance tests for evaluation steps/runs/queues/results + +## Vocabulary And Flags + +1. Use current backend names as canonical: + - `is_cached` + - `is_split` + - `repeats` +2. Normalize origin values: + - `auto` + - `human` + - `custom` +3. Add explicit source-family flags: + - `has_queries` + - `has_testsets` + - `has_traces` + - `has_testcases` +4. Separate source-family classification from simple-queue eligibility. +5. Redefine target `run.flags.is_queue` as: + +```text +active default queue exists + active human evaluator work exists +``` + +6. Document topology validation rules in one table used by implementation and tests. + +## Shared Runtime Models + +Introduce or consolidate shared internal models: + +1. `InputSourceSpec` +2. `ResolvedSourceItem` +3. `ScenarioBinding` +4. `EvaluationStep` +5. `TensorSlice` +6. `PlannedCell` +7. `ExecutionPlan` +8. `ProcessSummary` + +The common runtime contract should live in the SDK so SDK-local evaluations and API workers share the same planner/topology/result-cell model. Backend code should keep API-specific source, DAO, workflow-service, and worker-dispatch adapters in backend modules. + +## Source Resolution + +Create resolver interfaces that cover: + +1. query revision -> trace refs for live windows +2. query revision -> trace refs for source-backed queues +3. testset revision -> testcase refs for source-backed queues +4. testset revision -> testcase payloads for batch testset/invocation +5. direct trace IDs -> trace refs +6. direct testcase IDs -> testcase refs + +Resolver requirements: + +- preserve existing source behavior +- own live query windowing +- preserve original source references in input steps +- reject unsupported mixed-source combinations explicitly +- expose source-family flags consistently + +## Tensor Slice Operations + +Add or adapt backend service operations around existing CRUD: + +1. `probe(slice)` +2. `populate(slice, results)` +3. `prune(slice)` +4. `process(slice)` + +Requirements: + +- slice dimensions support all/none/explicit selections +- `probe` identifies missing, success, failure, and any cells +- `populate` writes by `scenario_id + step_key + repeat_idx` +- `prune` deletes by slice and refreshes affected metrics + +## Planner + +Implement planner logic that produces result slots before execution. + +Planner responsibilities: + +1. validate topology +2. order steps +3. materialize scenario bindings +4. create input cells +5. expand repeat slots +6. decide fan-out boundary +7. bind upstream context +8. mark `human` and `custom` cells pending +9. select `auto` cells for execution + +Planner requirements: + +- one planned cell exists for every required repeat slot +- unsupported topologies fail with structured validation errors +- human/custom steps are planned as pending rather than silently skipped + +## Cache Resolution + +Reuse the existing cache helpers: + +1. `make_hash(...)` +2. `fetch_traces_by_hash(...)` +3. `select_traces_for_reuse(...)` +4. `plan_missing_traces(...)` +5. per-slot trace binding + +Requirements: + +- cache lookup is skipped when `is_cached=false` +- full cache hit invokes nothing +- partial cache hit invokes only missing slots +- misses invoke all required slots +- reused and newly generated traces populate identical tensor cells + +## Runnable-Step Execution + +Add a runnable execution boundary for any auto step whose type maps to a runnable. + +Initial adapters: + +1. SDK workflow-runner protocols for application/evaluator execution +2. SDK/local adapters wrapping decorator/service endpoint execution +3. API adapters wrapping the current backend workflow invocation path +4. API application adapter wrapping the current legacy batch invocation path + +The interface should own: + +- request construction from step references and upstream bindings +- cache resolver integration +- invocation +- trace fetch/validation +- normalized `StepExecutionResult` + +## Queue Integration + +1. Treat default queues as persisted human-work views over the tensor, not orchestration. +2. Add `queue.flags.is_default` to identify the canonical queue. +3. Keep default queues open over scenarios, steps, and assignments. +4. Let source-family flags describe where scenarios come from. +5. Let `run.flags.is_queue` describe simple-queue eligibility. +6. Ensure queue eligibility depends on active human steps and active default queue lifecycle. + +## Mutation Semantics + +1. Decide whether ordinary evaluator removal is archival/deactivation rather than destructive deletion. +2. If history must remain visible, represent active versus archived step state in the graph model. +3. Make planner defaults operate on active steps. +4. Reserve hard remove/prune for explicit destructive cleanup. +5. Keep queue eligibility tied to active human steps. + +## Verification + +Add or preserve coverage for: + +1. topology classification +2. resolver behavior +3. repeat slot creation +4. cache reuse +5. human/custom pending behavior +6. query/testset/direct trace/direct testcase source families +7. source-family validation +8. tensor slice probe/populate/prune/process behavior +9. queue/default-queue integration semantics +10. active-versus-archived step behavior once chosen diff --git a/docs/designs/unified-eval-loops/proposal.md b/docs/designs/unified-eval-loops/proposal.md new file mode 100644 index 0000000000..ac5e2bcca1 --- /dev/null +++ b/docs/designs/unified-eval-loops/proposal.md @@ -0,0 +1,476 @@ +# Proposal + +## Goal + +Introduce unified evaluation loop(s) that avoid separate setup and execution functions for every evaluation shape while preserving the capabilities already implemented in the current backend: + +- input steps +- application and evaluator origins +- evaluation run flags +- input sources +- evaluation graph steps +- repeat and cache behavior through `repeats`, `is_cached`, and `is_split` +- live, batch, queue, and SDK/local execution contexts + +This proposal does not require every source to become the same thing. It requires every source to enter the same planning and tensor execution contract. + +The current code has already implemented several parts that older docs described as missing: + +- `EvaluationRunFlags.is_cached` +- `EvaluationRunFlags.is_split` +- `EvaluationRunFlags.is_queue` +- `EvaluationRunData.repeats` +- repeat helper functions in `evaluations/utils.py` +- hash/cache helper functions in `evaluations/utils.py` +- source-aware queue creation from query/testset-backed sources +- live and batch query human/custom pending behavior +- repeat-aware result creation in the inspected backend worker loops + +The proposal is therefore a unification/refactor proposal, not a first implementation of those behaviors. + +## Design Principle + +Separate source resolution, graph planning, execution, and tensor persistence. + +```text +setup request + -> source resolver + -> run graph + -> scenario materializer + -> execution planner + -> step executor + -> tensor writer + -> metrics refresh +``` + +Each current loop family becomes a configuration of this pipeline instead of a separate handwritten loop. + +## Canonical Concepts + +### Run Graph + +A run graph is a list of immutable step definitions: + +```python +class EvaluationStep: + key: str + type: Literal["input", "invocation", "annotation"] + origin: Literal["auto", "human", "custom"] + references: dict + inputs: list[StepInput] | None +``` + +Step references point to concrete revisions or direct-source descriptors. Editing a step means removing it and adding a new step. + +### Tensor Cell + +Every produced or pending result targets: + +```text +run_id + scenario_id + step_key + repeat_idx +``` + +All execution, retry, prune, cache binding, and manual annotation work should address this coordinate. + +### Tensor Slice + +Use one slice model for read, write, delete, and processing operations: + +```python +class TensorSlice: + scenarios: Literal["all", "none"] | list[UUID] + steps: Literal["all", "none"] | list[str] + repeats: Literal["all", "none"] | list[int] +``` + +The same slice shape should power: + +- `probe` +- `populate` +- `prune` +- `process` +- retry failed cells +- fill missing cells +- re-run a single evaluator +- materialize new repeat slots + +## Source Resolver Layer + +Input sources should be modeled as descriptors that resolve into concrete source items. + +| Descriptor | Resolver output | Scenario source | +|---|---|---| +| query revision | trace refs | queried traces | +| testset revision | testcase refs | testcases | +| direct trace source | trace refs | queued traces | +| direct testcase source | testcase refs | queued testcases | + +The resolver is responsible for source-specific rules: + +- live query windows +- batch query snapshots +- testset revision loading +- direct item validation +- source-aware queue expansion +- preserving original source references in input steps + +The executor should not care whether a trace came from live query, batch query, or a queue. It should receive concrete scenario bindings. + +## Annotation Queue Convenience Layer + +Annotation Queue v2 should be treated as a consumer-facing layer over the same unified evaluation infrastructure. + +Principles: + +- `EvaluationRun`, `EvaluationScenario`, `EvaluationResult`, and `EvaluationQueue` remain the backing entities. +- The annotation queue API hides run/scenario/result setup for trace and testset annotation use cases. +- Queue assignment remains based on `EvaluationQueue.data.user_ids`, optional `scenario_ids`, optional `step_keys`, and result `repeat_idx`. +- Queue creation from traces/testsets should translate into canonical source specs and graph steps, then use the same source resolver and planner as evaluation runs. +- Annotation submission can continue to create annotation traces and link them to evaluation results. + +Unified eval loops should provide the infrastructure contract for this layer. They should not replace the annotation queue convenience API. + +## Planner Layer + +The planner converts a graph and concrete scenarios into execution slots. + +```python +class PlannedCell: + scenario_id: UUID + step_key: str + repeat_idx: int + action: Literal["bind_input", "invoke", "pending", "skip"] + upstream: dict +``` + +Planner responsibilities: + +- validate topology +- derive step order +- materialize input cells +- decide repeat fan-out point +- compute required result slots +- bind upstream trace/testcase/application output context +- mark human/custom annotation cells as pending +- skip cells that are already successful when requested + +The planner is where topology-specific behavior belongs. + +## Execution Layer + +`process(slice)` executes planned `auto` cells only. + +```text +process(run, slice): + resolve concrete scenarios for input steps + plan cells for the requested slice + probe existing cells if skip-success is enabled + for each executable auto cell in dependency order: + resolve cache/reuse if enabled + invoke only missing work + populate result cells + create pending cells for human/custom work + refresh metrics for affected scope +``` + +The same executor can run in different contexts with different adapters: + +| Context | Adapter | +|---|---| +| backend worker | API source/DAO/workflow-service adapters | +| SDK/local | local decorator or remote service adapters | +| tests | in-memory adapter | +| frontend human annotation | direct `populate` adapter for submitted cells | + +The planner, topology classifier, and result-cell models should be SDK-owned so +SDK-local evaluation and backend workers use the same runtime contract. API code +should not fork the runtime; it should translate backend DTOs into SDK runtime +models and keep only backend-specific adapters beside the worker/service code. + +The backend implementation should have one scenario execution loop. Source +wrappers may still differ because live queries, query snapshots, direct queue +items, and testset rows resolve differently, but after resolution they should +all call one source-slice processor. That processor owns input cell creation, +application invocation, evaluator invocation, pending manual/custom cells, +cache resolution, metrics refresh, and run/scenario status updates. Batch +inference is therefore just the application-only graph shape, not a separate +loop. Task-level trace/testcase batch helpers are unnecessary once the slice +worker calls the source-slice processor directly; service/API wrapper methods +can remain for compatibility. + +The SDK should own the generic source-slice contract. SDK preview/local +execution can run through SDK-owned `process_evaluation_source_slice` now using +local decorator runners, SDK result logging, and SDK trace loading. The backend +processor should use that same contract with backend adapters for scenario +creation, result persistence, cache reuse, status updates, trace loading, and +workflow service execution. + +## Runnable Step Executor + +The unified loop should introduce a new runnable-step execution boundary rather than directly preserving the current helper calls inside each loop. + +Current application execution is still routed through legacy helper paths such as batch LLM app invocation. Those paths have accumulated patches and are not the right long-term abstraction. Evaluator execution is also assembled separately even though it has the same core shape: prepare a runnable request, bind upstream context, invoke or reuse a trace, validate the trace, and produce a result cell. + +Proposed contract: + +```python +class WorkflowRunner: + async def execute( + self, + request: WorkflowExecutionRequest, + ) -> WorkflowExecutionResult: + ... +``` + +`WorkflowExecutionResult` should be independent of whether the runnable was an application or evaluator: + +```python +class WorkflowExecutionResult: + status: EvaluationStatus + trace_id: str | None + span_id: str | None + hash_id: str | None + error: dict | None + outputs: dict | None +``` + +Responsibilities: + +- build the service request from step references and upstream bindings +- apply cache lookup/reuse when enabled +- invoke missing work +- fetch and validate traces where required +- normalize failures into result payloads +- return enough context for downstream steps + +The first implementation can wrap existing application and workflow services. +The SDK should expose the runner protocol and shared models. The API should +provide backend workflow-service and legacy batch-invocation adapters. The SDK +should provide local decorator and remote service adapters. This makes those +wrappers replaceable so the legacy batch helpers can be deprecated without +changing the planner, tensor operations, or queue APIs. + +## Origin Semantics + +`origin` controls who can populate a step: + +| Origin | Planner behavior | Executor behavior | +|---|---|---| +| `auto` | create executable cells | invoke and populate | +| `human` | create pending cells | do not invoke | +| `custom` | create pending cells or external-awaiting cells | do not invoke | + +This gives query-backed, testset-backed, and queue-backed runs the same pending/manual semantics. Current behavior where query-backed runs lack human/custom pending branches should become a topology limitation only until the planner supports those cells. + +## Flag Model + +Use the current backend flag set as canonical. Bridge older design names only where old clients or docs still mention them. + +| Canonical flag | Legacy design name | Purpose | +|---|---|---| +| `is_live` | `is_live` | periodic windowed source resolution | +| `is_active` | `is_active` | pause/resume live processing | +| `is_cached` | `reuse_traces` in older docs | enable hash-based trace reuse | +| `is_split` | `repeat_target` in older docs | select fan-out location where meaningful | +| `repeats` | `repeats` | number of repeat slots | +| `is_closed` | `is_closed` | block structural and tensor mutations | + +Recommended compatibility mapping: + +```text +repeat_target = "application" <=> is_split = true +repeat_target = "evaluator" <=> is_split = false +reuse_traces = true <=> is_cached = true +``` + +Do not introduce `reuse_traces` or `repeat_target` as new model fields unless there is a compatibility requirement. The current code already uses `is_cached` and `is_split`. + +## Topology Validation + +The unified planner should support every valid topology explicitly and reject invalid combinations before execution. + +| Topology | Valid? | Notes | +|---|---:|---| +| query -> evaluator | yes | live or batch; evaluator fan-out only | +| query -> human/custom evaluator | yes target | creates pending cells | +| query -> application -> evaluator | potentially useful | Requires query trace to application input adapter. Do not pass query traces as application `links`; that can make application traces look like annotations rather than invocations. | +| testset -> application -> evaluator | yes | app or evaluator fan-out | +| testset -> application | yes | Batch inference / batch invocation. Application fan-out only; no evaluator execution or evaluator metrics. | +| testset -> evaluator | potentially useful | Requires evaluator testcase-only contract. | +| direct trace -> evaluator | yes | queue trace shape | +| direct testcase -> evaluator | yes | queue testcase shape | +| mixed query + testset in one queue | not planned | Keep queues single-source-family for now. | +| multiple application steps | not planned | Use separate evaluations for A/B comparison for the foreseeable future. | +| live testset | not planned | Static sources do not make sense for live periodic evaluation. | + +The key shift is that unsupported shapes should fail through planner validation, not because there is no matching handwritten function. Potentially useful shapes should be explicitly modeled when implemented; not-planned shapes should stay rejected with clear errors. + +## Repeat Semantics + +Repeats are always represented as `repeat_idx` result slots. Fan-out determines which runnable step produces multiple traces. + +Rules: + +- query/queue trace/testcase evaluator-only runs fan out at evaluator steps. +- application-only runs, also called batch inference or batch invocation, fan out at application steps. +- testset -> application -> evaluator runs use `is_split`: + - `true`: application produces one trace per repeat, evaluators consume matching repeat traces + - `false`: application produces one trace, evaluators produce one trace per repeat +- if a topology has no application/evaluator boundary, `is_split` is ignored or rejected according to validation policy. + +## Cache Semantics + +Cache reuse is explicit through `is_cached`. + +At each runnable step: + +1. Compute the expected hash from step references and upstream links. +2. Fetch all candidate traces by hash. +3. Select deterministic traces for requested repeat slots. +4. Invoke missing slots only. +5. Populate the same tensor cells whether the trace was reused or newly generated. + +Cache lookup is step-local and already exists in the current backend loops inspected: + +- application steps reuse application traces +- evaluator steps reuse evaluator traces + +Cross-run reuse is structurally supported by project-scoped trace lookup by hash. The unified planner should reuse the existing helper functions instead of reimplementing this per loop. + +The cache resolver should sit inside or immediately beside the runnable-step executor so applications and evaluators use the same reuse semantics. + +## Setup API Direction + +Consolidate specialized setup functions behind one graph-oriented creation path plus convenience wrappers. + +Canonical create request: + +```python +class EvaluationCreate: + inputs: list[InputSourceSpec] + steps: list[ExecutableStepSpec] + flags: EvaluationFlags +``` + +Convenience wrappers may remain: + +- create auto testset evaluation +- create live query evaluation +- create annotation queue from traces +- create annotation queue from testcases +- create source-aware queue from query/testset +- create Annotation Queue v2 convenience flows from traces or testsets + +Several wrappers already use `_make_evaluation_run_data()`. The next step is to make that builder and its validations explicit enough that wrappers only translate into canonical graph/source specs and do not own separate graph semantics. + +## Operation API Direction + +Expose or normalize first-class operations: + +- `add_step` +- `remove_step` +- `add_scenario` +- `remove_scenario` +- `probe(slice)` +- `populate(slice, results)` +- `prune(slice)` +- `process(slice)` +- `refresh_metrics(scope)` +- `set_flag` + +This lets setup, retry, queue assignment, manual annotation, live ticks, and SDK/local runs share the same tensor contract. + +Some CRUD operations already exist in service/router form (`create_results`, `query_results`, `delete_results`, `refresh_metrics`, run start/stop, queue creation). The missing piece is a slice-shaped operation boundary and a shared `process(slice)` planner/executor. + +## Migration Strategy + +Do not rewrite all loops at once. Introduce the unified planner and adapters beside existing loops, then move topologies one at a time while preserving current behavior. + +Recommended order: + +1. Inventory current behavior and lock it with parity tests. +2. Define shared models: source descriptor, scenario binding, tensor slice, planned cell. +3. Extract current source resolution behavior into resolver interfaces. +4. Extract current repeat/cache planning into shared planner functions. +5. Introduce a runnable-step executor that initially wraps existing invocation services. +6. Route one simple topology through the planner, likely batch query or queue traces. +7. Move pending human/custom planning into the shared planner. +8. Move batch testset after repeat, cache, and runnable-executor parity are proven. +9. Move live query once windowed source resolution and idempotency are stable. +10. Collapse API-internal worker handlers to run/slice processors. +11. Share one backend source-slice processor across live query, batch query, queue slices, batch inference, and testset application evaluation. +12. Route SDK preview/local evaluation through SDK-owned source-slice processing with SDK-specific adapters. +13. Move backend execution onto the SDK source-slice contract through backend adapters that preserve current cache/result/status behavior. +14. Treat batch inference as the application-only shape of the testset application graph. +15. Retire specialized setup/execution branches after parity tests pass, leaving compatibility wrappers around the canonical processor. + +## Success Criteria + +The design succeeds when adding a new valid combination requires: + +- adding or extending a source resolver if the source is new +- adding or extending a step executor if the runnable is new +- adding planner validation if the topology is new + +It should not require creating a new end-to-end setup function and a new end-to-end execution loop. + +## Default Queue Integration + +Unified eval loops should treat default queues as a consumer-facing layer over the tensor, not as part of orchestration. + +```text +default queue = canonical persisted human-work view over the run tensor +``` + +A default queue is open over the run by default: + +```text +scenario_ids = None +step_keys = None +user_ids = None +``` + +The runtime should continue to own: + +- source resolution +- topology validation +- planning +- auto-step execution +- tensor persistence + +The queue layer should own: + +- human-work visibility +- assignment +- queue lifecycle +- simple queue interaction + +### Run flags + +The unified model should distinguish source family from queue eligibility. + +Source-family flags: + +- `has_queries` +- `has_testsets` +- `has_traces` +- `has_testcases` + +Queue eligibility flag: + +```text +run.flags.is_queue = active default queue exists + active human evaluator work exists +``` + +That keeps query-backed, testset-backed, trace-backed, and testcase-backed runs expressible through the same planner while allowing any human-bearing run with an active default queue to participate in the simple queue surface. + +### Step lifecycle + +The mutation model should distinguish lifecycle changes from destructive cleanup. + +If the product needs historical evaluator results to remain visible, then: + +- archive/deactivate should be the normal operation for steps that stop participating in future work +- remove/prune should remain available only for explicit destructive cleanup +- planner defaults should target active steps +- default queue eligibility should depend on active human steps diff --git a/docs/designs/unified-eval-loops/research.md b/docs/designs/unified-eval-loops/research.md new file mode 100644 index 0000000000..44e834b97c --- /dev/null +++ b/docs/designs/unified-eval-loops/research.md @@ -0,0 +1,344 @@ +# Research + +## Scope + +This document consolidates the existing evaluation-loop design notes and the +current implementation state from: + +- `application/docs/designs/eval-loops` +- `application/docs/designs/loops` +- `application/docs/designs/query-eval-loops` +- `application/docs/design/annotation-queue-v2` +- `application/api/oss/src/core/evaluations/types.py` +- `application/api/oss/src/core/evaluations/utils.py` +- `application/api/oss/src/core/evaluations/service.py` +- `application/api/oss/src/core/evaluations/tasks/source_slice.py` +- `application/api/oss/src/core/evaluations/tasks/query.py` +- `application/api/oss/tests/pytest/unit/evaluations/*` + +The goal is to identify the common execution model behind the current loop families and the places where setup and execution still diverge. + +## Current Loop Families + +The runtime historically had several explicit evaluation loop families: + +| Loop family | Source unit | Input steps | Application steps | Evaluator steps | Scenario represents | +|---|---|---:|---:|---:|---| +| Live query | trace returned by query | `1..N` query | `0` | `1..N` | queried trace | +| Batch query | trace returned by query | `1..N` query | `0` | `1..N` | queried trace | +| Batch testset | testcase | `1..N` testset | `1` | `1..N` | testcase | +| Batch inference / batch invocation | testcase | `1..N` testset | `1` | `0` | testcase | +| Queue traces | trace ID | `1` synthetic source | `0` | `1..N` | provided trace | +| Queue testcases | testcase ID | `1` synthetic source | `0` | `1..N` | provided testcase | +| SDK/local | runner-defined | run-defined | run-defined | run-defined | runner-defined | + +## Related Design: Annotation Queue v2 + +`application/docs/design/annotation-queue-v2` matters because annotation queues are one of the main consumers of unified evaluation loop infrastructure. + +The durable direction from that design is: + +- keep `EvaluationRun`, `EvaluationScenario`, `EvaluationResult`, and `EvaluationQueue` as backing infrastructure +- expose a simpler annotation queue API/UI that hides backing run/scenario/result setup +- do not introduce a separate annotation task runtime unless the existing entities prove insufficient +- map assignment/repeats through `EvaluationQueue.data.user_ids` and `EvaluationResult.repeat_idx` +- support trace and testset annotation as consumer-facing queue creation flows + +Some current-state claims in that older design are stale. In current code, source-aware queue creation and human/custom pending behavior are already partially implemented. The useful takeaway is the layering principle: annotation queues are a convenience layer over evaluation entities, not a separate execution model. + +The current worker dispatch only supports a subset of possible graphs: + +- `query(1..N) -> evaluator(1..N)` +- `testset(1..N) -> application(1) -> evaluator(1..N)` +- `testset(1..N) -> application(1)` +- `queue source(1) -> evaluator(1..N)` + +Unsupported by the current simple-evaluation worker dispatch, with product priority: + +| Unsupported shape | Priority | Notes | +|---|---|---| +| multiple application steps in one worker-dispatched run | not planned | A/B comparison can remain separate evaluations for the foreseeable future. | +| query inputs followed by application steps | potentially useful | The planner must treat query traces as input data, not as invocation links for the application step. If query trace IDs are placed in application `links`, the resulting application traces may be classified as annotations rather than invocations. | +| testset inputs followed directly by evaluator steps in non-queue mode | potentially useful | Useful for evaluators that can score testcase payloads without first invoking an application. Requires an explicit evaluator input contract. | +| mixed query and testset source families in one queue | not planned | Keep queues single-source-family for now. | +| live testset evaluation | not planned | Static testsets do not make sense as periodic live sources. | + +## Shared Runtime Model + +All loop families can be described with the same conceptual entities: + +- input source descriptors +- materialized scenarios +- executable steps +- result cells +- repeat slots +- execution flags + +The intended result identity is already visible in the persistence model: + +```text +scenario_id + step_key + repeat_idx +``` + +That identity is the core tensor coordinate. A unified loop should treat every execution as filling, probing, or pruning cells in this coordinate system. + +The current code already models this directly: + +- `EvaluationResult` has `scenario_id`, `step_key`, and `repeat_idx`. +- `EvaluationResultQuery` can filter by `scenario_ids`, `step_keys`, and `repeat_idxs`. +- worker loops now create repeat-indexed result rows in the main batch, queue, and live paths. + +## Steps + +Current run data already carries step definitions with: + +- `key` +- `type` +- `origin` +- `references` +- optional input links + +The shared step types are: + +| Step type | Meaning | Typical references | +|---|---|---| +| `input` | Source materialization | query revision, testset revision, direct trace/testcase source | +| `invocation` | Application/workflow execution | application revision, variant, workflow revision | +| `annotation` | Evaluator/judge/manual annotation | evaluator revision, annotation task | + +The shared origins are: + +| Origin | Populated by | Execution behavior | +|---|---|---| +| `auto` | Backend/SDK runner | invoked by `process` | +| `human` | UI/user annotation | runner creates or leaves pending work | +| `custom` | External/programmatic actor | runner creates or leaves pending work | + +Backend types use `auto`, `human`, and `custom`. Any frontend or generated-client naming drift should be treated as compatibility debt and verified before changing. + +## Input Sources + +The current code distinguishes source descriptors from concrete execution items. + +| Source descriptor | Concrete item | Current usage | +|---|---|---| +| query revision | trace | live query, batch query | +| testset revision | testcase | batch testset, batch inference / batch invocation | +| direct trace IDs | trace | queue traces | +| direct testcase IDs | testcase | queue testcases | + +Source-aware queue creation has been partially implemented. `SimpleQueuesService.create()` can accept query/testset-backed queue sources, builds run data through `_make_evaluation_run_data()`, preserves source revision references in input steps, and dispatches concrete trace/testcase batches through `_dispatch_source_batches()`. Direct trace/testcase queue additions remain supported. + +Annotation Queue v2 frames this as a consumer-facing convenience layer: users should be able to create annotation queues from traces or testsets without manually constructing the backing evaluation run, scenarios, results, and queue. + +## Scenario Semantics + +A scenario is a concrete source item inside a run. + +Depending on the source family, a scenario may represent: + +- a trace returned by a query +- a testcase from a testset revision +- a direct trace queue item +- a direct testcase queue item + +Live query scenarios additionally need temporal metadata such as timestamp and interval. Testset-backed online evaluation is intentionally unsupported because the same static testcases would be reprocessed every interval. + +## Application And Evaluator Boundaries + +Application steps produce application traces and outputs. Evaluator steps consume either: + +- an existing source trace from query/queue trace inputs +- an application trace/output from an invocation step +- testcase payload where the evaluator supports testcase-only input + +The current loop families differ mostly in which upstream object exists before evaluator execution. + +| Shape | Evaluator input | +|---|---| +| query -> evaluator | source trace | +| queue trace -> evaluator | source trace | +| testset -> application | no evaluator; output is application trace/result | +| testset -> application -> evaluator | application trace and outputs | +| queue testcase -> evaluator | testcase item | + +This difference is real and should be modeled as planning data, not hidden in separate handwritten loops. + +## Repeats And Fan-Out + +Older docs used two naming schemes for the same underlying concern: + +| Older eval-loop name | Current code name | Meaning | +|---|---|---| +| `repeat_target = "application"` | `is_split = true` | fan out at the application step | +| `repeat_target = "evaluator"` | `is_split = false` | fan out at evaluator steps | +| `reuse_traces` | `is_cached` | enable hash-based trace reuse | + +The current backend model uses `is_cached`, `is_split`, and `repeats`. `EvaluationRunFlags` contains `is_cached` and `is_split`; `EvaluationRunData.repeats` defaults to `1`. + +The worker loops now expand repeat slots in the core paths inspected: + +- batch testset creates input, invocation, and evaluator results per `repeat_idx` +- batch inference / batch invocation creates input and invocation results per `repeat_idx` +- batch trace/testcase queue items create input/source and evaluator results per `repeat_idx` +- live query creates query and evaluator results per `repeat_idx` + +The remaining issue is not absence of repeat support. It is that repeat planning is still duplicated inside specialized loops rather than centralized in one planner. + +Fan-out validity depends on topology: + +| Topology | Valid fan-out | +|---|---| +| query -> evaluator | evaluator only | +| queue source -> evaluator | evaluator only | +| testset -> application -> evaluator | application or evaluator | +| testset -> application | application only; this is batch inference / batch invocation | + +## Trace Reuse + +Hash-based trace reuse is explicit through `is_cached`. + +Reuse flow: + +1. Compute a stable hash for the runnable node from canonical references and upstream links. +2. Fetch matching traces by hash at project scope. +3. Select deterministic reusable traces for the requested repeat slots. +4. Invoke only the missing slots. +5. Populate result cells with reused or newly produced trace IDs. + +The lookup is already plural in `fetch_traces_by_hash(...)`, and helper tests cover selection and missing-count behavior. Cache lookup is now wired into the inspected application and evaluator worker boundaries. The remaining issue is duplicated per-loop cache resolution logic. + +## Setup Fragmentation + +Current setup is not one universal flow. It is split across: + +- auto evaluation creation for app + variant + testset + evaluators +- human evaluation creation for testset + single variant + evaluators +- live evaluation setup for query-backed trace sampling +- queue creation from trace IDs or testcase IDs +- SDK/local setup +- annotation queue convenience setup from traces/testsets, backed by evaluation entities + +These setup paths build similar run-data concepts through `_make_evaluation_run_data()` in several paths, but they still apply different validation and dispatch rules. That is why new combinations still tend to require setup and execution changes in multiple places. + +## Execution Consolidation + +The SDK and backend now route concrete source items through the SDK-owned +source-slice processor. Backend task modules are source/dispatch shells: + +- `run.py` classifies a run and dispatches to the right source resolver +- `query.py` resolves live/batch query source traces +- `source_slice.py` resolves direct/testset source items and builds backend adapters + +The remaining backend-specific work lives behind adapters for scenario creation, +result persistence, metrics refresh, trace loading, cache reuse, and workflow +execution. +- application invocation +- evaluator invocation +- human/custom pending behavior +- repeat handling +- cache lookup +- metrics refresh + +This duplication is now the main remaining problem. Some formerly missing capabilities are implemented, but they are implemented repeatedly across specialized loops. + +## Runnable Execution Debt + +Unifying the loop should not mean preserving every current invocation helper as-is. + +The current application execution path is especially legacy. Batch testset and batch inference paths still rely on older application invocation helpers such as the LLM app service batch invocation path, which has been patched repeatedly over time. Evaluator execution uses workflow invocation paths with similar but not identical request assembly, links, reference handling, cache handling, trace fetch handling, and error handling. + +The repeated pattern is broader than "application vs evaluator": + +- build runnable request from step references, upstream bindings, inputs, trace, and outputs +- optionally compute hash and reuse existing traces +- invoke a runnable when cache does not satisfy the slot +- fetch/validate the resulting trace +- convert response or failure into an evaluation result cell + +That should become a shared runnable-step execution contract. Application and evaluator steps can then be two runnable kinds handled by the same boundary, rather than separate loop-local helper stacks. + +## Research Conclusion + +The product does not need one flattened source type, but it does need one loop contract. + +The common contract should be: + +```text +resolve sources -> materialize scenarios -> plan result slots -> execute auto steps -> leave human/custom slots pending -> populate tensor cells -> refresh metrics +``` + +Current code has many pieces of that contract, including flags, repeat helpers, cache helpers, source-aware queue dispatch, and pending human/custom behavior in key loops. The missing layer is a shared planner that owns these decisions once. + +Source-specific behavior should live in resolvers and planners. Step execution should be generic over: + +- scenario +- step +- repeat slot +- upstream bindings +- origin +- cache policy +- fan-out policy +- runnable invocation policy + +## Relationship To Default Queues + +The queue-unification work clarifies that annotation queues are not another execution runtime. They are a persisted human-work view over the same evaluation tensor described in this document. + +The useful separation is: + +```text +evaluation runtime + = graph + tensor + process(slice) + +default queue + = canonical persisted human-work view over that tensor +``` + +The queue dimensions align with tensor dimensions: + +- scenario selection maps to scenarios +- step selection maps to steps +- repeat assignment maps to scenario × repeat lanes + +A default queue leaves those dimensions open: + +```text +scenario_ids = None +step_keys = None +user_ids = None +``` + +The queue layer decides how human work is exposed and assigned. It does not decide how auto steps are planned or executed. + +### Source-family flags versus queue eligibility + +The current runtime still uses `is_queue` in places where it is really distinguishing queue-style source ingestion. The cleaner target model is to expose source family directly through inferred flags: + +- `has_queries` +- `has_testsets` +- `has_traces` +- `has_testcases` + +Those flags should describe where scenarios come from and should drive topology validation and mixed-source prevention. + +Separately, `run.flags.is_queue` should answer the product question: + +```text +active default queue exists +and active human evaluator work exists +``` + +That makes source classification and simple-queue eligibility separate facts rather than overloading one flag. + +### Shared mutation question + +The queue-unification work also exposes a question this design must answer explicitly: whether step removal is usually destructive or archival. + +If historical results should remain visible after an evaluator is no longer active, then the graph model needs an active-versus-archived distinction. In that world: + +- `archive_step` is the common lifecycle operation +- hard `remove_step` and `prune` are stronger cleanup operations +- queue eligibility should depend on active human steps, not merely historical human steps + +This needs to be settled before hardening the mutation contract around remove/prune behavior. diff --git a/docs/designs/unified-eval-loops/run-status-finalization.md b/docs/designs/unified-eval-loops/run-status-finalization.md new file mode 100644 index 0000000000..391a2a6f05 --- /dev/null +++ b/docs/designs/unified-eval-loops/run-status-finalization.md @@ -0,0 +1,180 @@ +# Run-status finalization: analysis and options + +Status: analysis for UEL-028 (and its overlap with UEL-017 item 1). +Date: 2026-05-21. +Decision: **Option B implemented** (no aggregation) — corrected severity order + +reset `status=RUNNING` on every (re)dispatch. Batch testset/invocation runs are +single-slice today (`process_testset_source_run` issues exactly one slice), so the +multi-slice race (Option C) is not needed now and stays tracked under UEL-017 item 1. + +## 1. Problem + +A batch (non-queue) evaluation run that completes all its work never transitions to a +terminal `status` — it stays `running` with `flags.is_active=true` indefinitely. On the +dev DB this affected the majority of batch runs (`success=35` vs `running=114`, +`pending=211`). + +Reproduced end-to-end with the new LLM-free `mock_v0` workflow: a testset → app → +auto-evaluator run processes both scenarios to `success` and refreshes metrics, but the +run row stays `running` / `is_active=true`. + +## 2. Where run status is written + +Run status is finalized in `process_evaluation_source_slice` +(`api/oss/src/core/evaluations/tasks/source_slice.py`), and **only** when the caller +passes `update_run_status=True`. + +Dispatch (in `tasks/run.py::process_evaluation_run`, by topology): + +| topology | task | `update_run_status` | finalizes run status? | +| ------------------ | --------------------------- | ------------------- | --------------------- | +| `live_query` | `process_query_source_run` | **False** | no (stays running) | +| `batch_query` | `process_query_source_run` | **False** | no | +| `batch_testset` | `process_testset_source_run`| **True** | yes | +| `batch_invocation` | `process_testset_source_run`| **True** | yes | + +Implication: **live and batch-query runs never enter the finalize block.** Any change to +finalization affects only `batch_testset` / `batch_invocation`. (Whether `batch_query` +*should* finalize is a separate gap — see §6.) + +## 3. The current finalize logic + +After processing this slice's scenarios: + +```python +if any(item.has_errors for item in processed): run_status = ERRORS +elif any(item.has_pending for item in processed): run_status = RUNNING +else: run_status = SUCCESS +# (exception path) -> run_status = FAILURE +``` + +This `run_status` is computed from **this slice's `processed` subset only**, not from the +whole run. To reconcile across slices, a "severity floor" then compares it to the stored +status and keeps whichever is more severe: + +```python +severity = {FAILURE:4, ERRORS:3, RUNNING:2, SUCCESS:1, PENDING:0} # ORIGINAL +current = fetch_run(run_id) +if severity[current.status] > severity[run_status]: + run_status = current.status +``` + +### Root cause of UEL-028 + +`RUNNING` (2) outranks `SUCCESS` (1). When the stored status is `running` (the start +state of every run), a slice that computes `SUCCESS` is floored back **up** to `running`: +`severity[running]=2 > severity[success]=1`. So the run can never leave `running`. It pins +forever. + +This is a true P1: the floor's "keep the more severe status" rule treats the transient +`RUNNING` as more severe than the terminal `SUCCESS`, which is backwards for finalization. + +## 4. The cases finalization must satisfy + +| case | what should happen | +| ---- | ------------------ | +| **single-slice batch** (one slice = whole run; e.g. testset→app→eval) | all scenarios success → run `success`; any error → `errors`; exception → `failure`. | +| **multi-slice batch** (run dispatched as several slices; UEL-017 item 1) | run is terminal only when **all** slices/scenarios are done; an early SUCCESS-only slice must not finalize the whole run while other slices are pending. | +| **extended finished run** (a `success` run gets new steps/scenarios and is re-dispatched) | while new work is pending the run should read `running` again; when the new work completes it should read `success` (or `errors`/`failure` as appropriate). | +| **live / batch-query** | unaffected — never finalizes via this path (`update_run_status=False`). | + +The fundamental flaw shared by the original logic **and** the quick severity-reorder fix +is that `run_status` is derived from **one slice's subset** plus a `max()` against the +**stored** status. Neither reflects the run's *actual current* aggregate state, so: + +- single-slice: stored `running` wrongly floors over computed `success` (UEL-028). +- multi-slice: a slice can't see other slices' scenarios, so it either finalizes too early + (no floor) or never (bad floor). +- extended: stored `success` floors over a computed `running`, hiding in-flight pending + work — or, with the reorder, the opposite mishandling. + +## 5. Options + +**Constraint (per maintainer):** no full run-wide scenario aggregation in the finalize +path. The finalize must work from what the slice already has plus, at most, cheap O(1) +state — not a scan/count of all the run's scenarios on every slice. + +### Option A — Reorder the severity floor (minimal) + +Swap `RUNNING` and `SUCCESS` in the severity map so terminal statuses outrank the transient +ones: + +```python +severity = {FAILURE:4, ERRORS:3, SUCCESS:2, RUNNING:1, PENDING:0} +``` + +- **Fixes:** single-slice batch (UEL-028). A computed `SUCCESS` now replaces stored + `running`. FAILURE/ERRORS still floor over a later SUCCESS-only slice (preserves + UEL-017's "don't downgrade errors" intent). +- **Does NOT fix:** the **extended-finished** case. If an extension's new slice computes + `RUNNING` (pending cells), the floor keeps stored `success` (`2 > 1`), so the run wrongly + shows `success` while new work is in flight. Symmetrically, multi-slice early-finalize is + not addressed (a SUCCESS slice still finalizes the whole run). +- **Cost:** one-line change; lowest risk; lowest correctness. +- **Verified:** unit test `test_source_slice_processor_preserves_higher_queue_status` still + passes; the `mock_v0` single-slice flow reaches `success` in ~4s. + +### Option B — Correct the severity floor + scope it to a fresh-dispatch start state + +Two-part, no aggregation: + +1. **Reorder severity** so terminal statuses outrank transient ones (as in A): + `FAILURE:4 > ERRORS:3 > SUCCESS:2 > RUNNING:1 > PENDING:0`. Fixes UEL-028 single-slice. +2. **Reset the run to a known start state at dispatch** so the floor compares against a + meaningful baseline. The dispatch/start flow (`service.py:3540-3556`) already sets + `is_active=True`; have it also set `status=RUNNING` for **every** (re)dispatch — not just + `just_created`. Then a finished run that is extended starts the new dispatch at `running`, + and the slice's computed status (SUCCESS / ERRORS) cleanly replaces it via the corrected + floor. No run-wide scan. + +- **Fixes:** single-slice batch (UEL-028) **and** extended-finished (it restarts at + `running`, then the slice writes the new terminal status). +- **Partial on multi-slice:** with the corrected order, a SUCCESS-only slice can still + finalize the run before sibling slices finish. The `ERRORS`/`FAILURE` floor still prevents + *downgrading*, but `RUNNING→SUCCESS` can happen early. Acceptable if batch testset/invocation + runs are effectively single-slice today (confirm); otherwise pair with Option C. +- **Cost:** the severity one-liner + a one-line change to the start flow + (`status=RUNNING` on every dispatch). No new queries. +- **Risk:** changing the start flow to always reset `status=RUNNING` affects the perceived + status of any run being (re)started. This is arguably the correct semantic ("a dispatched + run is running"), but it is a behavior change for restart. + +### Option C — Finalize only on the last slice (for multi-slice, if needed) + +Pass `slice_index` / `expected_total_slices` (or a "last slice" flag) into +`process_evaluation_source_slice`; only run the finalize block on the final slice. Cheap +per-dispatch counter, **no aggregation**. + +- **Fixes:** multi-slice early-finalize. +- **Does NOT by itself fix:** UEL-028 (the last slice still needs the corrected severity + order). Layer on top of B only if batch runs are genuinely multi-slice. +- **Cost:** thread a slice counter through dispatch + the SDK boundary; more invasive than B. + +## 6. Adjacent gap (not UEL-028) + +`batch_query` runs dispatch through `process_query_source_run` with +`update_run_status=False`, so they **also never finalize** their run status — by a different +path than the testset/invocation one fixed here. This should be confirmed with a flow test +and tracked as its own finding (candidate: extend UEL-017 or a new UEL-0xx) rather than +folded into UEL-028. + +## 7. Recommendation + +No run-wide aggregation. Adopt **Option B**: corrected severity order + reset +`status=RUNNING` on every (re)dispatch in the start flow. Keep the two independent hardening +fixes already applied: + +- clear `flags.is_active` when status is terminal (SUCCESS/ERRORS/FAILURE); +- `dao.edit_run` persists `status` via `status.value` + `flag_modified`. + +This fixes single-slice (UEL-028) and extended-finished with only O(1) local changes (no +scenario scans). Prove with the flow suite: + +- single-slice batch → `success`; +- extended finished run → `running` during extension, `success` after; +- live eval → stays `running`/active (unchanged). + +**Multi-slice:** first confirm whether batch testset/invocation runs are ever dispatched as +multiple slices today. If not, B is sufficient and the multi-slice race stays tracked under +UEL-017 item 1. If they are, layer **Option C** (last-slice finalize, cheap per-dispatch +counter — still no aggregation) on top of B. diff --git a/docs/designs/unified-eval-loops/simplified-interface.md b/docs/designs/unified-eval-loops/simplified-interface.md new file mode 100644 index 0000000000..10b9cfd238 --- /dev/null +++ b/docs/designs/unified-eval-loops/simplified-interface.md @@ -0,0 +1,178 @@ +# Simplified Interface: the evaluation operation vocabulary + +Status: design / forward-looking. Concentrates the operation surface scattered +across `proposal.md`, `gap.md`, `operations.md`, and `step-removal-semantics.md` +into one place. No new semantics — it organizes what we want the **public +interface** to be. + +## Why this document + +We want a small, atomic vocabulary of evaluation operations that maps cleanly to: + +1. **HTTP endpoints** with clean URLs and explicit `operation_id`s, so they + surface as named methods in the generated Fern clients (TS + Python). +2. **The SDK `evaluate()` utility** — built *on top of* these atomic operations, + not as a parallel code path. +3. **User-built utilities** — a user should be able to assemble their own + evaluation flow from the same atomic operations `evaluate()` uses. + +The test: adding a new evaluation flow should mean *composing existing +operations*, never adding a new end-to-end endpoint. + +## The vocabulary + +There are three layers (and a fourth, queues, set aside for now): + +1. **Run** — the container. RPC-style lifecycle, not a graph/tensor mutator. +2. **Graph** — the shape: steps, scenarios, repeats, and their connections. +3. **Tensor** — the contents: results, and metrics. +4. *(Queues — a human-work view over the tensor. Noted, left aside here.)* + +```text +run: create / edit / delete | start / stop / close / open +graph: add_step / remove_step | add_scenario / remove_scenario | set_repeats +tensor: results → probe / process / populate / prune + metrics → refresh (variational / temporal / global) +``` + +### Run + +The run is acted on by **RPC operations**, not by editing fields: + +- `create` / `edit` / `delete` — `edit` mostly for completeness; real editing is + graph ops (add/remove steps & scenarios). +- `start` / `stop` / `close` / `open` — lifecycle. + +**There is no `set_flag`.** The `has_*` flags are inferred from the graph, +`is_queue` is reconciled, and the remaining config flags are set through +`create`/`edit` — never a dedicated flag RPC. + +### Graph → tensor shape + +Mutating the graph mutates the tensor's shape — the three graph axes map +one-to-one onto the three tensor dimensions: + +- **steps** → columns (add/remove a step adds/prunes a column of cells) +- **scenarios** → rows (add/remove a scenario adds/prunes a row of cells) +- **repeats** → depth (`set_repeats` materializes or prunes repeat slots) + +Steps and scenarios are named entities (add/remove); repeats is a count, so it +is **set** (`set_repeats`), which re-shapes the tensor depth. + +### Tensor: results + +Result ops act on a **slice** (`TensorSlice = scenarios x steps x repeats` = +rows x columns x depth), so retry, fill-missing, re-run-one-evaluator, queue +assignment, and live ticks are all a slice op: + +- `probe` — read cells (what exists, what's missing). +- `process` — run the runnable cells (whatever the current executor owns by + origin; see [`origin-execution-model.md`](./origin-execution-model.md)) and + populate the results. +- `populate` — write result cells directly. The shared write primitive for any + origin, including the runtimes themselves. +- `prune` — delete cells. + +### Tensor: metrics + +Metrics are derived from result cells. There are **three kinds**, differing by +scope and by *when* they may be refreshed. Each is an object **keyed by step**. + +| kind | aggregates over | refresh trigger | +| --- | --- | --- | +| **variational** | one scenario, across all its repeats | only when the scenario is **fully computed** — used in both live and batch evaluations | +| **temporal** | all scenarios + repeats within a timestamp **interval** | per interval — used for **live** evaluations | +| **global** | all scenarios + repeats in the run | at run scope — used for batch evaluations | + +**Results and metrics are decoupled.** `probe`/`process`/`populate`/`prune` +operate on **result cells only** — none of them refresh metrics. `refresh` is a +separate, first-class op that recomputes metrics, invoked by the caller on the +right boundary (scenario-complete / interval / run). `prune` deletes cells but +not metrics: a metric is an aggregate over a whole scenario/interval/run, so +pruning cells leaves it to be recomputed by `refresh`, not deleted. + +## Target endpoint shape + +All under `/api/evaluations/`. Conventions follow AGENTS.md (POST `/query`, +`/{id}/archive`, explicit `operation_id`, cursor pagination, `count` + payload +envelopes). + +### Run (RPC lifecycle) + +| operation_id | Method + path | Status | +| --- | --- | --- | +| `create_runs` / `edit_runs` / `delete_runs` / `query_runs` | `…/runs/` `/runs/query` | done | +| `fetch_run` / `edit_run` / `delete_run` | `…/runs/{run_id}` | done | +| `start_run` / `stop_run` | `POST /runs/{run_id}/start` `/stop` | done (via simple-evaluation start/stop) | +| `close_run` / `open_run` | `POST /runs/{run_id}/close` `/open` | done (lock) | + +No `set_flag`: `has_*` flags are inferred from the graph, `is_queue` is +reconciled, and the config `is_*` flags are written through `create`/`edit`. + +### Graph — steps & scenarios + +| operation_id | Method + path | Status | +| --- | --- | --- | +| `add_step` | `POST /runs/{run_id}/steps` | **deferred** — today a step is added by editing `data.steps`. | +| `remove_step` | `DELETE /runs/{run_id}/steps/{step_key}` | done as behavior (folded into `edit_run` reconcile + prune), **deferred** as a named endpoint. | +| `create_scenarios` / `edit_scenarios` / `delete_scenarios` | `…/scenarios/` | done (CRUD) | +| `add_scenario` | `POST /runs/{run_id}/scenarios` | **deferred** — graph-aware add (resolve source binding, plan cells), vs. raw CRUD. | +| `remove_scenario` | `DELETE /runs/{run_id}/scenarios/{scenario_id}` | **deferred** — cascade-aware (prune cells + flush metrics). | +| `set_repeats` | `POST /runs/{run_id}/repeats` | **deferred** — set the depth; materialize new repeat slots or prune surplus, then `process` the new cells. | + +### Tensor results — endpoints + +| operation_id | Method + path | Status | +| --- | --- | --- | +| `create_results` / `edit_results` / `delete_results` | `…/results/` | done (CRUD; the low-level cell write) | +| `probe_slice` | `POST /runs/{run_id}/slice/probe` | in-process (`TensorSliceOperations.probe`), **deferred** as endpoint. | +| `process_slice` | `POST /runs/{run_id}/slice/process` | in-process (`process` → `BackendSliceProcessor`), **deferred** as endpoint. | +| `populate_slice` | `POST /runs/{run_id}/slice/populate` | in-process, **deferred** as endpoint (bulk/slice write). | +| `prune_slice` | `POST /runs/{run_id}/slice/prune` | in-process, **deferred** as endpoint. | + +The four `*_slice` ops all take a `TensorSlice` body. They are the atomic +primitives `evaluate()` and user utilities compose; only their HTTP exposure is +deferred — the in-process implementations exist (`runtime/tensor.py`). + +### Tensor metrics — endpoints + +| operation_id | Method + path | Status | +| --- | --- | --- | +| `refresh_metrics` | `POST /metrics/refresh` | done — single endpoint today; refreshes the affected scope. | +| `create_metrics` / `edit_metrics` / `delete_metrics` / `query_metrics` | `…/metrics/` `/metrics/query` | done (CRUD) | + +Whether refresh stays one endpoint or splits per kind (variational / temporal / +global) — and whether it stays coupled to `process` or becomes its own +operation — is the open question above. + +## How `evaluate()` composes these + +The SDK utility is a thin client-side orchestration over the vocabulary: + +```text +evaluate(testset, app, evaluators): + create_run(...) # container + add_step per app + evaluator # graph + add_scenario per testset row # graph (resolve source bindings) + process_slice(all scenarios, all steps) # tensor: run the runnable cells + refresh metrics + -> returns {run, scenarios, metrics} +``` + +A user wanting a custom flow (e.g. re-run only one evaluator on failed rows) +calls the same ops directly: `probe_slice` to find failures, then +`process_slice` scoped to that evaluator's `step_key`. + +## What this unlocks + +- Generated clients expose `evaluations.processSlice(...)`, + `evaluations.addScenario(...)`, etc. as named methods. +- One mental model for setup, retry, queue assignment, manual annotation, live + ticks, and SDK/local runs — all are slice ops. +- New flows are compositions, not new endpoints. + +## Pointers + +- Operation list + direction: [`proposal.md`](./proposal.md) §"Operation API Direction". +- Per-op status (done / partial / deferred): [`operations.md`](./operations.md). +- Removal lifecycle (why `remove_step` prunes): [`step-removal-semantics.md`](./step-removal-semantics.md). +- Gaps: [`gap.md`](./gap.md) §"Tensor Operation Gaps". diff --git a/docs/designs/unified-eval-loops/single-loop-plan.md b/docs/designs/unified-eval-loops/single-loop-plan.md new file mode 100644 index 0000000000..12ab398f2b --- /dev/null +++ b/docs/designs/unified-eval-loops/single-loop-plan.md @@ -0,0 +1,229 @@ +# Single-Loop Plan: `populate` then `process` (two-level cache) + +## Goal + +**One execution loop. No two paths.** Today the backend has two wrappers around +the same SDK engine (`sdk_process_evaluation_source_slice`): + +- `process_evaluation_source_slice` (processor.py:636) — **ingest**: creates NEW + scenarios from direct ids / query / testset, runs all cells. +- `APISliceProcessor.process` (processor.py:238–488) — **re-execute**: operates on + EXISTING scenarios by coordinate, reconstructs source from stored cells. + +These are not different verbs. They are the **same operation** distinguished only +by whether the inputs already exist. This plan collapses them into one loop. + +## The model (the core idea) + +`process` is a **two-level cache walk**. For every addressed coordinate cell: + +``` +1. RESULT level — does the result cell already exist? + yes -> reuse it (skip), unless process_mode == "force" + no -> go to step 2 +2. INPUT/DATA level — is the underlying input already available? + - the scenario's input cell stores a trace_id / testcase_id? -> use it + - a reusable trace exists by HASH (make_hash + fetch_by_hash)? -> use it + - the run references a query / testset revision? -> load internally + - a DIRECT testcase_id / trace_id with nothing stored yet? -> must be POPULATED first +3. Execute only the genuine gaps (the cells with no result and resolvable input). +``` + +Both levels are the same question — *does it already exist?* — applied first to +the **result**, then to the **input**. "Hashing for traces" and "storing the +testcase_id" are exactly the input-level cache: you store the id instead of +re-invoking the thing that generates it. + +### Consequence: there is no create-vs-reuse branch + +The only thing the loop cannot derive is a **direct `testcase_id` / `trace_id` +that was never stored** — because the id *is* the source identity and there is no +internal way to load it. That, and only that, is what `populate` is for. + +| Source kind | Input derivable internally? | Needs explicit `populate`? | +|---|---|---| +| query revision | yes — load rows from the referenced revision | no | +| testset revision | yes — load rows from the referenced revision | no | +| already-run scenario (input cell stored) | yes — read the stored id | no | +| direct `testcase_id` / `trace_id` (fresh) | no — the id is the only identity | **yes** | + +### Three primitives — `process` never creates scenarios + +"populate then process" glossed over a missing verb. There are **three** +distinct operations, and crucially `process` operates only on scenarios that +**already exist** — it does not mint them: + +| Verb | Creates | Does NOT create | +|---|---|---| +| **materialize** | scenarios (the coordinate skeleton for the run) | results | +| **populate** | result cells (input cells carrying the trace_id/testcase_id) | scenarios | +| **process** | result cells (by executing existing cells) | **scenarios** | + +`materialize` = `create_scenarios` (service.py:936); `populate` = `set_results`; +`process` = the shared execution loop. Today `APIScenarioFactory`-inside-the-loop +does materialize *implicitly* as a side effect of `process` — this plan pulls it +out and names it, so `process` is uniform: it never creates scenarios for anyone. + +So every flow reduces to: + +- **direct id ingest** = `materialize(N)` → `populate(input cells for the ids)` → `process` +- **query / testset run** = `materialize(from internally-loaded rows)` → `process` (inputs loaded internally; no explicit populate) +- **re-execute / retry / add-evaluator** = `process` (scenarios + input cells already exist) + +`process` is the one execution loop; `materialize` precedes it when scenarios +don't exist; `populate` precedes it when the input identity (direct id) can't be +loaded internally. + +## Target API (the single loop) + +`process` already exists as `TensorSliceOperations.process` → +`SliceProcessor.process` (tensor.py:42). The unification makes its concrete +implementation the **only** execution loop, and turns the ingest wrapper into a +caller that `populate`s first. + +``` +process(slice) = + for each scenario coordinate in the slice (or all run scenarios if unaddressed): + ensure an input cell exists # else: skip — must populate first + resolve source from the input cell (id / hash / reference) + plan cells from the run's CURRENT graph + filter to addressed coordinates (step_keys / repeat_idxs) + for each cell: result-cache check -> input-cache check -> execute gap + populate result cells + (metrics refresh is the separate `refresh` op) +``` + +The SDK engine (`runtime/processor.py`) is already this loop. The work is in the +API layer: make one wrapper, delete the second. + +## Seam analysis (what actually differs today) + +Both wrappers call `sdk_process_evaluation_source_slice` and differ in: + +| Seam | ingest (line) | re-execute (line) | unified treatment | +|---|---|---|---| +| scenario | `APIScenarioFactory` (869) | `_ExistingScenario` (431) | **input-cache check** — reuse if input cell exists, else the caller populated it; no policy branch | +| source data | `resolve_direct_source_items` (800) | `_source_item_from_input_cells` (351) | one resolver: read stored id → hydrate; query/testset load by reference | +| cell filter | none | `target_keys` closure (466) | `cell_filter` pass-through (None = all) | +| metrics | `APIMetricsRefresher` (883) | `_noop_refresh_metrics` (442) | **shared** — re-execute stops opting out; both inject the real refresher so `process` refreshes incrementally per-scenario AND rolls up at the end, exactly as the SDK loop already does (sdk processor.py:270, 297). The metric *kind* (variational/temporal/global) is already derived from scope by `service._refresh_metrics`. The separate `refresh` op + `_noop` are deleted. | +| post-process: status | per-scenario `edit_scenario` + run severity-floor + is_active (991–1054) | none | **shared** — `process` finalizes from the touched set it already returns (`ProcessedScenario.has_errors/has_pending` per scenario; floored to the run). Two flags: `finalize_run_status` (False for live-query, which loops) and the is_active flip (terminal only). The concurrent-slice re-fetch / `is_active` race fix moves verbatim into the shared finalize. | + +**Why post-process is shared, not ingest-only:** status + which-metrics-to-refresh +are a function of *what coordinates were touched* — which the shared loop already +knows (it returns `processed`). The reasoning is identical regardless of how the +run was triggered; and since we hold the run lock (one process per run at a time), +the shared path has full control to finalize before releasing. A slice covering +the whole run IS a batch evaluation — its "done" must match today's batch path. + +Everything else (steps mapping, runners/revisions via +`_resolve_runners_and_revisions`, `trace_loader`, `is_split`, +batch_size/max_retries) is **already identical** and intrinsic to `run`. + +### The one-sentence shape + +- **ingest** = `populate` → `process` → done +- **re-execute** = `process` → done + +…where **`process` owns "done"** (status finalize + scoped, incremental metric +refresh from what was touched). `populate` is the only divergence, and only for +direct ids that can't be loaded internally. + +## Staging (incremental, never two paths in the middle) + +The end state is one loop, but the move is staged so the working ingest path is +never destabilized. Each stage leaves the suite green. + +### Stage 1 — collapse source shaping (pure dedup, zero behavior change) +- Extract one `_to_sdk_source_item(...)` helper; the + `SdkResolvedSourceItem(...)` construction is byte-identical at processor.py + 816–831 and 372–383. Replace both. +- Guard: existing unit + acceptance suites green. + +### Stage 2 — one source resolver (input-level cache) +- A single `resolve_source_for_scenario(...)` that implements the input-cache + ladder: stored id → hash-reuse → reference-load. `resolve_direct_source_items` + and `_source_item_from_input_cells` both fold into it. +- Re-execute and ingest both call it; the only difference becomes whether the + input cell pre-exists (re-execute) or was just populated (ingest). + +### Stage 3 — one execution loop +- A single `process_source_slice(...)` does the per-scenario loop + SDK call. + `APISliceProcessor.process` becomes a thin caller (adds coordinate filter + + ProcessSummary accounting). `process_evaluation_source_slice` becomes a thin + caller (adds queue validation + run finalization, both run-level). +- Delete the duplicated SDK call site. + +### Stage 4 — `process` owns "done" (shared finalize) + +- Fold post-process into the shared loop: per-scenario status writes + run + severity-floor + `is_active`, derived from the `processed` set the loop already + returns. Re-execute stops injecting `_noop_refresh_metrics` and injects the real + refresher, so metrics refresh incrementally per-scenario AND roll up at the end + for BOTH paths (timing unchanged from today — the SDK loop already does this). + The separate `refresh` op / `_noop` are deleted. +- Two flags split from the old single `update_run_status`: + `finalize_run_status` (False for the live-query loop, which never finalizes) and + the `is_active` flip (terminal status only). The concurrent-slice re-fetch + (`is_active` race fix) moves verbatim into the shared finalize. + +### Stage 5 — the coordinate-dimension ops (`process` never creates scenarios) + +The coordinate space is `scenarios × steps × repeats`. `process` operates only on +EXISTING coordinates; the dimensions are managed by first-class graph ops. This +pass lands the ones ingest needs (the others stay deferred): + +- **`add_scenarios(run, n)`** — create N scenario skeleton rows for the run (rows + only; no input cells, no results). This is the deferred `add_scenario` op from + operations.md:50, implemented thin (skeleton only — `populate` writes the input + cell, `process` plans+executes). Wraps `create_scenarios` (service.py:936). +- **`set_repeats(run, repeats)`** — resize the repeat dimension (today `repeats` + is fixed at create, EvaluationRunData.repeats:226). NB: a `set_repeats` pydantic + *validator* already exists (types.py:231) — the op needs a distinct name + (`resize_repeats` / `add_repeats`) to avoid the collision. +- `process` is made uniform: it never mints scenarios for anyone (the + `APIScenarioFactory`-inside-the-loop is removed). Callers `add_scenarios` first. + +Deferred siblings (noted, not in this pass): `add_steps`, `remove_scenarios`, +`set_flag`. + +### Stage 6 — direct-id ingest = add_scenarios → populate → process + +- `run.py` traces/testcases ingest becomes: `add_scenarios(N)` → + `populate(input cells carrying the ids)` → `process(those scenario_ids)` → + shared finalize. +- Query/testset keep loading rows internally and `add_scenarios` from them, then + `process` (no explicit populate — inputs load by reference). +- Guard: parity test — direct-id ingest and the explicit + add_scenarios→populate→process produce identical result cells + status + metrics. + +## The one genuine limit (by design, not a gap) + +`process` cannot run a coordinate whose scenario has **no input cell and no +internal reference** — there is nothing to resolve. Today this is counted as +`failed` and skipped (processor.py:355). That is correct: it is exactly the case +`populate` must handle first. The loop never tries to conjure a source from +nothing. + +## Risk / blast radius + +- **Run finalization** (severity-floor + `is_active`, the concurrent-slice race + fix) must stay a run-level caller concern, never leak into the per-slice loop — + re-execute must not finalize. +- **Queue validation** (`require_queue`) is ingest-only; stays in the ingest + caller. +- **timestamp / interval** (live-query window) must thread through to scenario + + result writes. +- **is_split** must be computed once and shared between the cell filter and the + SDK plan, or they diverge. + +## Tests guarding the move + +- `unit/evaluations/test_tensor_slice_ops.py` — the `process`/`probe`/`populate` + surface (the single-loop entry). +- `unit/evaluations/test_runtime_topology_planner.py` — source-slice + re-execute + parity, the "no input cell → skipped" branch. +- `unit/evaluations/test_query_eval_loops.py` — live/batch-query (finalize=False, + timestamp/interval seam). +- `acceptance/evaluations/test_tensor_slice_endpoints.py` — HTTP contract + + populate→probe round-trip. +- New parity test: `ingest(direct ids)` ≡ `populate + process`. diff --git a/docs/designs/unified-eval-loops/step-removal-semantics.md b/docs/designs/unified-eval-loops/step-removal-semantics.md new file mode 100644 index 0000000000..585e520f62 --- /dev/null +++ b/docs/designs/unified-eval-loops/step-removal-semantics.md @@ -0,0 +1,448 @@ +# Step Removal Semantics + +## Decision + +For now, evaluation step removal is **destructive**: + +```text +remove_step -> prune the removed step's tensor cells +``` + +Removing a step means: + +1. remove it from the active run graph +2. delete result cells for that step across scenarios and repeats +3. refresh/flush metrics that depended on that step +4. if the removed step is an input step, also remove scenarios that are sourced only from that step + +This keeps the stored graph and stored tensor aligned with the current evaluation definition. + +The alternative — archiving/deactivating steps while retaining historical cells — remains a valid future model, but it is **not** the model chosen for the current design. + +## Why This Decision Exists + +There are two coherent models for step lifecycle. + +### Model A — Destructive removal + +```text +stored graph = current active graph +stored tensor = cells for the current active graph +``` + +A removed step no longer exists in the graph, and its cells are pruned. + +### Model B — Archival lifecycle + +```text +stored graph = historical graph +active execution = projection over active steps +stored tensor = historical cells, including archived steps +``` + +An archived step remains historically present, but no longer participates in future work. + +Both models are internally coherent. The current design chooses **Model A** because it is simpler, cleaner, and matches the existing unified-loop operation model. + +## Existing Design Rationale For Remove + Prune + +The existing eval-loop documents already leaned toward destructive removal for good reasons. + +### Steps are immutable by reference + +A step points to a concrete referenced revision. Changing a reference should not mutate the step in place. + +Instead: + +```text +change evaluator revision = remove old step + add new step +``` + +That preserves step identity semantics and avoids silently rewriting what a historical step meant. + +### The graph defines tensor shape + +The design treats graph steps as tensor dimensions: + +- add a step -> add a tensor column dimension +- remove a step -> remove that tensor column's cells + +This creates a simple invariant: + +```text +current graph and current tensor have the same shape +``` + +### Remove + prune prevents stale state + +If a step disappears but its result cells remain: + +- cells exist for steps no longer in the graph +- metrics may still refer to retired steps +- UI needs to distinguish active from historical columns +- planner and topology logic need lifecycle-aware filtering + +Pruning avoids all of that in the default path. + +### The mutation model stays symmetric + +The lower-level operation model remains clean: + +```text +graph: add_step / remove_step +tensor: populate / prune +``` + +That symmetry is useful for reasoning, implementation, and testing. + +## Why Archival Was Considered + +Archival has one major product advantage: + +> it preserves auditability. + +If a human evaluator or automatic evaluator is no longer active, retaining the old step and its cells would preserve: + +- who evaluated what +- what outputs existed before the step was retired +- historical metric context +- a full explanation of past evaluation state + +That is especially attractive if evaluations are treated as long-lived collaborative records rather than disposable execution definitions. + +## Cost Of Destructive Removal + +The chosen model deliberately gives up some history. + +When a step is removed: + +- its result cells are deleted +- metrics derived from it disappear from the active run +- prior human work for that step is no longer represented in the run tensor +- the run no longer explains that the step ever existed + +If auditability becomes a product requirement later, destructive removal will not satisfy it by itself. + +## Cost Of Archival + +Archival avoids data loss, but it has broad implications across every layer of the system. + +The rest of this document records those implications so the tradeoff remains explicit. + +# Archival Implications + +## 1. Model implications + +Archival requires step lifecycle state, for example: + +```python +archived_at: datetime | None +archived_by_id: UUID | None +``` + +A run would then contain two conceptual graphs: + +```text +historical graph = all steps ever attached to the run +active graph = historical graph minus archived steps +``` + +Any presence-style flags would need explicit semantics: + +- `has_evaluators` +- `has_human` +- `has_auto` +- `has_custom` + +For most product behavior, they would likely need to mean **active presence**, not historical presence. + +If historical presence also matters, that would require separate query behavior or additional flags. + +## 2. Data implications + +Archived steps retain their tensor cells: + +```text +scenario_id + step_key + repeat_idx +``` + +That preserves history, but results now divide into: + +- active-step results +- archived-step results + +Queries and APIs would need to decide whether they default to: + +- active-only results +- all historical results +- or support explicit `include_archived_steps` + +If archived steps remain embedded in JSON run data, active/historical filtering is service-derived and less relationally natural. If steps become first-class rows, lifecycle handling becomes cleaner but requires a larger schema refactor. + +Archival also increases retained data volume over time because old cells remain instead of being pruned. + +## 3. Metrics implications + +Archival makes metric meaning more complex. + +At minimum, the system would need to distinguish: + +### Active metrics + +Metrics over the current active graph, used for: + +- current dashboards +- current summary views +- present-tense evaluation interpretation + +### Historical metrics + +Metrics including archived steps, used for: + +- audit +- history +- lineage + +Without that distinction, archived evaluators would continue to affect current dashboards. + +Metric refresh would need to know whether it is computing over active steps only or over historical steps as well. Run mappings may also need lifecycle awareness so archived step mappings do not keep contributing to current aggregates. + +## 4. Compute and planner implications + +The planner would need to operate on **active steps only** by default. + +Every execution path would need a shared helper such as: + +```python +active_steps(run) +has_active_human_steps(run) +``` + +If archived steps remained in `run.data.steps`, raw iteration over `run.data.steps` would become unsafe. + +`process(slice)` would need explicit semantics: + +- `steps="all"` likely means all **active** steps +- archived steps require explicit inclusion for any historical replay or audit operation + +Planner complexity would remain manageable if active filtering is centralized, but every planner, topology classifier, queue reconciler, and flag refresher would need to use the same lifecycle-aware projection. + +## 5. Queue implications + +Default queue eligibility would need to depend on **active** human steps: + +```text +active default queue exists +and active human evaluator work exists +``` + +For default queues: + +```text +step_keys=None +``` + +would need to mean all **active** queue-relevant steps, not all historical steps. + +Custom queues that explicitly reference later-archived steps would need a policy, such as: + +- retain the queue row +- stop generating active work for archived steps +- surface that the queue references inactive steps +- perhaps mark the queue degraded/inactive if all included steps are archived + +## 6. API implications + +Archival would require new lifecycle operations: + +- `archive_step` +- `unarchive_step` + +or equivalent run-mutation semantics. + +Any response that exposes step definitions would need archival metadata so clients can distinguish active from historical steps. + +The API would also need explicit lifecycle-aware query semantics, likely including some form of: + +- active-only default behavior +- optional archived inclusion for audit/history views + +Backward compatibility becomes non-trivial because older clients may assume every returned step is active. + +## 7. UI implications + +The UI would need an explicit active-versus-archived presentation model. + +Likely implications: + +- active steps shown normally +- archived steps grouped under a collapsed historical section +- current results tables show active columns by default +- archived result columns appear only in audit/history contexts or behind an explicit toggle +- current metric charts exclude archived steps by default +- historical metric views expose archived-step data intentionally +- queue screens show only active human work +- archived human work remains visible in evaluation history but not as new actionable queue work + +Action labels would also need to change: + +- ordinary user action: `Archive evaluator` +- stronger destructive action: `Delete step and results` + +Without UI support for archived state, archival would preserve data technically but create user confusion. + +## 8. Controller and service implications + +Archival requires centralized lifecycle orchestration. + +A step archive/unarchive transition would need to coordinate: + +- active graph projection +- run flag recomputation +- queue reconciliation +- metric refresh +- possible custom-queue invalidation/degradation + +Those changes should not be scattered across ad hoc call sites. They require one authoritative lifecycle path. + +## 9. Conceptual implication + +Archival changes the core invariant from: + +```text +stored tensor = current graph +``` + +to: + +```text +stored tensor = historical graph +active execution = projection over active graph +``` + +That is a richer but more expensive model. + +# Destructive Removal Implications + +## 1. Model implications + +No additional step lifecycle state is required. + +The run graph remains: + +```text +run.data.steps = active graph +``` + +Presence flags continue to reflect the graph directly. + +## 2. Data implications + +Removed step cells are deleted. + +This avoids: + +- stale cells +- historical-vs-active result interpretation +- extra retained data volume for removed steps + +But it sacrifices historical traceability inside the run. + +## 3. Metrics implications + +Metric handling stays simple: + +- prune step cells +- refresh/flush dependent metrics +- current metrics remain aligned with the current graph + +No separate active/historical metric families are required. + +## 4. Compute and planner implications + +Planner logic remains simpler: + +- every step in the graph is active +- `steps="all"` means literally every stored step +- topology validation does not need step lifecycle filtering + +## 5. Queue implications + +Queue eligibility can be computed from the current graph without active/historical distinction. + +A removed human step no longer contributes to queue eligibility because it no longer exists. + +## 6. API implications + +Only destructive graph operations are needed: + +- `add_step` +- `remove_step` + +No step archive/unarchive surface is required. + +## 7. UI implications + +The UI stays much simpler: + +- no archived-step sections +- no archived-result toggles +- no historical metric mode +- “remove” means the thing is gone + +The downside is that users cannot inspect retired step history through the run afterward. + +## 8. Controller implications + +Mutation side effects remain narrow: + +- remove step +- prune cells +- refresh metrics +- if needed, reconcile queue flags from the new active graph + +No long-lived archival state needs to remain synchronized. + +# Comparison + +| Concern | Destructive remove + prune | Archive/deactivate | +|---|---|---| +| Auditability | weak | strong | +| Current-state simplicity | strong | weaker | +| Storage growth | lower | higher | +| Planner complexity | lower | higher | +| Metric semantics | simple | active vs historical required | +| UI complexity | lower | higher | +| Queue semantics | simpler | must ignore archived steps | +| API lifecycle surface | smaller | larger | +| Graph/tensor invariant | identical current graph/tensor | historical storage + active projection | + +# Current Choice + +The current unified-eval-loop design chooses: + +```text +remove + prune +``` + +as the normal behavior. + +This is intentionally destructive, and the tradeoff is accepted for now because it provides: + +- a clean graph/tensor invariant +- simpler planning and topology logic +- simpler metrics +- simpler UI/API behavior +- direct alignment with the existing operation model + +If auditability becomes a product requirement later, the design should be revisited explicitly rather than approximated halfway. A future archival model would need full support across: + +- step lifecycle metadata +- active/historical result semantics +- metric semantics +- queue eligibility +- APIs +- UI +- planner defaults + +Until then, retaining removed-step cells without modeling archival everywhere is not acceptable because it would introduce ambiguity without delivering coherent auditability. diff --git a/docs/designs/unify-evals-and-queues/gap.md b/docs/designs/unify-evals-and-queues/gap.md new file mode 100644 index 0000000000..81b0efa296 --- /dev/null +++ b/docs/designs/unify-evals-and-queues/gap.md @@ -0,0 +1,92 @@ +# Gap Analysis + +## Queue Semantics + +Missing from current state: + +- no explicit meaning that `step_keys=None` is the open/default step scope +- current auto-created human queues snapshot human step keys instead of leaving step scope open +- no canonical default-queue marker distinct from arbitrary custom queues + +Already present: + +- `scenario_ids=None` already leaves scenario scope open over the run +- `user_ids=None` already means unassigned +- repeats are already run-owned rather than queue-owned + +## Default Queue Lifecycle + +Missing from current state: + +- no default-queue reconciliation tied to run creation/editing +- current helper is path-dependent and only reached from selected execution flows +- no two-policy model separating: + - human-step structural condition + - unconditional default-queue global setting +- no logic to archive/unarchive the default queue as human evaluator availability changes + +## Queue Archival + +Missing from current state: + +- no queue archive endpoint +- no queue unarchive endpoint +- no queue service/DAO archive lifecycle path +- no `include_archived` support on queue query/fetch surfaces +- default-queue lookup cannot currently search archived queues for restoration + +Present but underused: + +- queue DTOs already inherit lifecycle fields such as `deleted_at` and `deleted_by_id` + +## Queue Identity + +Missing from current state: + +- no reliable way to distinguish the canonical default queue from a custom queue with the same open shape +- current ensure logic stops if any queue exists for the run, which is insufficient once default and custom queues coexist + +## Run Semantics + +Needs clarification or adjustment: + +- `is_queue` currently distinguishes simple queue-created runs from simple evaluations +- linked default queues should not require ordinary evaluation runs to become queue-ingest runs +- the old meaning of `is_queue` must be replaced by persisted simple-queue eligibility + +## Configuration + +Missing from current state: + +- no global policy toggle for unconditional default queues +- no shared policy helper for deciding default-queue lifecycle mode + +## Tests + +Missing from current state: + +- open default queue behavior with `step_keys=None` +- default queue creation for simple evaluations under unconditional mode +- conditional creation when human evaluator steps exist +- no creation / archive when conditional mode has no active human evaluator steps +- unarchive of an existing archived default queue instead of duplicate creation +- coexistence of default and custom queues +- archived-inclusive queue query behavior +- regression tests for existing queue assignment and scenario selection behavior + +## API Surface + +Missing from current state: + +- archive/unarchive queue endpoints +- `include_archived` request/query support for queues +- response behavior that lets callers distinguish active from archived queues where relevant + +## UI Surface + +Not covered by this backend design: + +- whether auto-only evaluations show an empty queue +- whether users are nudged to add human evaluators +- how the default queue appears inside evaluation details versus Queues +- any migration of frontend terminology from “human evaluation” to “evaluation with human evaluators” diff --git a/docs/designs/unify-evals-and-queues/plan.md b/docs/designs/unify-evals-and-queues/plan.md new file mode 100644 index 0000000000..2f62a34e2c --- /dev/null +++ b/docs/designs/unify-evals-and-queues/plan.md @@ -0,0 +1,16 @@ +# Plan + +1. Define the canonical default queue shape with open filters: `scenario_ids=None`, `step_keys=None`, `user_ids=None`, and no default batching restrictions. +2. Add a durable default-queue identifier, preferably an explicit queue flag or role that distinguishes default queues from custom queues independently of shape. +3. Add queue archival support across DTOs, service methods, DAO methods, and API endpoints, including archive and unarchive operations. +4. Extend queue query/fetch paths with `include_archived` support and ensure archived default queues can be found during reconciliation. +5. Add global policy toggle for unconditional default queues, e.g. `EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS`, as a module-level global. +6. Implement shared policy helpers for: + - whether a run has active human evaluator steps + - whether default queues are unconditional for all runs +7. Replace the current path-specific human-queue helper with a default-queue reconciliation operation that can create, unarchive, no-op, or archive according to the two-policy model. +8. Invoke default-queue reconciliation from simple evaluation run creation and run-editing flows so queue lifecycle follows evaluation lifecycle rather than dispatch timing. +9. Use source-family flags for ingestion semantics; persist `is_queue` as active default queue + active human evaluator work. +10. Update simple queue/default queue creation paths so the default queue leaves `step_keys` open instead of snapshotting human step keys. +11. Add backend tests for unconditional mode, conditional mode, archive/unarchive behavior, coexistence with custom queues, open step scope, and existing queue regressions. +12. Update design/API documentation to describe the default queue model, the two policies, queue archival semantics, and the frontend decisions intentionally left outside this backend work. diff --git a/docs/designs/unify-evals-and-queues/proposal.md b/docs/designs/unify-evals-and-queues/proposal.md new file mode 100644 index 0000000000..63666386a1 --- /dev/null +++ b/docs/designs/unify-evals-and-queues/proposal.md @@ -0,0 +1,144 @@ +# Proposal + +## Goal + +Unify human evaluation and annotation queues at the backend model level by making the queue a default companion of evaluation runs rather than a separately created product concept. + +This proposal covers API and service semantics only. Frontend behavior, copy, and product nudges can vary later without requiring a different backend model. + +## Proposed Model + +Keep the current evaluation substrate: + +- runs define evaluation structure and repeats +- scenarios are concrete work items +- results are step × repeat outputs +- queues overlay a run to expose and distribute human work + +Add one canonical **default queue** concept for evaluation runs. + +A default queue has: + +- `scenario_ids=None` +- `step_keys=None` +- `user_ids=None` +- no queue-specific batching restriction + +Those open fields mean: + +- all scenarios in the run are eligible +- all queue-relevant steps are eligible +- no users are assigned by default +- run repeats remain fully covered because repeats belong to the run + +## Queue Axes + +The queue has three independent axes: + +| Axis | Governs | +|---|---| +| scenario selection | which scenarios belong to the queue | +| repeat assignment | which scenario × repeat lanes a user receives | +| step selection | which steps must be completed for each assigned scenario × repeat | + +Default queues leave all three axes open except for the run boundary itself. + +## Default Queue Policies + +### Structural policy + +The structural condition is simple: + +```text +has_human_evaluator_steps(run) +``` + +This says whether a run warrants a default queue when queues are conditional. + +### Global lifecycle policy + +A configuration value controls whether default queues exist for all runs regardless of human steps, for example: + +```text +EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS +``` + +When enabled: + +- every run gets a default queue at creation +- the default queue is never archived merely because no active human evaluator steps remain + +When disabled: + +- a default queue exists only while active human evaluator steps exist +- adding/restoring human evaluator work creates or unarchives the default queue +- removing/archiving the last active human evaluator archives the default queue + +## Default Queue Lifecycle + +Default queue reconciliation should use durable identity: + +- missing + required -> create +- archived + required -> unarchive +- active + required -> no-op +- active + not required -> archive + +Default queues should not be hard-deleted as part of normal reconciliation. + +## Queue Lifecycle Support + +Queues should gain product-level soft-delete support: + +- archive endpoint/service/DAO path +- unarchive endpoint/service/DAO path +- `include_archived` query support +- archived-inclusive lookup for default-queue reconciliation + +Hard delete may remain available for existing low-level semantics, but default-queue lifecycle should use archive/unarchive. + +## Canonical Queue Identity + +The system needs a reliable way to identify the default queue independently of shape. A custom queue may coincidentally have no scenario filter, no step filter, and no assignments. + +The proposal requires one of: + +- an explicit queue role/flag such as `is_default` +- or another canonical linkage that uniquely identifies the default queue for a run + +An explicit marker is the clearer fit. + +## Service Placement + +Default-queue reconciliation belongs with run creation and run mutation, not only dispatch flows. + +The current `_ensure_human_annotation_queue(...)` seam should evolve into a more general lifecycle operation such as: + +```text +reconcile_default_queue(run) +``` + +It should evaluate the global lifecycle policy and, when needed, the structural human-step policy. + +## Compatibility + +This proposal preserves: + +- existing evaluation-run primitives +- existing custom queue behavior +- existing queue-backed execution paths +- hard-delete support where still needed + +It changes the default composition: + +- default queue existence becomes managed by run lifecycle +- open `step_keys` become a supported queue shape rather than a snapshot omission +- simple evaluations can participate in the same queue model as simple queues + +## Product Boundary + +The backend supports both product postures: + +- default queues for every evaluation, including auto-only evaluations +- default queues only when human evaluator work exists + +The frontend can later decide whether to expose empty queues, nudge users toward adding human evaluators, or hide queues until human work appears. The API does not need to change again for that choice. diff --git a/docs/designs/unify-evals-and-queues/research.md b/docs/designs/unify-evals-and-queues/research.md new file mode 100644 index 0000000000..fcfbc2cf36 --- /dev/null +++ b/docs/designs/unify-evals-and-queues/research.md @@ -0,0 +1,500 @@ +# Research: Unifying Evaluations and Queues + +## Scope + +This note maps the evaluation/queue model that exists today and answers one concrete exploration question: + +> What would it mean, in the current architecture, for a regular evaluation to always have a default linked queue that behaves like a simple queue when human evaluators are present? + +The focus is backend behavior in: + +- `api/oss/src/core/evaluations/*` +- `api/oss/src/apis/fastapi/evaluations/*` +- `api/oss/src/dbs/postgres/evaluations/*` +- the neighboring annotations layer where it clarifies the boundary + +## Executive Summary + +The system already has a single low-level evaluation substrate: + +- an **evaluation run** defines the workflow graph and repeats +- **scenarios** are concrete work items within a run +- **results** are per-step, per-repeat outputs for a scenario +- **metrics** summarize results +- an **evaluation queue** is an overlay over a run that selects which scenarios and annotation steps are visible to which users + +The split users see today is mostly created by wrapper layers: + +- `SimpleEvaluationsService` wraps runs without queue-centric defaults +- `SimpleQueuesService` wraps runs plus a queue with queue-centric defaults + +That means the proposed product direction is structurally plausible: it does **not** require inventing a new primitive. It mostly requires deciding what the canonical/default queue attached to a run means and when it should be created or updated. + +The codebase is also already partway toward that direction: + +- `EvaluationsService._ensure_human_annotation_queue(...)` creates a queue for a run with human annotation steps when none exists. +- Human-bearing live runs call this during refresh. +- Queue-backed batch dispatch paths call it before processing traces/testcases. + +However, that helper currently creates a **narrow, snapshot-style** queue: + +- only when human steps exist +- only if the run has no queue at all +- with `step_keys` captured from the run at that moment +- with no assignments +- with no explicit scenario restriction + +It is a useful seam, but not yet the full default-queue model. + +The sharper target model is simpler than the current helper: + +- `scenario_ids=None` means all scenarios in the run +- `step_keys=None` means all steps included by the queue policy +- `user_ids=None` means unassigned +- repeats remain owned by the run, while assignments distribute scenario × repeat work + +## Current Domain Model + +### 1. Evaluation runs are the canonical execution object + +`EvaluationRun` stores the durable definition of an evaluation: + +- `data.steps`: input, invocation, and annotation steps +- `data.repeats`: repeat count for the run +- `data.mappings`: metric/result extraction mappings +- flags such as `is_live`, `is_queue`, `has_human`, `has_auto`, `has_testsets`, `has_queries` + +A run is therefore already capable of representing: + +- automatic-only evaluations +- human-only evaluations +- mixed human + automatic evaluations +- queue-backed and non-queue-backed flows + +The evaluator origin is not a separate resource type. It is encoded on annotation steps as `origin in {custom, human, auto}`. + +### 2. Queues are overlays over runs, not separate executions + +`EvaluationQueue` points to a `run_id` and stores queue-specific selection/distribution state in `EvaluationQueueData`: + +- `user_ids: List[List[UUID]] | None` +- `scenario_ids: List[UUID] | None` +- `step_keys: List[str] | None` +- optional batching controls + +The queue does **not** own scenarios or results. It derives visible scenarios from the underlying run and optionally filters them. + +This is important for the proposed default queue: + +- if `scenario_ids is None`, the queue automatically covers **all current scenarios in the run** +- if `user_ids is None`, the queue is effectively **unassigned** +- if the queue has no user filter, the scenario query path returns the run scenarios directly + +So the desired “default queue that follows future scenarios” already matches existing semantics **if** we leave `scenario_ids=None`. + +### 3. Scenario assignment is derived, not persisted per scenario + +Assignment behavior is computed from queue data at read time: + +- no `user_ids` -> everyone sees the run’s scenarios +- with `user_ids` -> `filter_scenario_ids(...)` deterministically partitions scenarios per repeat/user lane +- sequential vs randomized distribution is controlled by queue flags/settings + +That means the queue model already supports: + +- no assignees +- assignees per repeat lane +- repeated review lanes +- deterministic re-computation as scenarios are added later + +The subtle point is that **repeats live on the run**, while **assignment lanes live on the queue**. `SimpleQueuesService.create(...)` currently reconciles the two by setting run repeats to at least the number of assignment lanes. + +## Current Public/Service Surfaces + +### `SimpleEvaluationsService` + +This is a convenience wrapper over runs. It builds run steps from query/testset/application/evaluator revision IDs and exposes CRUD/lifecycle operations as “simple evaluations.” + +Notably: + +- evaluator inputs can be lists or explicit origin maps +- run flags are inferred from step origins +- it is run-first, not queue-first + +### `SimpleQueuesService` + +This is a different convenience wrapper over the same substrate. It: + +1. builds or reuses run data +2. creates a run with `is_queue=True` +3. creates one linked `EvaluationQueue` +4. stores queue-specific behavior such as assignments, step keys, and batching + +It is effectively a preset constructor for “evaluation run + annotation queue.” + +### Low-level evaluation endpoints + +The main evaluations API exposes separate resources for: + +- runs +- scenarios +- results +- metrics +- queues + +This exposes the true underlying shape more directly than either simple wrapper. + +### Annotations + +The annotations module is adjacent but distinct. It creates/edit annotations as trace-linked artifacts and may provision evaluators, but it is not the queue abstraction itself. The queue system is still implemented in evaluations. + +## How Simple Queues Work Today + +A simple queue is not a separate backend domain. It is a prescribed composition: + +1. Create an evaluation run whose input is either: + - direct traces/testcases, or + - source-backed queries/testsets +2. Add evaluator annotation steps. +3. Create one queue against that run. +4. Store only the annotation `step_keys` in the queue. +5. Optionally store assignments and batching settings. + +The queue then queries scenarios by: + +- starting from scenarios belonging to the run +- optionally applying `queue.data.scenario_ids` +- optionally applying user/repeat distribution + +That is why a queue with: + +- `scenario_ids=None` +- `step_keys=None` +- `user_ids=None` + +is the natural shape of the default queue. + +These are three independent axes: + +- scenario selection decides which scenarios are in the queue +- repeat assignment decides which scenario × repeat lanes a user gets +- step selection decides which steps must be completed for each assigned scenario × repeat + +`step_keys` do not participate in scenario or repeat selection, and they do not need to. Leaving them open is still the correct queue-level analogue of leaving `scenario_ids` open. + +## The Existing Proto-Unification Seam + +`EvaluationsService._ensure_human_annotation_queue(...)` currently does this: + +1. inspect run steps +2. collect human annotation step keys +3. if there are no human steps, do nothing +4. if any queue already exists for the run, do nothing +5. otherwise create an `EvaluationQueue` with: + - `run_id=run.id` + - `status=RUNNING` + - `step_keys=` + - no assignments + - no explicit scenario IDs + +This already gives the queue open scenario coverage and no default assignments, but it freezes step membership instead of leaving the queue open over the run’s steps. + +Today it is invoked from: + +- live run refresh before dispatch +- queue-backed trace/testcase batch evaluation dispatch + +That tells us two things: + +1. The architecture already treats queues as a natural companion to human annotation work. +2. The current behavior is still opportunistic and path-dependent, not a universal invariant of evaluation creation/editing. + +## What the Desired Default Queue Maps To in Current Terms + +| Desired behavior | Current primitive that already supports it | +|---|---| +| Queue linked to an evaluation | `EvaluationQueue.run_id` | +| No scenario selection; include all current/future scenarios | `queue.data.scenario_ids = None` | +| No assigned users by default | `queue.data.user_ids = None` | +| Cover all repeats | run-level `data.repeats`; queue assignment lanes can be absent | +| Step scope is not frozen | `queue.data.step_keys = None` | +| New scenarios added later become visible | queue scenario lookup derives from `run_id`, not a frozen list | + +So the cleanest first interpretation of a **default queue** is: + +```text +one canonical queue per evaluation run, +with no scenario restriction, +no step-key restriction, +and no assignees. +``` + +## Default Queue Policy + +The target model has two separate policies. + +### Structural policy + +This is the run-level condition: + +```text +has_human_evaluator_steps(run) +``` + +When default queues are conditional, this decides whether a run should currently have one. + +### Global lifecycle policy + +This is a configuration choice, for example: + +```text +EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS +``` + +When enabled: + +- create a default queue for every run +- never archive it merely because the run has no active human evaluators + +When disabled: + +- create or unarchive the default queue when the run has human evaluator steps +- archive the default queue when the run has no active human evaluator steps + +These policies are related but not interchangeable. The global setting defines whether default queues are unconditional. The structural rule only governs lifecycle when default queues are conditional. + +## Default Queue Lifecycle + +The desired queue identity is durable: + +- if the default queue does not exist and policy requires one, create it +- if it exists and is archived, unarchive it +- if it exists and is active, leave it alone +- if policy no longer requires it, archive it rather than hard-delete it + +This fits the broader evaluator model if evaluators are archived rather than removed. A queue can disappear from normal views while retaining identity and later return if human evaluator work becomes active again. + +The current queue API is not yet aligned with that lifecycle: + +- queues have lifecycle fields +- queue endpoints currently expose hard deletion +- queue queries do not yet expose `include_archived` +- queue lookup does not currently distinguish active from archived queues + +If default queues become durable linked objects, queue archive/unarchive operations and archived-aware lookup become part of the needed foundation. + +## Remaining Model Gaps + +### 1. `step_keys` are currently stored as a snapshot + +The current helper captures the human step keys that exist at creation time. The default queue should instead leave step scope open with `step_keys=None`, so later step changes do not require queue rewrites. + +### 2. There is no first-class distinction between default and custom queues + +Currently, a queue is just a queue. `_ensure_human_annotation_queue(...)` only checks whether **any** queue exists for the run. + +That creates ambiguity: + +- if a custom filtered queue exists, should it suppress creation of the evaluation’s default queue? +- if multiple queues exist, which one is the queue shown inside the evaluation? +- which archived queue should be restored when the default-queue invariant becomes true again? + +A stable default-queue marker or equivalent canonical linkage is needed once default queues and custom queues can coexist. + +### 3. `is_queue` still encodes a product distinction on the run + +Simple queues create runs with `is_queue=True`; simple evaluations generally do not. This flag is used for querying and queue-specific dispatch guards. + +If ordinary evaluations can have linked queues, `is_queue` should remain a technical execution flag rather than the signal that a run has a queue companion. + +### 4. Queue lifecycle is currently path-dependent + +Today automatic queue creation is reached from execution paths, not from the run mutation paths that define whether human work exists. + +Default-queue reconciliation belongs next to run creation/editing so it can enforce either: + +- unconditional queue existence, or +- conditional existence based on active human evaluator steps. + +## Multiple Human Evaluators, Assignments, and Repeats + +### Multiple human evaluators + +The run model supports many human evaluator steps already. A queue can target multiple annotation steps through `step_keys`. + +So there is no fundamental blocker to one default queue covering multiple human evaluators. The real question is product semantics: + +- should one task card represent one scenario with multiple human fields? +- or one scenario × evaluator step as separate queue work? + +The current queue primitive points at multiple step keys but scenario listing is scenario-oriented, not step-oriented. That suggests today’s model is closer to “one scenario can carry several annotation steps” than “each evaluator creates a separate queue item.” + +### Assignments + +The queue model already supports repeat-lane assignments: + +```text +user_ids = [[repeat_0 users], [repeat_1 users], ...] +``` + +A default queue with `user_ids=None` naturally means “unassigned.” + +What still needs product clarification is how assignment should behave when: + +- the evaluation repeat count increases later +- a human evaluator is added after assignments exist +- different human evaluators should have different assignee pools + +The current queue data model has one assignment matrix for the whole queue, not per evaluator step. If evaluators need separate assignment rules, one shared default queue may be insufficient or the model must evolve. + +### Repeats + +Repeats are owned by `EvaluationRunData.repeats`, not by the queue. Simple queue creation enforces: + +```text +run.repeats >= number of assignment lanes +``` + +That is compatible with a default queue that “covers all existing repeats,” provided the queue derives from the run rather than freezing repeat-specific scope. + +The open design question is what happens if repeats are later edited downward/upward after a queue already has assignments. The current storage model permits temporary mismatch. + +## Likely Design Direction + +### Recommended conceptual model + +Treat the queue as a **view/controller over human annotation work for an evaluation run**, not as a sibling product object that users must create manually. + +A practical backend direction would be: + +1. Every evaluation run may have one **canonical/default queue**. +2. The default queue is created automatically when the run first contains human annotation steps. +3. The default queue has: + - no scenario filter + - no assignments + - derived human-step membership, or managed synchronization +4. Additional custom queues may still exist for advanced filtered/assigned workflows. +5. The evaluation API should expose the canonical queue link directly so the UI can render the same queue both inside the evaluation and on the Queues surface. + +### Why this fits the current code well + +It reuses what already exists: + +- run/scenario/result storage +- evaluation queue storage +- scenario derivation by `run_id` +- assignment logic by repeat lane +- the already-present `_ensure_human_annotation_queue(...)` seam + +The largest required addition is not storage volume; it is **semantics**: + +- how to mark the canonical queue +- how default queue step membership stays current +- where invariant enforcement lives + +## Concrete Gaps to Resolve Before Implementation + +### Product/behavior questions + +1. Is the default queue only for **human** steps, or for all annotation steps? + - The current helper chooses human-only. + - The product statement sounds human-focused. + +2. With multiple human evaluators, is assignment shared across them or evaluator-specific? + - Current queue data supports shared assignment only. + +3. When a run already has a custom queue, should the default queue still exist? + - Current helper says no because it stops if *any* queue exists. + +4. If human evaluators are removed, should the default queue remain, archive, or disappear? + +5. Should all evaluation runs have a default queue immediately, or only runs with human work? + - The latter better matches current semantics and avoids empty queues for automatic-only runs. + +### Technical questions + +1. Should default queue membership be derived with `step_keys=None`, or synchronized explicitly? +2. How is “default queue” represented? +3. Should queue creation/update happen in service-layer run mutation methods rather than dispatch flows? +4. Does `is_queue` remain meaningful once ordinary evaluations can have linked queues? +5. What migration/backfill is needed for existing runs with human steps but no queue? + +## Candidate Implementation Shapes + +### Option A — Minimal evolution + +Keep explicit `step_keys`, add a default queue marker, and update `_ensure_human_annotation_queue(...)` plus run-edit paths to keep the default queue synced. + +**Pros** + +- smallest delta from current code +- easiest to reason about with existing queue reads +- preserves explicit custom queue semantics + +**Cons** + +- sync bugs remain possible +- every run edit touching human steps must remember to update the queue + +### Option B — Derived default queues + +Define `step_keys=None` on a default queue as “all current human annotation steps for this run.” Custom queues continue to use explicit step keys. + +**Pros** + +- best match for “follows the evaluation as it changes” +- new human evaluators appear automatically +- fewer synchronization paths + +**Cons** + +- requires read-time logic to distinguish default/derived queues from unconstrained queues +- needs crisp semantics for historical behavior and custom queues + +### Option C — No persisted default queue; virtualize it + +Do not persist a canonical queue. Derive one virtually from the run whenever human steps exist. + +**Pros** + +- no sync issue +- cleanest conceptual model + +**Cons** + +- weaker fit with “visible in Queues” unless the Queues API/UI also supports virtual resources +- assignments and future edits become awkward because there is no row to mutate + +**Current recommendation:** Option B looks like the strongest long-term fit, with Option A as the lower-risk incremental step if we want a small migration first. + +## Key References + +- `api/oss/src/core/evaluations/types.py` + - `EvaluationRun*` + - `EvaluationQueue*` + - `SimpleEvaluation*` + - `SimpleQueue*` +- `api/oss/src/core/evaluations/service.py` + - `EvaluationsService._ensure_human_annotation_queue(...)` + - `EvaluationsService.fetch_queue_scenarios(...)` + - `SimpleEvaluationsService` + - `SimpleQueuesService` +- `api/oss/src/core/evaluations/utils.py` + - `filter_scenario_ids(...)` +- `api/oss/src/dbs/postgres/evaluations/dao.py` + - queue CRUD and user filtering +- `api/oss/src/apis/fastapi/evaluations/router.py` + - low-level evaluation endpoints + - simple evaluation endpoints + - simple queue endpoints +- `api/oss/src/core/annotations/service.py` + - neighboring annotation abstraction, distinct from queueing + +## Bottom Line + +The backend already has the right primitive split for unification: + +- **evaluation run** = what is being evaluated +- **queue** = how human annotation work over that run is exposed/distributed + +The exploration should therefore avoid introducing a new “human evaluation” abstraction. The more promising path is to make a linked default queue a first-class, automatically maintained aspect of evaluation runs that contain human steps, while keeping custom queues as an advanced overlay when users need narrower assignment or filtering behavior. diff --git a/docs/designs/unify-evals-and-queues/unify-evals-extension-synthesis.md b/docs/designs/unify-evals-and-queues/unify-evals-extension-synthesis.md new file mode 100644 index 0000000000..832b12eae5 --- /dev/null +++ b/docs/designs/unify-evals-and-queues/unify-evals-extension-synthesis.md @@ -0,0 +1,263 @@ +# Unify Evals Extension Synthesis + +## Purpose + +This note captures the refined model that emerged after relating the queue-unification work to the parallel eval-loop unification work. + +The central clarification is that several concepts currently overloaded into “queue” should be separated: + +- source family +- default queue identity +- simple-queue eligibility +- queue lifecycle + +## Final Vocabulary + +### Source-family flags on runs + +Runs should expose distinct inferred flags for each source family: + +- `has_queries` +- `has_testsets` +- `has_traces` +- `has_testcases` + +These flags answer where scenarios come from and should drive: + +- validation +- topology classification +- source-family filtering +- mixed-input prevention + +They should replace the current tendency to infer direct trace/testcase behavior indirectly through `is_queue` plus synthetic step-key inspection. + +### `run.flags.is_queue` + +`is_queue` should become the persisted derived flag that answers: + +> Can this evaluation currently be interacted with through the simple annotation queue surface? + +The intended condition is: + +```text +active default queue exists +and active human evaluator work exists +``` + +This aligns the name with the product meaning and makes the flag directly useful for querying. + +It should be maintained eagerly, like the other persisted run flags, whenever: + +- the default queue is created +- the default queue is archived or unarchived +- active human evaluator work appears or disappears + +### `queue.flags.is_default` + +Queues need an explicit canonical-default marker: + +```text +queue.flags.is_default = true +``` + +Shape alone is not enough to identify the canonical queue, because a custom queue may coincidentally have the same open filters. + +## Default Queue Model + +A default queue is the canonical persisted queue view for a run. + +Its invariant shape is: + +```text +scenario_ids = None +step_keys = None +user_ids = None +``` + +and no queue-specific batching constraints. + +Interpretation: + +- no scenario filter -> all run scenarios +- no step filter -> all included steps +- no user assignments -> unassigned +- repeat coverage remains run-owned + +### Uniqueness + +There must be at most one default queue per run, including archived queues. + +Because queue flags are persisted JSONB, this can be enforced with a partial unique index over the materialized JSONB flag: + +```sql +CREATE UNIQUE INDEX ux_evaluation_queues_default_per_run +ON evaluation_queues (project_id, run_id) +WHERE (flags ->> 'is_default')::boolean = true; +``` + +An archived default queue still occupies the uniqueness slot. Reconciliation should unarchive it rather than create a duplicate. + +### Edit restrictions + +When `is_default=true`, editing must not allow: + +- scenario filters +- step-key filters +- assignments +- batching settings + +Default queues are canonical open views, not user-customizable slices. + +## Default Queue Lifecycle + +There are two policies. + +### Structural policy + +```text +has_active_human_evaluator_steps(run) +``` + +This says whether a run warrants a default queue when queues are conditional. + +### Global lifecycle policy + +A global policy toggle determines whether default queues are unconditional for all runs, for example: + +```text +EVALUATIONS_DEFAULT_QUEUES_FOR_ALL_RUNS +``` + +When enabled: + +- every run gets a default queue +- the default queue is not archived merely because active human work disappears + +When disabled: + +- active human work requires a default queue +- absence of active human work archives the default queue + +### Reconciliation behavior + +```text +required + missing -> create +required + archived -> unarchive +required + active -> no-op +not required + active -> archive +``` + +Queue lifecycle should use soft deletion for this behavior. Hard deletion may remain available separately where still needed. + +Queue queries need archived-aware support so reconciliation can restore the existing canonical row. + +## Simple Queue Semantics + +A `SimpleQueue` should be understood as: + +```text +a simplified human-work projection of an evaluation's default queue +``` + +not as a wrapper around runs that happen to use a special ingestion mode. + +### Eligibility + +A run is simple-queue eligible when: + +```text +run.flags.is_queue == true +``` + +under the redefined meaning above. + +That means all of these can appear through the simple queue surface when they have active human work and an active default queue: + +- query-backed evaluations +- testset-backed evaluations +- direct trace-backed evaluations +- direct testcase-backed evaluations + +Auto-only evaluations with an eager but empty default queue are not simple-queue eligible unless product later decides otherwise. + +### Identifiers + +Simple queue endpoints should remain queue-ID based. + +If a caller starts from a run, add one small lookup endpoint: + +```http +GET /evaluations/runs/{run_id}/default-queue +``` + +This returns the canonical queue resource or ID, after which existing simple queue endpoints can continue using queue IDs. + +There is no need for run-scoped archive/unarchive endpoints. + +## Relationship to Unified Eval Loops + +The parallel eval-loop work formalizes: + +```text +evaluation = graph + tensor + process(slice) +``` + +The queue model fits above that as: + +```text +default queue = canonical persisted human-work view over the tensor +``` + +The queue axes align naturally with tensor dimensions: + +- scenarios +- steps +- repeats + +A default queue is the open/default view over those dimensions. + +### Boundary + +The default queue is not orchestration. + +- eval runtime owns planning, processing, and tensor population +- queues own visibility, assignment, queue lifecycle, and user workflow + +## Shared Design Tension: Step Lifecycle + +The unified-loop design currently leans toward: + +```text +remove_step -> prune tensor cells +``` + +But if product semantics require evaluators or steps to be archived rather than hard-removed, the graph model needs to support active versus archived steps. + +That affects queue logic directly: + +- queue eligibility should depend on active human steps +- archived human steps may remain visible historically without keeping the queue active +- old tensor cells may remain instead of being pruned + +This needs explicit alignment with the eval-loop mutation model. + +## Recommended Backend Changes + +1. Add inferred run flags: + - `has_traces` + - `has_testcases` +2. Redefine persisted `run.flags.is_queue` as simple-queue eligibility. +3. Add `queue.flags.is_default`. +4. Enforce one default queue per run with a partial unique index over `flags.is_default`. +5. Reject filters/assignments/batching edits on default queues. +6. Add queue archive/unarchive support and archived-aware queries. +7. Add default-queue reconciliation tied to run creation/editing and queue lifecycle changes. +8. Persist and eagerly refresh `run.flags.is_queue` when the default queue or active human-work state changes. +9. Keep simple queue endpoints queue-ID based. +10. Add a small run-scoped lookup endpoint for the default queue: + +```http +GET /evaluations/runs/{run_id}/default-queue +``` + +11. Align eval-loop mutation semantics around active vs archived steps before hardening remove/prune behavior. diff --git a/docs/designs/unify-evals-and-queues/unify-evals-extension-verbatim.md b/docs/designs/unify-evals-and-queues/unify-evals-extension-verbatim.md new file mode 100644 index 0000000000..66e41f27e7 --- /dev/null +++ b/docs/designs/unify-evals-and-queues/unify-evals-extension-verbatim.md @@ -0,0 +1,533 @@ +Yes — I think this is a better model than the one I was circling around before. + +It gives each fact its own place instead of forcing `is_queue` to carry a hidden runtime meaning that does not match its name. + +## The revised flag model + +### Add inferred source flags + +Alongside: + +- `has_queries` +- `has_testsets` +- `has_evaluators` + +add: + +- `has_traces` +- `has_testcases` + +Then the source family becomes explicit: + +| Source shape | Flags | +|---|---| +| query-backed | `has_queries=True` | +| testset-backed | `has_testsets=True` | +| externally supplied traces | `has_traces=True` | +| externally supplied testcases | `has_testcases=True` | + +That gives us the right vocabulary for: + +- validation +- topology classification +- query filtering +- avoiding mixed source families +- distinguishing declared sources from externally supplied concrete items + +And importantly: + +> that source distinction does not need to be smuggled through `is_queue`. + +I like that a lot. + +## Then redefine `is_queue` + +Under your proposal: + +```text +is_queue = there is an active default queue + and at least one active human evaluator step +``` + +or more plainly: + +```text +this evaluation is currently interactable as a simple annotation queue +``` + +That is much more intuitive. + +It answers the product-relevant question: + +> Can this run be interacted with through the simple queue / annotation queue surface? + +And it can turn off when: +- the default queue is archived +- there are no active human evaluator steps + +That means `is_queue` is no longer an execution/input-source flag. It becomes a derived **human-work availability** flag. + +I think that is actually the right use of the name. + +## The clean separation becomes + +| Fact | Where represented | +|---|---| +| declared query source | `has_queries` | +| declared testset source | `has_testsets` | +| external trace source | `has_traces` | +| external testcase source | `has_testcases` | +| has evaluators | `has_evaluators` | +| has active human evaluators | `has_human` or eventually `has_active_human` if step archival becomes explicit | +| can be used via simple queues | `is_queue` | +| this particular queue is canonical | `EvaluationQueue.flags.is_default` | + +That is much clearer. + +--- + +# A few important consequences + +## 1. `is_queue` can no longer be derived from the run alone + +Today run flags are derived only from run data. + +Under this model, `is_queue` depends on: + +- run graph state +- queue existence +- queue archival state + +So we should stop thinking of all run flags as purely “derived from steps.” + +Some are graph-derived: +- `has_queries` +- `has_testsets` +- `has_traces` +- `has_testcases` +- `has_evaluators` +- `has_human` +- `has_auto` + +But `is_queue` is relationship/lifecycle-derived: +- active default queue exists +- active human work exists + +That is fine, but it means the reconciliation path needs to update the run flag whenever the default queue is created, archived, or unarchived. + +## 2. `has_traces` / `has_testcases` solve the current synthetic-source hack + +Right now the code has this awkward logic: + +```python +if flags.is_queue and not _references: + inspect step_key text for "query" or "testset" +``` + +That exists because direct queues use synthetic input steps and the system still wants to classify them as trace/testcase-ish through the old flags. + +With explicit: + +- `has_traces` +- `has_testcases` + +we can stop overloading: +- `has_queries` to mean “trace-like” +- `has_testsets` to mean “testcase-like” +- `is_queue` to help infer source family from missing references + +That should make validation and topology much more honest. + +## 3. Source-backed human evaluations can naturally be simple queues + +Then your desired product rule becomes straightforward: + +```text +simple queue eligible +iff +run.is_queue == True +``` + +because `is_queue` now means exactly: + +```text +active default queue + active human evaluator work +``` + +So: + +| Run | `is_queue` | +|---|---:| +| query-backed + human evaluator + active default queue | true | +| testset-backed + human evaluator + active default queue | true | +| direct traces + human evaluator + active default queue | true | +| direct testcases + human evaluator + active default queue | true | +| auto-only eval with eager default queue | false | +| human eval whose default queue is archived | false | + +That is much more useful than the current meaning. + +--- + +# On the default queue itself + +Yes: I think we need an explicit queue flag: + +```python +EvaluationQueueFlags.is_default: bool +``` + +because shape alone is not enough once custom queues exist. A custom queue could happen to also have: + +- no scenario filter +- no step filter +- no assignments + +and still not be the canonical default queue. + +## Default queue invariants + +For `is_default=True`, enforce: + +```text +scenario_ids is None +step_keys is None +user_ids is None +``` + +You mentioned “no repeat index”; today queues do not have a repeat-index filter field, so in current schema that translates to **no assignments**, because assignment lanes are the queue-level way repeats are partitioned. If repeat-index filtering is later added explicitly, then yes, default queues should also forbid that. + +And I agree with the behavior: + +- default queue archived → it still exists +- creating another default queue for same run should fail +- reconciliation should unarchive the existing one rather than create a second row + +So the invariant is: + +```text +at most one default queue per run, including archived queues +``` + +That likely wants a real uniqueness guarantee, not just service logic. Since `is_default` inside JSONB flags is awkward for constraints, I would at least consider making `is_default` a real DB column or adding an explicit indexed field rather than burying it only in JSON flags. If we keep it only inside JSONB, enforcement is still possible but less pleasant and less obvious. + +--- + +# On simple queue identifiers + +Yes — today the simple queue endpoints are queue-ID-centric. + +If after unification the simple queue is fundamentally: + +```text +the simple projection of the run’s default queue +``` + +then using the **run ID** is probably the better canonical identifier for that surface. + +Because the user-facing thing is increasingly “the evaluation’s queue,” not “some arbitrary queue row.” + +## Compatibility shape + +Your compatibility idea is reasonable: + +- if the identifier resolves to a queue ID, use that queue +- otherwise, if it resolves to a run ID, fetch that run’s default queue +- eventually prefer run IDs in the new API/UI paths + +Since both are UUIDs, this is a pragmatic compatibility shim rather than a semantically clean contract. It works, but the implementation should be careful and deterministic: + +1. try queue ID lookup first +2. if no queue exists, try default queue by run ID +3. if both could theoretically exist, queue ID wins + +UUID collision across tables is extraordinarily unlikely, but the precedence should still be explicit. + +Longer term, I would probably prefer: +- queue-specific endpoints remain queue-ID based +- simple queue endpoints become run-ID based +- maybe add explicit routes like `/simple/queues/by-run/{run_id}` during transition + +But as an internal compatibility move, your suggestion is sound. + +--- + +# The resulting model is much stronger + +I would now summarize it this way: + +```text +Source flags describe where scenarios come from: + has_queries + has_testsets + has_traces + has_testcases + +Queue flags describe queue identity: + is_default + +Run.is_queue describes product eligibility: + active default queue + active human evaluator work + +SimpleQueue is the simplified API view over: + the run's default queue +``` + +That gives each concern one home. + +It also removes the awkwardness where a run could “have a queue” but `is_queue=False`, or have `is_queue=True` mostly because it accepts source batches. Under your model, the name and the meaning line up again. + +## One thing I would still decide explicitly + +Whether `is_queue` should be persisted at all or always recomputed. + +Because it depends on both: +- run state +- queue state + +If it is persisted, reconciliation must update it transactionally whenever: +- default queue created +- default queue archived/unarchived +- active human evaluator steps appear/disappear + +If it is recomputed, queries become more expensive or need joins. + +Given that you explicitly want it queryable, persisting it as a denormalized derived flag is reasonable — but then I would treat **default queue reconciliation + run flag refresh** as one logical operation. + +This is a much cleaner design than the earlier alternatives. + +--- + +Yes — on both points. + +## 1. Uniqueness for the default queue + +`scenario_ids`, `step_keys`, and `user_ids` all currently live inside `EvaluationQueueData`, which is stored in the queue’s JSON `data` column. So if we define the default queue only by the shape: + +```text +scenario_ids is None +step_keys is None +user_ids is None +``` + +then a normal relational unique index over those fields is not straightforward. + +You could technically build expression indexes over JSON paths, but that would be a poor foundation for a first-class invariant like: + +```text +at most one default queue per run +``` + +I think your instinct is right: if **defaultness** matters enough to enforce uniqueness, it wants a first-class indexed column. + +### Recommended shape + +Add something like: + +```python +is_default: bool +``` + +as a real queue column, even if it is also mirrored into DTO flags for API symmetry. + +Then enforce: + +```text +unique(project_id, run_id) where is_default = true +``` + +That gives you the exact invariant you want: + +- only one default queue per run +- archived default queue still occupies the slot +- trying to create another default queue fails +- reconciliation can find and unarchive the existing one + +And separately, enforce in service validation: + +```text +if is_default: + scenario_ids must be None + step_keys must be None + user_ids must be None + batch settings must be None +``` + +The uniqueness invariant should key on **identity** (`is_default`), not on a coincidental combination of empty filters. + +That is much safer than saying “whichever queue happens to have no filters is default.” + +## 2. Simple queues can stay queue-ID based if default queue lookup is easy + +I agree with your revised take. + +If we add a very small direct way to fetch the default queue for a run, then we do **not** need to overload simple queue identifiers with run IDs. + +Something like: + +```http +GET /evaluations/runs/{run_id}/queue +``` + +or more explicitly: + +```http +GET /evaluations/runs/{run_id}/default-queue +``` + +would be enough. + +Then the flow becomes: + +1. evaluation screen has `run_id` +2. fetch default queue in one cheap call +3. receive `queue_id` +4. use existing simple queue endpoints with `queue_id` + +That preserves the clean existing identity model: +- queue resources use queue IDs +- runs use run IDs +- no ambiguous UUID interpretation +- no compatibility shim needed unless product wants one for some other reason + +I’d favor: + +```http +GET /evaluations/runs/{run_id}/default-queue +``` + +because it is unambiguous and leaves room for other queue relationships later. + +Possibly also: + +```http +POST /evaluations/runs/{run_id}/default-queue/archive +POST /evaluations/runs/{run_id}/default-queue/unarchive +``` + +if we want run-scoped convenience operations, though the generic queue archive/unarchive endpoints may already be enough. + +## 3. Persisting `run.flags.is_queue` + +I agree with you. + +Because we want: + +- cheap filtering +- parity with the other persisted inferred run flags +- no joins for the common read paths +- and queue archival/unarchival is a meaningful state transition anyway + +then `is_queue` should be persisted and eagerly maintained. + +Under the new meaning: + +```text +run.flags.is_queue = + active default queue exists + and active human evaluator work exists +``` + +And yes, that means queue lifecycle operations must update the run flags eagerly: + +| Event | `run.flags.is_queue` effect | +|---|---| +| default queue created and human work exists | `true` | +| default queue unarchived and human work exists | `true` | +| default queue archived | `false` | +| last active human evaluator archived | `false` | +| human evaluator added/restored while default queue active | `true` | + +This is analogous to the other denormalized flags: +- they are persisted +- they are inferred/maintained eagerly +- query paths stay cheap + +The only thing I would be careful about is **where** that synchronization lives. I would want one narrow service-level operation that owns the invariant, rather than scattered manual edits: + +```text +reconcile_default_queue_and_run_flags(...) +``` + +or maybe two lower-level pieces used transactionally: + +```text +reconcile_default_queue(...) +refresh_run_flags(...) +``` + +But conceptually, yes: persist it and keep it in sync. + +## Updated model after your refinements + +```text +EvaluationRunFlags + has_queries + has_testsets + has_traces + has_testcases + has_evaluators + has_human + has_auto + is_queue # can be used via simple queue surface + +EvaluationQueue + is_default # first-class indexed identity field + +Default queue invariant + at most one per run + scenario_ids=None + step_keys=None + user_ids=None + no batching constraints + +Simple queue endpoints + remain queue-ID based + +Default queue lookup + add a direct run-scoped endpoint to get queue_id cheaply +``` + +That feels quite settled to me now. + +--- + +Yes — if `is_default` is materialized inside the queue `flags` JSONB, then Postgres can enforce uniqueness from that directly. We do **not** necessarily need a dedicated physical column just for the index. + +For example, conceptually: + +```sql +CREATE UNIQUE INDEX ux_evaluation_queues_default_per_run +ON evaluation_queues (project_id, run_id) +WHERE (flags ->> 'is_default')::boolean = true; +``` + +That would give us the important invariant: + +```text +at most one default queue per run, including archived rows +``` + +because archived rows still satisfy the partial-index predicate. + +So yes: if we are comfortable treating `flags.is_default` as a materialized persisted field, a partial unique index over the JSONB expression is enough. + +The tradeoff is mainly ergonomics: +- **JSONB expression index**: no extra column, keeps queue flags grouped together +- **real column**: easier to inspect/query/index conventionally + +But technically, your approach is sound, and in this codebase it may be the more consistent choice if other queue flags already live in JSONB. + +And agreed: **no** to run-scoped archive/unarchive endpoints. + +We only need: +- generic queue archive/unarchive operations +- plus a simple way to fetch the default queue for a run + +So the useful addition is something like: + +```http +GET /evaluations/runs/{run_id}/default-queue +``` + +but not special lifecycle endpoints hanging off the run. diff --git a/examples/python/evaluators/ag/secrets_check.py b/examples/python/evaluators/ag/secrets_check.py index 2427ecc81b..1754a7848f 100644 --- a/examples/python/evaluators/ag/secrets_check.py +++ b/examples/python/evaluators/ag/secrets_check.py @@ -35,7 +35,7 @@ def evaluate( ) response = requests.get( - f"{host}/api/vault/v1/secrets/", + f"{host}/api/secrets/", headers=headers, timeout=10, ) diff --git a/hosting/docker-compose/ee/docker-compose.dev.yml b/hosting/docker-compose/ee/docker-compose.dev.yml index c7b3198d1b..812578fa9c 100644 --- a/hosting/docker-compose/ee/docker-compose.dev.yml +++ b/hosting/docker-compose/ee/docker-compose.dev.yml @@ -143,7 +143,7 @@ services: image: agenta-ee-dev-api:latest # === EXECUTION ============================================ # command: > - watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive -- + watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive --ignore-patterns=*/tests/* -- python -m entrypoints.worker_evaluations # === STORAGE ============================================== # volumes: @@ -180,7 +180,7 @@ services: image: agenta-ee-dev-api:latest # === EXECUTION ============================================ # command: > - watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive -- + watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive --ignore-patterns=*/tests/* -- python -m entrypoints.worker_tracing # === STORAGE ============================================== # volumes: @@ -217,7 +217,7 @@ services: image: agenta-ee-dev-api:latest # === EXECUTION ============================================ # command: > - watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive -- + watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive --ignore-patterns=*/tests/* -- python -m entrypoints.worker_webhooks # === STORAGE ============================================== # volumes: @@ -260,7 +260,7 @@ services: image: agenta-ee-dev-api:latest # === EXECUTION ============================================ # command: > - watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive -- + watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive --ignore-patterns=*/tests/* -- python -m entrypoints.worker_events # === STORAGE ============================================== # volumes: diff --git a/hosting/docker-compose/oss/docker-compose.dev.yml b/hosting/docker-compose/oss/docker-compose.dev.yml index 942d528664..3e320ebe57 100644 --- a/hosting/docker-compose/oss/docker-compose.dev.yml +++ b/hosting/docker-compose/oss/docker-compose.dev.yml @@ -141,7 +141,7 @@ services: image: agenta-oss-dev-api:latest # === EXECUTION ============================================ # command: > - watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive -- + watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive --ignore-patterns=*/tests/* -- python -m entrypoints.worker_evaluations # === STORAGE ============================================== # volumes: @@ -178,7 +178,7 @@ services: image: agenta-oss-dev-api:latest # === EXECUTION ============================================ # command: > - watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive -- + watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive --ignore-patterns=*/tests/* -- python -m entrypoints.worker_tracing # === STORAGE ============================================== # volumes: @@ -215,7 +215,7 @@ services: image: agenta-oss-dev-api:latest # === EXECUTION ============================================ # command: > - watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive -- + watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive --ignore-patterns=*/tests/* -- python -m entrypoints.worker_webhooks # === STORAGE ============================================== # volumes: @@ -258,7 +258,7 @@ services: image: agenta-oss-dev-api:latest # === EXECUTION ============================================ # command: > - watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive -- + watchmedo auto-restart --directory=/app/ --pattern=*.py --recursive --ignore-patterns=*/tests/* -- python -m entrypoints.worker_events # === STORAGE ============================================== # volumes: diff --git a/sdks/python/agenta/sdk/engines/running/errors.py b/sdks/python/agenta/sdk/engines/running/errors.py index 9cbd2c1f33..6b68465681 100644 --- a/sdks/python/agenta/sdk/engines/running/errors.py +++ b/sdks/python/agenta/sdk/engines/running/errors.py @@ -340,6 +340,19 @@ def __init__(self, message: str, stacktrace: Optional[str] = None): ) +class MockV0Error(ErrorStatus): + code: int = 500 + type: str = f"{ERRORS_BASE_URL}#v0:workflows:mock-error" + + def __init__(self, message: str, stacktrace: Optional[str] = None): + super().__init__( + code=self.code, + type=self.type, + message=message, + stacktrace=stacktrace, + ) + + class SnippetV0Error(ErrorStatus): code: int = 500 type: str = f"{ERRORS_BASE_URL}#v0:workflows:snippet-error" diff --git a/sdks/python/agenta/sdk/engines/running/handlers.py b/sdks/python/agenta/sdk/engines/running/handlers.py index 871f918af9..4499943616 100644 --- a/sdks/python/agenta/sdk/engines/running/handlers.py +++ b/sdks/python/agenta/sdk/engines/running/handlers.py @@ -6,6 +6,7 @@ import socket import ipaddress import traceback +from inspect import isawaitable from difflib import SequenceMatcher from json import dumps, loads from typing import Any, Dict, List, Optional, Union, Tuple @@ -60,6 +61,7 @@ WebhookServerV0Error, MatchV0Error, CodeV0Error, + MockV0Error, ConfigV0Error, FeedbackV0Error, ) @@ -330,8 +332,7 @@ def auto_exact_match_v0( Returns: Evaluation result with success flag (True for match, False for mismatch) """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} correct_answer_key = str(parameters.get("correct_answer_key", "correct_answer")) @@ -372,8 +373,7 @@ def auto_regex_test_v0( Returns: Evaluation result with success flag """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "regex_pattern" not in parameters: raise MissingConfigurationParameterV0Error(path="regex_pattern") @@ -430,18 +430,14 @@ def field_match_test_v0( Returns: Evaluation result with success flag """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "json_field" not in parameters: raise MissingConfigurationParameterV0Error(path="json_field") json_field = str(parameters["json_field"]) - if "correct_answer_key" not in parameters: - raise MissingConfigurationParameterV0Error(path="correct_answer_key") - - correct_answer_key = str(parameters["correct_answer_key"]) + correct_answer_key = str(parameters.get("correct_answer_key", "correct_answer")) if inputs is None or not isinstance(inputs, dict): raise InvalidInputsV0Error(expected="dict", got=inputs) @@ -526,8 +522,7 @@ def json_multi_field_match_v0( Dict with per-field scores and aggregate_score, e.g.: {"name": 1.0, "email": 0.0, "aggregate_score": 0.5} """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "fields" not in parameters: raise MissingConfigurationParameterV0Error(path="fields") @@ -541,10 +536,7 @@ def json_multi_field_match_v0( got=fields, ) - if "correct_answer_key" not in parameters: - raise MissingConfigurationParameterV0Error(path="correct_answer_key") - - correct_answer_key = str(parameters["correct_answer_key"]) + correct_answer_key = str(parameters.get("correct_answer_key", "correct_answer")) if inputs is None or not isinstance(inputs, dict): raise InvalidInputsV0Error(expected="dict", got=inputs) @@ -637,8 +629,7 @@ async def auto_webhook_test_v0( Returns: Evaluation result with score from the webhook """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "webhook_url" not in parameters: raise MissingConfigurationParameterV0Error(path="webhook_url") @@ -756,8 +747,7 @@ async def auto_custom_code_run_v0( Returns: Evaluation result with score from the custom code """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "code" not in parameters: raise MissingConfigurationParameterV0Error(path="code") @@ -823,10 +813,7 @@ def _run_v2() -> Any: ) from e def _run_v1() -> Any: - if "correct_answer_key" not in parameters: - raise MissingConfigurationParameterV0Error(path="correct_answer_key") - - correct_answer_key = str(parameters["correct_answer_key"]) + correct_answer_key = str(parameters.get("correct_answer_key", "correct_answer")) if inputs is None or not isinstance(inputs, dict): raise InvalidInputsV0Error(expected="dict", got=inputs) @@ -888,8 +875,7 @@ async def auto_ai_critique_v0( Returns: Evaluation result with score from the AI """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} correct_answer_key = parameters.get("correct_answer_key") @@ -1109,8 +1095,7 @@ def auto_starts_with_v0( Returns: Evaluation result with success flag """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "prefix" not in parameters: raise MissingConfigurationParameterV0Error(path="prefix") @@ -1158,8 +1143,7 @@ def auto_ends_with_v0( Returns: Evaluation result with success flag """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "suffix" not in parameters: raise MissingConfigurationParameterV0Error(path="suffix") @@ -1207,8 +1191,7 @@ def auto_contains_v0( Returns: Evaluation result with success flag """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "substring" not in parameters: raise MissingConfigurationParameterV0Error(path="substring") @@ -1256,8 +1239,7 @@ def auto_contains_any_v0( Returns: Evaluation result with success flag """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "substrings" not in parameters: raise MissingConfigurationParameterV0Error(path="substrings") @@ -1314,8 +1296,7 @@ def auto_contains_all_v0( Returns: Evaluation result with success flag """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "substrings" not in parameters: raise MissingConfigurationParameterV0Error(path="substrings") @@ -1414,13 +1395,9 @@ def auto_json_diff_v0( Returns: Evaluation result with score only (no diff explanation) """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) - - if "correct_answer_key" not in parameters: - raise MissingConfigurationParameterV0Error(path="correct_answer_key") + parameters = parameters or {} - correct_answer_key = str(parameters["correct_answer_key"]) + correct_answer_key = str(parameters.get("correct_answer_key", "correct_answer")) if inputs is None or not isinstance(inputs, dict): raise InvalidInputsV0Error(expected="dict", got=inputs) @@ -1511,13 +1488,9 @@ def auto_levenshtein_distance_v0( Dictionary with normalized similarity score (0 to 1), or error message if evaluation fails. """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} - if "correct_answer_key" not in parameters: - raise MissingConfigurationParameterV0Error(path="correct_answer_key") - - correct_answer_key = str(parameters["correct_answer_key"]) + correct_answer_key = str(parameters.get("correct_answer_key", "correct_answer")) case_sensitive = parameters.get("case_sensitive", True) is True @@ -1616,13 +1589,9 @@ def auto_similarity_match_v0( Returns: Evaluation result with similarity score """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} - if "correct_answer_key" not in parameters: - raise MissingConfigurationParameterV0Error(path="correct_answer_key") - - correct_answer_key = str(parameters["correct_answer_key"]) + correct_answer_key = str(parameters.get("correct_answer_key", "correct_answer")) case_sensitive = parameters.get("case_sensitive", True) is True @@ -1709,13 +1678,9 @@ async def auto_semantic_similarity_v0( Returns: Evaluation result with cosine similarity score """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) - - if "correct_answer_key" not in parameters: - raise MissingConfigurationParameterV0Error(path="correct_answer_key") + parameters = parameters or {} - correct_answer_key = str(parameters["correct_answer_key"]) + correct_answer_key = str(parameters.get("correct_answer_key", "correct_answer")) embedding_model = parameters.get("embedding_model", "text-embedding-3-small") @@ -2143,7 +2108,7 @@ async def completion_v0( required_keys = set(config.prompt.input_keys) provided_keys = set(_variables.keys()) - if required_keys != provided_keys: + if not required_keys.issubset(provided_keys): raise InvalidInputsV0Error( expected=sorted(required_keys), got=sorted(provided_keys), @@ -2217,7 +2182,7 @@ async def chat_v0( required_keys = set(config.prompt.input_keys) - {"messages"} provided_keys = set(_variables.keys()) - if required_keys != provided_keys: + if not required_keys.issubset(provided_keys): raise InvalidInputsV0Error( expected=sorted(required_keys), got=sorted(provided_keys), @@ -2857,7 +2822,10 @@ async def code_v0( The raw dict / str when code returns one of those. """ if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + raise InvalidConfigurationParametersV0Error( + expected="dict", + got=parameters, + ) if "code" not in parameters: raise MissingConfigurationParameterV0Error(path="code") @@ -2933,6 +2901,128 @@ async def config_v0( ) +# --------------------------------------------------------------------------- +# mock_v0 — deterministic, LLM-free, sandbox-free workflow for testing +# --------------------------------------------------------------------------- +# +# A single handler that stands in for BOTH an application (invocation step, +# returns workflow outputs) and an evaluator (annotation step, returns +# {"score", "success"}). The behavior is selected by name so evaluations can +# run end-to-end through the real worker without any LLM call or code sandbox. +# +# Parameter contract (mirrors the LLM `MOCKS[key](**kwargs)` selector pattern): +# parameters = {"key": "", "kwargs": {...}} +# +# App-role selectors return outputs; evaluator-role selectors return a result. + + +def _mock_echo(*, inputs=None, **_) -> Any: + # App-role: echo the inputs back as the workflow output. + return inputs or {} + + +def _mock_static(*, output=None, **_) -> Any: + # App-role: return a fixed payload supplied via kwargs. + return output if output is not None else {} + + +def _mock_pass(**_) -> Any: + # Evaluator-role: always passing result. + return {"score": 1.0, "success": True} + + +def _mock_fail(**_) -> Any: + # Evaluator-role: always failing result. + return {"score": 0.0, "success": False} + + +def _mock_score(*, score=0.0, threshold=0.5, **_) -> Any: + # Evaluator-role: explicit score with threshold-based success. + _score = float(score) + return {"score": _score, "success": _score >= float(threshold)} + + +def _mock_error(*, message="mock_v0 error behavior", **_) -> Any: + # Failure path (shared by app and evaluator roles). + raise MockV0Error(message=str(message)) + + +async def _mock_delay(*, seconds=0.1, then="pass", inputs=None, **kwargs) -> Any: + # Sleep then defer to another (synchronous) behavior. Useful to exercise + # RUNNING/timing without an LLM. `then` must name a non-delay behavior. + await asyncio.sleep(float(seconds)) + behavior = MOCK_V0_BEHAVIORS.get(str(then), _mock_pass) + if behavior is _mock_delay: + behavior = _mock_pass + return behavior(inputs=inputs, **kwargs) + + +MOCK_V0_BEHAVIORS = { + # app-role + "echo": _mock_echo, + "static": _mock_static, + # evaluator-role + "pass": _mock_pass, + "fail": _mock_fail, + "score": _mock_score, + # shared + "error": _mock_error, + "delay": _mock_delay, +} + + +@instrument() +async def mock_v0( + request: Optional[Data] = None, + revision: Optional[Data] = None, + inputs: Optional[Data] = None, + parameters: Optional[Data] = None, + outputs: Optional[Union[Data, str]] = None, + trace: Optional[Data] = None, + testcase: Optional[Data] = None, +) -> Any: + """ + Deterministic mock workflow (agenta:custom:mock:v0). + + Selects a predefined behavior by name — no LLM, no code sandbox. Serves as + both an application (invocation) and an evaluator (annotation) depending on + the selected behavior. + + Parameters: + key: selector name — one of MOCK_V0_BEHAVIORS + (echo, static, pass, fail, score, error, delay). + kwargs: structured args for the selected behavior (optional). + + Returns: + The behavior's output (workflow outputs for app-role selectors, or + {"score", "success"} for evaluator-role selectors). + """ + parameters = parameters or {} + + if "key" not in parameters: + raise MissingConfigurationParameterV0Error(path="key") + + key = str(parameters["key"]) + if key not in MOCK_V0_BEHAVIORS: + raise InvalidConfigurationParameterV0Error( + path="key", + expected=list(MOCK_V0_BEHAVIORS.keys()), + got=key, + ) + + kwargs = parameters.get("kwargs") or {} + if not isinstance(kwargs, dict): + raise InvalidConfigurationParameterV0Error( + path="kwargs", expected="dict", got=kwargs + ) + + behavior = MOCK_V0_BEHAVIORS[key] + result = behavior(inputs=inputs, outputs=outputs, trace=trace, **kwargs) + if isawaitable(result): + result = await result + return result + + async def match_v0( request: Optional[Data] = None, revision: Optional[Data] = None, @@ -2970,8 +3060,7 @@ async def match_v0( Returns: {key: result_node, ..., "score": float, "success": bool} — flat result dict """ - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} if "matchers" not in parameters: raise MissingConfigurationParameterV0Error(path="matchers") @@ -3461,8 +3550,7 @@ async def llm_v0( from agenta.sdk.engines.running.errors import LLMUnavailableV0Error # --- Validate parameters - if parameters is None or not isinstance(parameters, dict): - raise InvalidConfigurationParametersV0Error(expected="dict", got=parameters) + parameters = parameters or {} llms = parameters.get("llms") if not llms or not isinstance(llms, list): diff --git a/sdks/python/agenta/sdk/engines/running/interfaces.py b/sdks/python/agenta/sdk/engines/running/interfaces.py index b7d9a0569c..03eb6b4ed6 100644 --- a/sdks/python/agenta/sdk/engines/running/interfaces.py +++ b/sdks/python/agenta/sdk/engines/running/interfaces.py @@ -176,6 +176,11 @@ def llm_inputs_schema( schemas=None, ) +mock_v0_interface = WorkflowRevisionData( + uri="agenta:custom:mock:v0", + schemas=None, +) + config_v0_interface = WorkflowRevisionData( uri="agenta:custom:config:v0", schemas=None, diff --git a/sdks/python/agenta/sdk/engines/running/utils.py b/sdks/python/agenta/sdk/engines/running/utils.py index f350dba905..da84036e5a 100644 --- a/sdks/python/agenta/sdk/engines/running/utils.py +++ b/sdks/python/agenta/sdk/engines/running/utils.py @@ -12,6 +12,7 @@ feedback_v0, hook_v0, code_v0, + mock_v0, config_v0, match_v0, llm_v0, @@ -43,6 +44,7 @@ feedback_v0_interface, hook_v0_interface, code_v0_interface, + mock_v0_interface, config_v0_interface, match_v0_interface, llm_v0_interface, @@ -76,6 +78,7 @@ feedback=dict(v0=feedback_v0_interface), hook=dict(v0=hook_v0_interface), code=dict(v0=code_v0_interface), + mock=dict(v0=mock_v0_interface), snippet=dict(v0=config_v0_interface), ), builtin=dict( @@ -337,6 +340,7 @@ def _catalog_entry() -> dict: hook=dict(v0=hook_v0), snippet=dict(v0=config_v0), code=dict(v0=code_v0), + mock=dict(v0=mock_v0), ), builtin=dict( # --- NEW URI @@ -534,6 +538,8 @@ def infer_url_from_uri(uri: Optional[str]) -> Optional[str]: ("custom", "code"): (True, True, False), ("custom", "hook"): (True, True, False), ("custom", "feedback"): (True, True, False), + # agenta:custom:mock — deterministic test workflow (app + evaluator, no LLM/sandbox) + ("custom", "mock"): (True, True, False), # agenta:builtin:* — application-only (not evaluators) ("builtin", "chat"): (True, False, False), ("builtin", "completion"): (True, False, False), diff --git a/sdks/python/agenta/sdk/evaluations/metrics.py b/sdks/python/agenta/sdk/evaluations/metrics.py index c08a035352..d9dde933b5 100644 --- a/sdks/python/agenta/sdk/evaluations/metrics.py +++ b/sdks/python/agenta/sdk/evaluations/metrics.py @@ -7,6 +7,42 @@ # TODO: ADD TYPES +async def aquery_global( + *, + run_id: UUID, +) -> Optional[EvaluationMetrics]: + """Read back the GLOBAL (whole-run) metric row for a run. + + Mirrors `POST /evaluations/metrics/query` with the global selector + `scenario_ids=False, timestamps=False` — the DAO reads these bools as + "scenario_id IS NULL" and "timestamp IS NULL", i.e. the single aggregate row + (not the per-scenario/variational or temporal rows). The SDK calls this + after executing+refreshing to surface the run's headline metrics. + """ + response = authed_api()( + method="POST", + endpoint="/evaluations/metrics/query", + json=dict( + metrics=dict( + run_id=str(run_id), + scenario_ids=False, + timestamps=False, + ) + ), + ) + + try: + response.raise_for_status() + except Exception: + print(response.text) + raise + + response = response.json() + + metrics = [EvaluationMetrics(**m) for m in response.get("metrics", [])] + return metrics[0] if metrics else None + + async def arefresh( run_id: UUID, scenario_id: Optional[UUID] = None, diff --git a/sdks/python/agenta/sdk/evaluations/preview/evaluate.py b/sdks/python/agenta/sdk/evaluations/preview/evaluate.py index 8747b294fa..87afe1a0ac 100644 --- a/sdks/python/agenta/sdk/evaluations/preview/evaluate.py +++ b/sdks/python/agenta/sdk/evaluations/preview/evaluate.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Any, Union, Optional, Tuple +from typing import Dict, List, Any, Union, Optional from uuid import UUID from copy import deepcopy from datetime import datetime @@ -10,17 +10,6 @@ Target, SimpleEvaluationData, ) -from agenta.sdk.models.shared import Link, Reference -from agenta.sdk.models.workflows import ( - ApplicationRevision, - EvaluatorRevision, - WorkflowServiceRequestData, - ApplicationServiceRequest, - EvaluatorServiceRequest, -) -from agenta.sdk.models.testsets import TestsetRevision - -from agenta.sdk.evaluations.preview.utils import fetch_trace_data from agenta.sdk.managers.testsets import ( acreate as acreate_testset, @@ -35,25 +24,30 @@ aretrieve as aretrieve_evaluator, ) from agenta.sdk.evaluations.runs import ( + RunData, acreate as acreate_run, aclose as aclose_run, aurl as aget_url, ) from agenta.sdk.evaluations.scenarios import ( - acreate as aadd_scenario, + aadd as aadd_scenarios, + aedit_scenario, ) from agenta.sdk.evaluations.results import ( - acreate as alog_result, + apopulate as apopulate_slice, ) from agenta.sdk.evaluations.metrics import ( - arefresh as acompute_metrics, + arefresh, + aquery_global as aquery_metrics, +) +from agenta.sdk.evaluations.runtime.processor import process_sources +from agenta.sdk.evaluations.runtime.executor import AsyncioEvaluationTaskRunner +from agenta.sdk.evaluations.runtime.adapters import ( + SDKWorkflowRunner, ) +from agenta.sdk.evaluations.preview.utils import afetch_trace -from agenta.sdk.decorators.running import ( - invoke_application, - invoke_evaluator, -) from agenta.sdk.utils.logging import get_module_logger @@ -169,9 +163,59 @@ async def _parse_evaluate_kwargs( return simple_evaluation_data -async def _upsert_entities( +async def _resolve_testset_steps_to_revisions( + testset_steps: Any, +) -> Dict[str, Origin]: + """Pin each testset step key to a concrete revision id. + + A normalized testset step is `{id: origin}`, but `id` may be a testset id or + a revision id. Resolve each: try it as a revision id, fall back to a testset + id (latest revision). The backend persists these step refs verbatim and does + no JIT resolution, so this must happen before the run is created. + """ + if not testset_steps or not isinstance(testset_steps, dict): + return testset_steps + + resolved: Dict[str, Origin] = {} + for testset_id_str, origin in testset_steps.items(): + try: + testset_uuid = UUID(str(testset_id_str)) + except Exception: + continue + + testset_revision = await aretrieve_testset( + testset_revision_id=testset_uuid, + ) + + if not testset_revision or not testset_revision.id: + # Fallback: treat as testset_id (latest revision) + testset_revision = await aretrieve_testset( + testset_id=testset_uuid, + ) + + if testset_revision and testset_revision.id: + resolved[str(testset_revision.id)] = origin + + return resolved + + +async def _resolve_entities( simple_evaluation_data: SimpleEvaluationData, ) -> SimpleEvaluationData: + """Resolve a parsed draft into a revision-pinned SimpleEvaluationData. + + The draft's `*_steps` arrive loose (a `Target`: revision ids, entity ids, + callables, or inline testcase data). This is THE phase that turns each into + a `{revision_id: origin}` dict — its postcondition is the invariant the rest + of the pipeline relies on: every step map is keyed by a concrete revision id. + + Two mechanisms get there: + - mint: callable -> revision (`aupsert_*`), inline data -> testset revision + (`acreate_testset`); these return revision ids directly; + - disambiguate: a passed testset UUID may be a testset id OR a revision id, + so testset steps are pinned to revision ids here (no JIT resolution in + the backend, which persists these step refs verbatim on the run). + """ if simple_evaluation_data.testset_steps: if isinstance(simple_evaluation_data.testset_steps, list): testset_steps: Dict[str, Origin] = {} @@ -208,6 +252,14 @@ async def _upsert_entities( step_name="testset steps", ) + # Pin testset steps to revision ids. A passed UUID may be a testset id or a + # revision id; the backend does no JIT resolution and persists these step + # refs verbatim on the run, so disambiguate to revision ids here. (Apps and + # evaluators already resolve to revision ids above, via id or upsert.) + simple_evaluation_data.testset_steps = await _resolve_testset_steps_to_revisions( + simple_evaluation_data.testset_steps + ) + if simple_evaluation_data.application_steps: if isinstance(simple_evaluation_data.application_steps, list): application_steps: Dict[str, Origin] = {} @@ -289,60 +341,57 @@ async def _upsert_entities( return simple_evaluation_data -async def _retrieve_entities( - simple_evaluation_data: SimpleEvaluationData, -) -> Tuple[ - Dict[UUID, TestsetRevision], - Dict[UUID, ApplicationRevision], - Dict[UUID, EvaluatorRevision], -]: - testset_revisions: Dict[UUID, TestsetRevision] = {} - for testset_ref, origin in simple_evaluation_data.testset_steps.items(): - testset_revision = await aretrieve_testset( - testset_revision_id=testset_ref, - ) - - if not testset_revision or not testset_revision.id: - testset_revision = await aretrieve_testset( - testset_id=testset_ref, - ) - - if not testset_revision or not testset_revision.id: - continue - - testset_revisions[testset_revision.id] = testset_revision - - application_revisions: Dict[UUID, ApplicationRevision] = {} - for ( - application_revision_id, - origin, - ) in simple_evaluation_data.application_steps.items(): - application_revision = await aretrieve_application( - application_revision_id=application_revision_id, - ) - - if not application_revision: - continue - - application_revisions[application_revision_id] = application_revision - - evaluator_revisions: Dict[UUID, EvaluatorRevision] = {} - for evaluator_revision_id, origin in simple_evaluation_data.evaluator_steps.items(): - evaluator_revision = await aretrieve_evaluator( - evaluator_revision_id=evaluator_revision_id, - ) +def _timestamp_suffix(): + suffix = datetime.now().strftime("%y-%m-%d · %H:%M") + return f" [{suffix}]" - if not evaluator_revision: - continue - evaluator_revisions[evaluator_revision_id] = evaluator_revision +async def _prepare_run_data( + *, + name: Optional[str], + description: Optional[str], + testsets: Optional[Target], + applications: Optional[Target], + evaluators: Optional[Target], + repeats: Optional[int], + specs: Optional[Union[EvaluateSpecs, Dict[str, Any]]], +) -> RunData: + """Turn the raw `evaluate()` kwargs into the `RunData` for `acreate_run`. + + The full input phase: + 1. parse — pack the loose kwargs into a draft (`*_steps` still a `Target`: + ids, callables, or inline data); + 2. resolve — mint + disambiguate every step to a revision id, so the + revision-pinned invariant holds before the run is created; + 3. assemble — pair the resolved step maps with the timestamped run name + (defaulting to "SDK Eval"), description, and repeats. + """ + draft = await _parse_evaluate_kwargs( + testsets=testsets, + applications=applications, + evaluators=evaluators, + repeats=repeats, + specs=specs, + ) - return testset_revisions, application_revisions, evaluator_revisions + simple_evaluation_data = await _resolve_entities( + simple_evaluation_data=draft, + ) + base_name = name.strip() if isinstance(name, str) else "" + if not base_name: + base_name = "SDK Eval" -def _timestamp_suffix(): - suffix = datetime.now().strftime("%y-%m-%d · %H:%M") - return f" [{suffix}]" + return RunData( + name=f"{base_name}{_timestamp_suffix()}", + description=description, + # + testset_steps=simple_evaluation_data.testset_steps, + application_steps=simple_evaluation_data.application_steps, + evaluator_steps=simple_evaluation_data.evaluator_steps, + # + repeats=simple_evaluation_data.repeats, + ) UNICODE = { @@ -370,18 +419,19 @@ async def aevaluate( # specs: Optional[Union[EvaluateSpecs, Dict[str, Any]]] = None, ): - simple_evaluation_data = await _parse_evaluate_kwargs( + run_data = await _prepare_run_data( + name=name, + description=description, + # testsets=testsets, applications=applications, evaluators=evaluators, + # repeats=repeats, + # specs=specs, ) - simple_evaluation_data = await _upsert_entities( - simple_evaluation_data=simple_evaluation_data, - ) - print() print( "────────────────────────────────────────────────────────────────────────────" @@ -391,440 +441,80 @@ async def aevaluate( "────────────────────────────────────────────────────────────────────────────" ) - # Normalize testset_steps to revision ids (no JIT transfers in backend) - if simple_evaluation_data.testset_steps and isinstance( - simple_evaluation_data.testset_steps, dict - ): - normalized_testset_steps: Dict[str, Origin] = {} - for testset_id_str, origin in simple_evaluation_data.testset_steps.items(): - try: - testset_uuid = UUID(str(testset_id_str)) - except Exception: - continue - - testset_revision = await aretrieve_testset( - testset_revision_id=testset_uuid, - ) - - if not testset_revision or not testset_revision.id: - # Fallback: treat as testset_id (latest revision) - testset_revision = await aretrieve_testset( - testset_id=testset_uuid, - ) - - if testset_revision and testset_revision.id: - normalized_testset_steps[str(testset_revision.id)] = origin - - simple_evaluation_data.testset_steps = normalized_testset_steps - - suffix = _timestamp_suffix() - base_name = name.strip() if isinstance(name, str) else "" - if not base_name: - base_name = "SDK Eval" - name = f"{base_name}{suffix}" - run = await acreate_run( - name=name, - description=description, + name=run_data.name, + description=run_data.description, # - testset_steps=simple_evaluation_data.testset_steps, - application_steps=simple_evaluation_data.application_steps, - evaluator_steps=simple_evaluation_data.evaluator_steps, + testset_steps=run_data.testset_steps, + application_steps=run_data.application_steps, + evaluator_steps=run_data.evaluator_steps, # - repeats=simple_evaluation_data.repeats, - ) - - print( - f"{UNICODE['here']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f" run_id={str(run.id)}", + repeats=run_data.repeats, ) if not run.id: print("[failure] could not create evaluation") return None - ( - testset_revisions, - application_revisions, - evaluator_revisions, - ) = await _retrieve_entities( - simple_evaluation_data=simple_evaluation_data, + log.info( + "[EVAL] run created", + run_id=str(run.id), + **({"name": run_data.name} if run_data.name else {}), + testsets=len(run_data.testset_steps or {}), + applications=len(run_data.application_steps or {}), + evaluators=len(run_data.evaluator_steps or {}), + repeats=run_data.repeats or 1, ) - scenarios = list() - - metrics = dict() - - for testset_revision in testset_revisions.values(): - if not testset_revision.data or not testset_revision.data.testcases: - continue - - testcases = testset_revision.data.testcases - - print( - f"{UNICODE['next']}" - f"{UNICODE['here']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f" testset_id={str(testset_revision.testset_id)}", - ) - - for testcase_idx, testcase in enumerate(testcases): - print( - f"{UNICODE['pipe']}" - f"{UNICODE['pipe']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - "-----------------------" - "--------------------------------------" - ) - - print( - f"{UNICODE['pipe']}" - f"{UNICODE['next' if testcase_idx < len(testcases) - 1 else 'last']}" - f"{UNICODE['here']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f"testcase_id={str(testcase.id)}", - ) - - scenario = await aadd_scenario( - run_id=run.id, - ) - - print( - f"{UNICODE['pipe']}" - f"{UNICODE['pipe' if testcase_idx < len(testcases) - 1 else 'skip']}" - f"{UNICODE['next']}" - f"{UNICODE['here']}" - f"{UNICODE['skip']}" - f"scenario_id={str(scenario.id)}", - ) - - results = dict() - - result = await alog_result( - run_id=run.id, - scenario_id=scenario.id, - step_key="testset-" + testset_revision.slug, # type: ignore - testcase_id=testcase.id, - ) - - print( - f"{UNICODE['pipe']}" - f"{UNICODE['pipe' if testcase_idx < len(testcases) - 1 else 'skip']}" - f"{UNICODE['pipe']}" - f"{UNICODE['next']}" - f"{UNICODE['here']}" - f" result_id={str(result.id)} (testcase)", - ) - - results[testset_revision.slug] = result - - _testcase = testcase.model_dump( - mode="json", - exclude_none=True, - ) # type: ignore - inputs = testcase.data - if isinstance(inputs, dict): - if "testcase_dedup_id" in inputs: - del inputs["testcase_dedup_id"] - - for application_revision in application_revisions.values(): - if not application_revision or not application_revision.data: - print("Missing or invalid application revision") - if application_revision: - print(application_revision.model_dump(exclude_none=True)) - continue - - # print(f" Application {application_revision.model_dump(exclude_none=True)}") # type: ignore - - references = dict( - testset=Reference( - id=testset_revision.testset_id, - ), - testset_variant=Reference( - id=testset_revision.testset_variant_id, - ), - testset_revision=Reference( - id=testset_revision.id, - slug=testset_revision.slug, - version=testset_revision.version, - ), - application=Reference( - id=application_revision.application_id, - ), - application_variant=Reference( - id=application_revision.application_variant_id, - ), - application_revision=Reference( - id=application_revision.id, - slug=application_revision.slug, - version=application_revision.version, - ), - ) - links = None - - _revision = application_revision.model_dump( - mode="json", - exclude_none=True, - ) - parameters = ( - application_revision.data.parameters - if application_revision.data - else None - ) - - _trace = None - outputs = None - - workflow_service_request_data = WorkflowServiceRequestData( - revision=_revision, - parameters=parameters, - # - testcase=_testcase, - inputs=inputs, - # - trace=_trace, - outputs=outputs, - ) - - application_request = ApplicationServiceRequest( - data=workflow_service_request_data, - # - references=references, # type: ignore - links=links, # type: ignore - ) - - application_response = await invoke_application( - request=application_request, - ) - - if ( - not application_response - or not application_response.data - or not application_response.trace_id - ): - print("Missing or invalid application response") - if application_response: - print(application_response.model_dump(exclude_none=True)) - continue - - trace_id = application_response.trace_id - - if not application_revision.slug: - print("Missing application revision slug") - continue - - application_slug = application_revision.slug - - trace = fetch_trace_data(trace_id, max_retries=30, delay=1.0) - - result = await alog_result( - run_id=run.id, - scenario_id=scenario.id, - step_key="application-" + application_slug, # type: ignore - trace_id=trace_id, - ) - - print( - f"{UNICODE['pipe']}" - f"{UNICODE['pipe' if testcase_idx < len(testcases) - 1 else 'skip']}" - f"{UNICODE['pipe']}" - f"{UNICODE['next']}" - f"{UNICODE['here']}" - f" result_id={str(result.id)} (invocation)", - ) - - results[application_slug] = result - - trace = await trace - - if not trace: - print("Failed to fetch trace data for application") - continue - - root_span = list(trace.get("spans", {}).values())[0] - trace_attributes: dict = root_span.get("attributes", {}) - trace_attributes_ag: dict = trace_attributes.get("ag", {}) - trace_attributes_ag_data: dict = trace_attributes_ag.get("data", {}) - outputs = trace_attributes_ag_data.get("outputs") - inputs = inputs or trace_attributes_ag_data.get("inputs") - - for i, evaluator_revision in enumerate(evaluator_revisions.values()): - if not evaluator_revision or not evaluator_revision.data: - print("Missing or invalid evaluator revision") - if evaluator_revision: - print(evaluator_revision.model_dump(exclude_none=True)) - continue - - references = dict( - testset=Reference( - id=testset_revision.testset_id, - ), - testset_variant=Reference( - id=testset_revision.testset_variant_id, - ), - testset_revision=Reference( - id=testset_revision.id, - slug=testset_revision.slug, - version=testset_revision.version, - ), - evaluator=Reference( - id=evaluator_revision.evaluator_id, - ), - evaluator_variant=Reference( - id=evaluator_revision.evaluator_variant_id, - ), - evaluator_revision=Reference( - id=evaluator_revision.id, - slug=evaluator_revision.slug, - version=evaluator_revision.version, - ), - ) - links = ( - dict( - invocation=Link( - trace_id=application_response.trace_id, - span_id=application_response.span_id, - ) - ) - if application_response.trace_id - and application_response.span_id - else None - ) - - _revision = evaluator_revision.model_dump( - mode="json", - exclude_none=True, - ) - parameters = ( - evaluator_revision.data.parameters - if evaluator_revision.data - else None - ) - - workflow_service_request_data = WorkflowServiceRequestData( - revision=_revision, - parameters=parameters, - # - testcase=_testcase, - inputs=inputs, - # - trace=trace, - outputs=outputs, - ) - - evaluator_request = EvaluatorServiceRequest( - version="2025.07.14", - # - data=workflow_service_request_data, - # - references=references, # type: ignore - links=links, # type: ignore - ) - - evaluator_response = await invoke_evaluator( - request=evaluator_request, - ) - - if ( - not evaluator_response - or not evaluator_response.data - or not evaluator_response.trace_id - ): - print("Missing or invalid evaluator response") - if evaluator_response: - print(evaluator_response.model_dump(exclude_none=True)) - continue - - trace_id = evaluator_response.trace_id - - trace = fetch_trace_data(trace_id, max_retries=30, delay=1.0) - - result = await alog_result( - run_id=run.id, - scenario_id=scenario.id, - step_key="evaluator-" + evaluator_revision.slug, # type: ignore - trace_id=trace_id, - ) - - print( - f"{UNICODE['pipe']}" - f"{UNICODE['pipe' if testcase_idx < len(testcases) - 1 else 'skip']}" - f"{UNICODE['pipe']}" - f"{UNICODE['last' if (i == len(evaluator_revisions) - 1) else 'next']}" - f"{UNICODE['here']}" - f" result_id={str(result.id)} (annotation)", - ) - - results[evaluator_revision.slug] = result - - trace = await trace - - if not trace: - print("Failed to fetch trace data for evaluator") - continue - - metrics = await acompute_metrics( - run_id=run.id, - scenario_id=scenario.id, - ) - - print( - f"{UNICODE['pipe']}" - f"{UNICODE['pipe' if testcase_idx < len(testcases) - 1 else 'skip']}" - f"{UNICODE['last']}" - f"{UNICODE['here']}" - f"{UNICODE['skip']}" - f" metrics_id={str(metrics.id)}", - ) + runner = AsyncioEvaluationTaskRunner( + retrieve_testset=aretrieve_testset, + retrieve_application=aretrieve_application, + retrieve_evaluator=aretrieve_evaluator, + # + fetch_trace=afetch_trace, + # + add_scenarios=aadd_scenarios, + edit_scenario=aedit_scenario, + # + populate_slice=apopulate_slice, + refresh_metrics=arefresh, + # + process_sources=process_sources, + # + workflow_runner=SDKWorkflowRunner(), + ) - scenarios.append( - { - "scenario": scenario, - "results": results, - "metrics": metrics, - }, - ) + scenarios, run_status = await runner.process_run_locally( + run_id=run.id, + run_data=run_data, + ) - print( - f"{UNICODE['pipe']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - "-----------------------" - "--------------------------------------" + if not scenarios: + log.warning( + "[EVAL] evaluation produced no scenarios; check testset", + run_id=str(run.id), ) - metrics = dict() - - if len(scenarios) > 0: - metrics = await acompute_metrics( - run_id=run.id, - ) + run = await aclose_run( + run_id=run.id, + status=run_status.value, + ) - print( - f"{UNICODE['last']}" - f"{UNICODE['here']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f"{UNICODE['skip']}" - f" metrics_id={str(metrics.id)}", - ) + log.info( + "[EVAL] run closed", + run_id=str(run.id), + status=run_status.value, + scenarios=len(scenarios), + ) - run = await aclose_run( + # Global metrics only + metrics = await aquery_metrics( run_id=run.id, ) - run_url = await aget_url(run_id=run.id) + run_url = await aget_url( + run_id=run.id, + ) print( "────────────────────────────────────────────────────────────────────────────" diff --git a/sdks/python/agenta/sdk/evaluations/preview/utils.py b/sdks/python/agenta/sdk/evaluations/preview/utils.py index 93fc199c66..e33ae0e0a6 100644 --- a/sdks/python/agenta/sdk/evaluations/preview/utils.py +++ b/sdks/python/agenta/sdk/evaluations/preview/utils.py @@ -562,7 +562,7 @@ async def display_evaluation_results( # Fetch and process trace data using services module try: - trace_data = await fetch_trace_data(result.trace_id) + trace_data = await afetch_trace(result.trace_id) if trace_data and "spans" in trace_data: for span_key in trace_data["spans"].keys(): step_data = extract_trace_step_data(trace_data, span_key) @@ -690,8 +690,8 @@ async def display_evaluation_results( from typing import Dict, Any, Optional # noqa: E402 -async def fetch_trace_data( - trace_id: str, max_retries: int = 3, delay: float = 1.0 +async def afetch_trace( + trace_id: str, max_retries: int = 30, delay: float = 1.0 ) -> Optional[Dict[str, Any]]: """ Fetch trace data from the API with retry logic. @@ -735,10 +735,10 @@ async def fetch_trace_data( await asyncio.sleep(delay) continue else: - print(f"Error fetching trace data: {e}") + print(f"Error fetching trace: {e}") return None - print("Failed to fetch trace data after retries") + print("Failed to fetch trace after retries") return None diff --git a/sdks/python/agenta/sdk/evaluations/results.py b/sdks/python/agenta/sdk/evaluations/results.py index ca0aebb170..8d186367e9 100644 --- a/sdks/python/agenta/sdk/evaluations/results.py +++ b/sdks/python/agenta/sdk/evaluations/results.py @@ -1,4 +1,4 @@ -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, List from uuid import UUID from agenta.sdk.utils.client import authed_api @@ -7,12 +7,52 @@ # TODO: ADD TYPES +async def apopulate( + *, + results: List[Dict[str, Any]], +) -> List[EvaluationResult]: + """Bulk-write finished result cells in one call (the `populate_slice` op). + + Mirrors `POST /simple/evaluations/{id}/populate`: each item is a fully-formed + result cell (run_id, scenario_id, step_key, repeat_idx, status, and the + trace_id/testcase_id/error it carries). Callers that computed cells + themselves (the SDK local evaluator) write them all at once here instead of + one `acreate` per cell. `run_id` is taken from the cells, not the path + builder, so every cell must carry its own. + """ + if not results: + return [] + + run_ids = {r.get("run_id") for r in results} + if len(run_ids) != 1 or None in run_ids: + raise ValueError( + "apopulate requires all result cells to carry the same run_id." + ) + run_id = run_ids.pop() + + response = authed_api()( + method="POST", + endpoint=f"/simple/evaluations/{run_id}/populate", + json=dict(results=results), + ) + + try: + response.raise_for_status() + except: + print(response.text) + raise + + response = response.json() + + return [EvaluationResult(**r) for r in response.get("results", [])] + + async def acreate( *, run_id: UUID, scenario_id: UUID, step_key: str, - # repeat_idx: str, + repeat_idx: Optional[int] = 0, # timestamp: datetime, # interval: float, # @@ -37,7 +77,7 @@ async def acreate( # # interval=interval, # timestamp=timestamp, - # repeat_idx=repeat_idx, + repeat_idx=repeat_idx, step_key=step_key, run_id=str(run_id), scenario_id=str(scenario_id), diff --git a/sdks/python/agenta/sdk/evaluations/runs.py b/sdks/python/agenta/sdk/evaluations/runs.py index af4fbf762e..f907919a93 100644 --- a/sdks/python/agenta/sdk/evaluations/runs.py +++ b/sdks/python/agenta/sdk/evaluations/runs.py @@ -1,11 +1,43 @@ from typing import Optional, Dict, Any + +from pydantic import BaseModel from uuid import UUID from agenta.sdk.utils.client import authed_api -from agenta.sdk.models.evaluations import EvaluationRun, Target +from agenta.sdk.models.evaluations import EvaluationRun, Origin, Target import agenta as ag + +class RunData(BaseModel): + """Typed input contract for `acreate` — the RESOLVED run-creation payload. + + Callers assemble a `RunData` and pass its fields to `acreate` explicitly + (`acreate(name=run_data.name, ...)`), keeping the create call's surface + visible at the call site while resolving the data in one place. + + `*_steps` are typed `Dict[str, Origin]` (revision-id-keyed), NOT the loose + `Target` `acreate` accepts: by the time a `RunData` exists, every step is + resolved to a `{revision_id: origin}` map. Using `Target` here would let + pydantic coerce the string keys back to UUID (the `Dict[UUID, Origin]` union + member), which then fails to JSON-serialize. + """ + + name: Optional[str] = None + description: Optional[str] = None + # + flags: Optional[Dict[str, Any]] = None + tags: Optional[Dict[str, Any]] = None + meta: Optional[Dict[str, Any]] = None + # + query_steps: Optional[Dict[str, Origin]] = None + testset_steps: Optional[Dict[str, Origin]] = None + application_steps: Optional[Dict[str, Origin]] = None + evaluator_steps: Optional[Dict[str, Origin]] = None + # + repeats: Optional[int] = None + + # TODO: ADD TYPES @@ -100,7 +132,8 @@ async def aclose( ) -> Optional[EvaluationRun]: response = authed_api()( method="POST", - endpoint=f"/evaluations/runs/{run_id}/close/{status}", + endpoint=f"/evaluations/runs/{run_id}/close", + params={"status": status} if status else None, ) try: diff --git a/sdks/python/agenta/sdk/evaluations/runtime/__init__.py b/sdks/python/agenta/sdk/evaluations/runtime/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdks/python/agenta/sdk/evaluations/runtime/adapters.py b/sdks/python/agenta/sdk/evaluations/runtime/adapters.py new file mode 100644 index 0000000000..060654c00e --- /dev/null +++ b/sdks/python/agenta/sdk/evaluations/runtime/adapters.py @@ -0,0 +1,190 @@ +from asyncio import Semaphore, gather +from typing import Any, Dict, Optional + +from agenta.sdk.decorators.running import invoke_application, invoke_evaluator +from agenta.sdk.evaluations.runtime.models import ( + ResultLogRequest, + WorkflowExecutionRequest, + WorkflowExecutionResult, +) +from agenta.sdk.models.evaluations import EvaluationStatus +from agenta.sdk.models.workflows import ( + ApplicationServiceRequest, + EvaluatorServiceRequest, + WorkflowServiceRequestData, +) + + +class SDKWorkflowRunner: + """SDK adapter for executing workflow steps through local decorators. + + One runner for both runnable step kinds — invocation (application) and + annotation (evaluator) — mirroring the API's single `APIWorkflowRunner`. It + branches on `request.step.type`: an application step invokes the app + decorator, an evaluator step the evaluator decorator. The two paths differ + only in which decorator they call and the request envelope; the request + DATA is built identically. + """ + + async def execute( + self, + request: WorkflowExecutionRequest, + ) -> WorkflowExecutionResult: + data = WorkflowServiceRequestData( + revision=request.revision, + parameters=request.parameters, + testcase=request.source.testcase, + inputs=request.source.inputs, + trace=request.upstream_trace, + outputs=request.upstream_outputs, + ) + + if request.step.type == "invocation": + response = await invoke_application( + request=ApplicationServiceRequest( + data=data, + references=request.references, # type: ignore[arg-type] + links=request.links, # type: ignore[arg-type] + ) + ) + else: + response = await invoke_evaluator( + request=EvaluatorServiceRequest( + version="2025.07.14", + data=data, + references=request.references, # type: ignore[arg-type] + links=request.links, # type: ignore[arg-type] + ) + ) + + return _normalize_service_response(response) + + async def execute_batch( + self, + requests: list[WorkflowExecutionRequest], + semaphore: Optional[Semaphore] = None, + ) -> list[WorkflowExecutionResult]: + # Concurrent, semaphore-bounded — same shape as APIWorkflowRunner. The + # engine passes a shared semaphore (from batch_size); honoring it here is + # what makes a scenario's repeats/steps run concurrently instead of + # serially. + async def _guarded( + request: WorkflowExecutionRequest, + ) -> WorkflowExecutionResult: + if semaphore is not None: + async with semaphore: + return await self.execute(request) + return await self.execute(request) + + return list(await gather(*(_guarded(request) for request in requests))) + + +class SDKResultSetter: + """Result setter that WRITES each cell live, like the API's APIResultSetter. + + Aligns the SDK with the API persistence model: each finished cell is + populated to the backend as the engine produces it (one `populate` call per + cell), instead of collected and bulk-written after the slice. Writing live is + what lets the engine's inline per-scenario metric refresh see persisted + cells, so the SDK gets the SAME variational-inline + global-at-end refresh + shape as the API. The returned dict is what the engine remembers as the + cell's value. + """ + + def __init__(self, *, populate: Any) -> None: + self._populate = populate + + async def set(self, request: ResultLogRequest) -> Dict[str, Any]: + cell = request.cell + payload = dict( + run_id=str(cell.run_id), + scenario_id=str(cell.scenario_id), + step_key=cell.step_key, + repeat_idx=cell.repeat_idx, + status=getattr(cell.status, "value", cell.status), + trace_id=request.trace_id + if request.trace_id is not None + else cell.trace_id, + testcase_id=str( + request.testcase_id + if request.testcase_id is not None + else cell.testcase_id + ) + if (request.testcase_id is not None or cell.testcase_id is not None) + else None, + error=request.error if request.error is not None else cell.error, + ) + await self._populate(results=[payload]) + return payload + + +class SDKScenarioEditor: + """Engine `edit_scenario` adapter — SDK peer of `APIScenarioEditor`. + + Bridges the engine's `(scenario, status)` contract to the SDK client's + `aedit_scenario(scenario_id=, status=, tags=, meta=)`, carrying the + scenario's tags/meta through like the API adapter. The injected `edit` + client tolerates a run closed mid-flight (HTTP 409 -> None). + """ + + def __init__(self, *, edit: Any) -> None: + self._edit = edit + + async def __call__(self, scenario: Any, status: Any) -> Any: + return await self._edit( + scenario_id=scenario.id, + status=getattr(status, "value", status), + tags=getattr(scenario, "tags", None), + meta=getattr(scenario, "meta", None), + ) + + +class SDKMetricsRefresher: + """Engine `refresh_metrics` adapter — SDK peer of `APIMetricsRefresher`. + + The engine calls this `(run_id, scenario_id)` per scenario (variational) and + `(run_id, None)` once at the end (global). Wraps the injected `arefresh` + client; the SDK has no temporal axis, so unlike the API adapter it does not + handle timestamp/interval buckets. + """ + + def __init__(self, *, refresh: Any) -> None: + self._refresh = refresh + + async def __call__(self, run_id: Any, scenario_id: Any = None) -> Any: + return await self._refresh(run_id, scenario_id) + + +class SDKTraceFetcher: + """Engine `fetch_trace` adapter — SDK peer of `APITraceFetcher`. + + A callable `(trace_id) -> trace | None`, wrapping the injected `afetch_trace` + client so the engine loads a runner's trace after a step executes. + """ + + def __init__(self, *, fetch: Any) -> None: + self._fetch = fetch + + async def __call__(self, trace_id: str) -> Any: + return await self._fetch(trace_id) + + +def _normalize_service_response(response: Any) -> WorkflowExecutionResult: + # A response with no trace_id is treated as a FAILURE on purpose: the SDK + # evaluation pipeline keys cache reuse, upstream-context links, and metric + # provenance off the produced trace, so a step that returns outputs but no + # trace cannot be wired into the rest of the run and is not a usable success. + # (If a trace-free "outputs only" success mode is ever needed, branch here on + # `response.data.outputs` before falling through to FAILURE.) + if not response or not getattr(response, "data", None) or not response.trace_id: + return WorkflowExecutionResult( + status=EvaluationStatus.FAILURE, + error={"message": "Missing or invalid workflow response"}, + ) + + return WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id=response.trace_id, + span_id=getattr(response, "span_id", None), + outputs=getattr(response.data, "outputs", None), + ) diff --git a/sdks/python/agenta/sdk/evaluations/runtime/executor.py b/sdks/python/agenta/sdk/evaluations/runtime/executor.py new file mode 100644 index 0000000000..87b604873b --- /dev/null +++ b/sdks/python/agenta/sdk/evaluations/runtime/executor.py @@ -0,0 +1,527 @@ +from asyncio import Lock, Semaphore, gather +from datetime import datetime +from typing import Any, Awaitable, Callable, Dict, List, Optional, Protocol, Tuple +from uuid import UUID + +from agenta.sdk.evaluations.runtime.adapters import ( + SDKResultSetter, + SDKScenarioEditor, + SDKMetricsRefresher, + SDKTraceFetcher, +) +from agenta.sdk.evaluations.runtime.models import ( + EvaluationStep, + PlannedCell, + ResolvedSourceItem, + ResultLogRequest, + WorkflowExecutionRequest, + WorkflowExecutionResult, +) +from agenta.sdk.models.shared import Reference +from agenta.sdk.utils.logging import get_module_logger + +_log = get_module_logger(__name__) + + +class WorkflowRunner(Protocol): + """Adapter boundary for application/evaluator execution. + + SDK-local evaluation, API service execution, and backend-internal workflow + invocation should each implement this protocol instead of changing the + planner or topology classifier. + """ + + async def execute( + self, + request: WorkflowExecutionRequest, + ) -> WorkflowExecutionResult: ... + + +class WorkflowBatchRunner(WorkflowRunner, Protocol): + """Optional batch execution boundary for any runnable workflow step.""" + + async def execute_batch( + self, + requests: List[WorkflowExecutionRequest], + semaphore: Optional[Semaphore] = None, + ) -> List[WorkflowExecutionResult]: ... + + +async def execute_workflow_batch( + *, + runner: WorkflowRunner, + requests: List[WorkflowExecutionRequest], + semaphore: Optional[Semaphore] = None, +) -> List[WorkflowExecutionResult]: + execute_batch = getattr(runner, "execute_batch", None) + + async def _guarded(request: WorkflowExecutionRequest) -> WorkflowExecutionResult: + if semaphore is not None: + async with semaphore: + return await runner.execute(request) + return await runner.execute(request) + + if execute_batch is not None: + return await execute_batch(requests, semaphore=semaphore) + + return list(await gather(*(_guarded(request) for request in requests))) + + +class EvaluationTaskRunner(Protocol): + """Generic evaluation task dispatch boundary. + + SDK/local code should use an in-process asyncio implementation. API code can + adapt this protocol to Taskiq without Taskiq leaking into SDK runtime code. + """ + + async def process_run_from_source( + self, + *, + project_id: UUID, + user_id: UUID, + run_id: UUID, + newest: Optional[datetime] = None, + oldest: Optional[datetime] = None, + ) -> Any: ... + + async def process_run_from_batch( + self, + *, + project_id: UUID, + user_id: UUID, + run_id: UUID, + source_kind: str, + trace_ids: Optional[List[str]] = None, + testcase_ids: Optional[List[UUID]] = None, + input_step_key: Optional[str] = None, + ) -> Any: ... + + +class AsyncioEvaluationTaskRunner: + """In-process evaluation executor for SDK/local runs. + + The SDK counterpart of the API's Taskiq worker. It mirrors the API's slice-op + sequence — add_scenarios -> populate -> refresh — but adapted to the SDK's + irreducible differences: it works by VALUE (entities already resolved in + `aevaluate`), executes workflows LOCALLY (the user's decorated functions, via + `SDKWorkflowRunner`), and runs inline with no broker. + + Per testset revision the sequence is: + + 1. add_scenarios — bulk-mint N skeleton scenarios (the `add_scenarios` op), + 2. process — ONE slice over all scenarios via the SDK engine with local + runners. Each cell is written live (SDKResultSetter -> populate), the + engine refreshes metrics inline per scenario (variational) and once at + the end (global), and writes each scenario's status — the SAME shape as + the API worker, only the runner is local. + + The source processor (`process_sources`) is the same engine the API funnels + through; only the injected adapters (local runner) differ. + """ + + def __init__( + self, + *, + process_sources: Callable[..., Awaitable[List[Any]]], + add_scenarios: Callable[..., Awaitable[List[Any]]], + populate_slice: Callable[..., Awaitable[Any]], + refresh_metrics: Callable[..., Awaitable[Any]], + edit_scenario: Callable[..., Awaitable[Any]], + retrieve_testset: Callable[..., Awaitable[Any]], + retrieve_application: Callable[..., Awaitable[Any]], + retrieve_evaluator: Callable[..., Awaitable[Any]], + workflow_runner: Any, + fetch_trace: Any, + ): + self._process_sources = process_sources + self._add_scenarios = add_scenarios + self._populate_slice = populate_slice + self._refresh_metrics = refresh_metrics + self._edit_scenario = edit_scenario + self._retrieve_testset = retrieve_testset + self._retrieve_application = retrieve_application + self._retrieve_evaluator = retrieve_evaluator + # Stateless dispatcher (branches on step type); one shared instance is + # wired into every runnable step — no per-step minting needed. + self._workflow_runner = workflow_runner + self._fetch_trace = fetch_trace + + def _build_steps_and_runners( + self, + *, + testset: Tuple[Any, Any], + input_step_key: str, + application_revisions: List[Tuple[Any, Any]], + evaluator_revisions: List[Tuple[Any, Any]], + ) -> tuple: + """Build the step graph + wire local runners for one testset revision. + + All three entity kinds arrive as the same `(revision, origin)` shape, + consumed symmetrically (testset is a single pair; applications and + evaluators are lists of pairs). The input step carries the testset's + origin, the invocation step the application's, the annotation step the + evaluator's; only evaluators vary (auto / custom / human), and only + non-human runnable steps get a local runner. The one shared + `SDKWorkflowRunner` is wired into every runnable step; it branches on + step type internally. + """ + testset_revision, testset_origin = testset + steps = [ + EvaluationStep( + key=input_step_key, + type="input", + origin=testset_origin, + references={ + "testset": Reference(id=testset_revision.testset_id), + "testset_variant": Reference( + id=testset_revision.testset_variant_id + ), + "testset_revision": Reference( + id=testset_revision.id, + slug=testset_revision.slug, + version=testset_revision.version, + ), + }, + ) + ] + runners: Dict[str, Any] = {} + revisions: Dict[str, Any] = {} + + for application_revision, origin in application_revisions: + if not application_revision or not application_revision.data: + continue + application_step_key = "application-" + application_revision.slug + steps.append( + EvaluationStep( + key=application_step_key, + type="invocation", + origin=origin, + references={ + "application": Reference( + id=application_revision.application_id + ), + "application_variant": Reference( + id=application_revision.application_variant_id, + ), + "application_revision": Reference( + id=application_revision.id, + slug=application_revision.slug, + version=application_revision.version, + ), + }, + ) + ) + if origin != "human": + runners[application_step_key] = self._workflow_runner + revisions[application_step_key] = application_revision + + for evaluator_revision, origin in evaluator_revisions: + if not evaluator_revision or not evaluator_revision.data: + continue + evaluator_step_key = "evaluator-" + evaluator_revision.slug + steps.append( + EvaluationStep( + key=evaluator_step_key, + type="annotation", + origin=origin, + references={ + "evaluator": Reference(id=evaluator_revision.evaluator_id), + "evaluator_variant": Reference( + id=evaluator_revision.evaluator_variant_id, + ), + "evaluator_revision": Reference( + id=evaluator_revision.id, + slug=evaluator_revision.slug, + version=evaluator_revision.version, + ), + }, + ) + ) + # auto + custom evaluators run locally; only human is left to the web. + if origin != "human": + runners[evaluator_step_key] = self._workflow_runner + revisions[evaluator_step_key] = evaluator_revision + + return steps, runners, revisions + + def _build_source_items( + self, + *, + testset_revision: Any, + input_step_key: str, + ) -> List[Any]: + """One hydrated source item per testcase in the revision.""" + source_items: List[Any] = [] + for testcase in testset_revision.data.testcases: + inputs = dict(testcase.data or {}) + inputs.pop("testcase_dedup_id", None) + source_items.append( + ResolvedSourceItem( + kind="testcase", + step_key=input_step_key, + references={ + "testcase": Reference(id=testcase.id), + "testset": Reference(id=testset_revision.testset_id), + "testset_variant": Reference( + id=testset_revision.testset_variant_id, + ), + "testset_revision": Reference( + id=testset_revision.id, + slug=testset_revision.slug, + version=testset_revision.version, + ), + }, + testcase_id=testcase.id, + testcase=testcase.model_dump(mode="json", exclude_none=True), + inputs=inputs, + ) + ) + return source_items + + async def _retrieve_revisions( + self, + run_data: Any, + ) -> Tuple[ + List[Tuple[Any, Any]], + List[Tuple[Any, Any]], + List[Tuple[Any, Any]], + ]: + """Fetch the revision objects for the run's step ids. + + `run_data` carries revision IDS; local execution needs the revision + OBJECTS (slug/data/refs), so fetch them via the injected per-entity + retrievers — the SDK analogue of the API executor's + `_resolve_runners_and_revisions`. Returns `(revision, origin)` pairs per + kind. Testsets get the revision-id-then-testset-id fallback. + """ + testset_revisions: List[Tuple[Any, Any]] = [] + for testset_ref, origin in (run_data.testset_steps or {}).items(): + testset_revision = await self._retrieve_testset( + testset_revision_id=testset_ref, + ) + if not testset_revision or not testset_revision.id: + # Fallback: treat the id as a testset id (latest revision). + testset_revision = await self._retrieve_testset( + testset_id=testset_ref, + ) + if not testset_revision or not testset_revision.id: + _log.warning( + "[EVAL] testset reference could not be retrieved; skipping", + testset_ref=str(testset_ref), + ) + continue + testset_revisions.append((testset_revision, origin)) + + application_revisions: List[Tuple[Any, Any]] = [] + for application_revision_id, origin in ( + run_data.application_steps or {} + ).items(): + application_revision = await self._retrieve_application( + application_revision_id=application_revision_id, + ) + if not application_revision: + continue + application_revisions.append((application_revision, origin)) + + evaluator_revisions: List[Tuple[Any, Any]] = [] + for evaluator_revision_id, origin in (run_data.evaluator_steps or {}).items(): + evaluator_revision = await self._retrieve_evaluator( + evaluator_revision_id=evaluator_revision_id, + ) + if not evaluator_revision: + continue + evaluator_revisions.append((evaluator_revision, origin)) + + return testset_revisions, application_revisions, evaluator_revisions + + async def process_run_locally( + self, + *, + run_id: UUID, + run_data: Any, + ) -> Tuple[List[Dict[str, Any]], Any]: + """Run the evaluation locally via the API-mirroring slice sequence. + + Takes `run_data` (revision IDS) and fetches the revision OBJECTS itself + (`_retrieve_revisions`, via injected retrievers) — mirroring the API + executor, which fetches its revisions inside `process`, not before. Per + testset: add_scenarios (bulk) -> ONE process_sources slice over all + scenarios (live cell writes, inline + global metric refresh, status + writes). Empty/unresolved testsets are skipped, not failed. + """ + # Local import: processor.py imports from this module, so importing it at + # module load would be circular. The verdict→status ranking lives there + # so every driver shares one definition. + from agenta.sdk.evaluations.runtime.processor import ( + run_status, + scenario_status, + ) + + ( + testset_revisions, + application_revisions, + evaluator_revisions, + ) = await self._retrieve_revisions(run_data) + repeats = run_data.repeats + + scenarios: List[Dict[str, Any]] = [] + # Accumulate the engine's per-scenario verdicts across every slice so the + # run status is rolled up once via the shared `run_status` — not + # re-derived by the caller at close time. + all_processed: List[Any] = [] + + for testset in testset_revisions: + testset_revision, _origin = testset + if not testset_revision.data or not testset_revision.data.testcases: + # An empty testset produces no scenarios. Warn per-testset so a + # partially-empty run (some testsets have data, others don't) is + # still surfaced — the caller's aggregate "no scenarios" warning + # only fires when EVERY testset was empty. + _log.warning( + "[EVAL] testset has no testcases; skipping", + testset_revision_id=str(testset_revision.id), + ) + continue + + _log.info( + "[EVAL] processing testset", + testset_id=str(testset_revision.testset_id), + ) + + input_step_key = "testset-" + testset_revision.slug + steps, runners, revisions = self._build_steps_and_runners( + testset=testset, + input_step_key=input_step_key, + application_revisions=application_revisions, + evaluator_revisions=evaluator_revisions, + ) + source_items = self._build_source_items( + testset_revision=testset_revision, + input_step_key=input_step_key, + ) + + # add_scenarios — bulk-mint one skeleton per source item, in order. + minted = await self._add_scenarios( + run_id=run_id, + count=len(source_items), + ) + if len(minted) != len(source_items): + _log.warning( + "[EVAL] add_scenarios returned an unexpected count; skipping", + wanted=len(source_items), + got=len(minted), + testset_revision_id=str(testset_revision.id), + ) + continue + + # ONE slice over ALL scenarios — the design's `process_slice(all + # scenarios, all steps)`. The engine's internal gather + semaphore run + # the scenarios concurrently (bounded by batch_size), which is what + # makes concurrency real; an outer per-scenario loop would feed the + # engine one item at a time and leave the semaphore inert. + # Live persistence, aligned with the API: each cell is populated as + # the engine produces it (SDKResultSetter), so the engine's inline + # per-scenario metric refresh (arefresh) sees persisted cells, and + # the engine's end-of-slice global refresh rolls up the run — the + # same variational-inline + global-at-end shape as the API worker. + processed = await self._process_sources( + run_id=run_id, + source_items=source_items, + steps=steps, + repeats=repeats, + create_scenario=_PreMintedScenarios(minted), + set_results=SDKResultSetter(populate=self._populate_slice), + refresh_metrics=SDKMetricsRefresher(refresh=self._refresh_metrics), + edit_scenario=SDKScenarioEditor(edit=self._edit_scenario), + runners=runners, + revisions=revisions, + fetch_trace=SDKTraceFetcher(fetch=self._fetch_trace), + # The SDK evaluate() loop IS the executor for custom-origin steps. + execute_custom=True, + ) + + # Cells are live-written and status is written in-loop by the engine's + # edit_scenario adapter (same as the API), so here we only assemble + # the return payload. + for item in processed: + scenarios.append( + { + "scenario": item.scenario, + "results": item.results, + "metrics": item.metrics, + "status": scenario_status( + has_errors=item.has_errors, + has_pending=item.has_pending, + ), + } + ) + all_processed.extend(processed) + + # Run status rolled up once from every touched scenario (shared with the + # API). The caller applies it (closes the run with it); it is NOT + # re-derived there. + return scenarios, run_status(all_processed) + + +class _PreMintedScenarios: + """`create_scenario` adapter handing back bulk-minted scenarios in order. + + The engine calls `create_scenario(run_id)` once per source item; the SDK + minted them all up front via `add_scenarios`, so this cursor returns the next + pre-minted scenario instead of creating a new one — the SDK analogue of the + API's `_ExistingScenario` over a seed-bindings set. + + Order/concurrency: the engine now runs scenarios concurrently (gather + + semaphore), so multiple coroutines call this. `create_scenario` is the FIRST + statement of the engine's `_process_one` and this body has no `await`, so + each task runs through the pop synchronously before reaching any real + suspension point — i.e. the pops happen in `source_items` order, pairing + scenario i with source_item i. The lock makes the index increment atomic so + that ordering can never degrade into a double-hand-out if scheduling shifts. + """ + + def __init__(self, scenarios: List[Any]): + self._scenarios = list(scenarios) + self._idx = 0 + self._lock = Lock() + + async def __call__(self, run_id: UUID) -> Any: + async with self._lock: + scenario = self._scenarios[self._idx] + self._idx += 1 + return scenario + + +class ResultSetter(Protocol): + """Adapter boundary for persisting planned result cells.""" + + async def set(self, request: ResultLogRequest) -> Any: ... + + +# Adapter boundary for loading a runner's trace after a step executes: a plain +# async callable `(trace_id) -> trace | None`. The SDK passes `afetch_trace` +# directly; the API passes its (callable) APITraceLoader instance. +TraceLoader = Callable[[str], Awaitable[Optional[Any]]] + + +class RuntimeExecutionContext: + """Small mutable context shared by runner adapters while processing a scenario.""" + + def __init__(self) -> None: + self.results: Dict[str, Any] = {} + self.traces: Dict[str, Any] = {} + self.outputs: Dict[str, Any] = {} + + def remember_result(self, *, cell: PlannedCell, result: Any) -> None: + self.results[cell.step_key] = result + + def remember_execution( + self, + *, + cell: PlannedCell, + execution: WorkflowExecutionResult, + ) -> None: + if execution.trace is not None: + self.traces[cell.step_key] = execution.trace + if execution.outputs is not None: + self.outputs[cell.step_key] = execution.outputs diff --git a/sdks/python/agenta/sdk/evaluations/runtime/models.py b/sdks/python/agenta/sdk/evaluations/runtime/models.py new file mode 100644 index 0000000000..99a526dfaf --- /dev/null +++ b/sdks/python/agenta/sdk/evaluations/runtime/models.py @@ -0,0 +1,129 @@ +from typing import Any, Dict, List, Literal, Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from agenta.sdk.models.evaluations import EvaluationStatus, Origin + +StepType = Literal["input", "invocation", "annotation"] +SourceKind = Literal["query", "testset", "trace", "testcase", "direct"] +TopologyStatus = Literal["supported", "potential", "not_planned", "unsupported"] +DispatchKind = Literal[ + "batch_query", + "batch_testset", + "batch_invocation", + "queue_traces", + "queue_testcases", + "live_query", +] + + +class EvaluationStep(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + key: str + type: StepType + origin: Origin = "custom" + references: Dict[str, Any] = Field(default_factory=dict) + inputs: List[str] = Field(default_factory=list) + + +class ResolvedSourceItem(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + kind: SourceKind + step_key: str + references: Dict[str, Any] = Field(default_factory=dict) + trace_id: Optional[str] = None + span_id: Optional[str] = None + testcase_id: Optional[UUID] = None + testcase: Optional[Any] = None + trace: Optional[Any] = None + inputs: Optional[Any] = None + outputs: Optional[Any] = None + + +class ScenarioBinding(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + scenario_id: UUID + source: ResolvedSourceItem + interval: Optional[int] = None + timestamp: Optional[Any] = None + + +class PlannedCell(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + run_id: UUID + scenario_id: UUID + step_key: str + step_type: StepType + origin: Origin + repeat_idx: int + status: EvaluationStatus + should_execute: bool = False + trace_id: Optional[str] = None + span_id: Optional[str] = None + testcase_id: Optional[UUID] = None + error: Optional[Dict[str, Any]] = None + + +class ExecutionPlan(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + run_id: UUID + cells: List[PlannedCell] + + @property + def executable_cells(self) -> List[PlannedCell]: + return [cell for cell in self.cells if cell.should_execute] + + +class TopologyDecision(BaseModel): + status: TopologyStatus + label: str + reason: str + dispatch: Optional[DispatchKind] = None + + +class WorkflowExecutionRequest(BaseModel): + """Runner-agnostic request for an application or evaluator step.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + step: EvaluationStep + cell: PlannedCell + source: ResolvedSourceItem + revision: Any + parameters: Optional[Any] = None + references: Dict[str, Any] = Field(default_factory=dict) + links: Optional[Dict[str, Any]] = None + upstream_trace: Optional[Any] = None + upstream_outputs: Optional[Any] = None + + +class WorkflowExecutionResult(BaseModel): + """Normalized result produced by any workflow runner adapter.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + status: EvaluationStatus + trace_id: Optional[str] = None + span_id: Optional[str] = None + hash_id: Optional[str] = None + outputs: Optional[Any] = None + trace: Optional[Any] = None + error: Optional[Dict[str, Any]] = None + + +class ResultLogRequest(BaseModel): + """Runner-agnostic request for persisting a planned result cell.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + cell: PlannedCell + trace_id: Optional[str] = None + span_id: Optional[str] = None + testcase_id: Optional[UUID] = None + error: Optional[Dict[str, Any]] = None diff --git a/sdks/python/agenta/sdk/evaluations/runtime/planner.py b/sdks/python/agenta/sdk/evaluations/runtime/planner.py new file mode 100644 index 0000000000..fda93ab5cb --- /dev/null +++ b/sdks/python/agenta/sdk/evaluations/runtime/planner.py @@ -0,0 +1,213 @@ +from typing import List, Optional +from uuid import UUID + +from agenta.sdk.evaluations.runtime.models import ( + EvaluationStep, + ExecutionPlan, + PlannedCell, + ResolvedSourceItem, + ScenarioBinding, +) +from agenta.sdk.models.evaluations import EvaluationStatus + + +def build_repeat_indices(repeats: Optional[int]) -> List[int]: + count = repeats or 1 + if count < 1: + count = 1 + return list(range(count)) + + +def effective_is_split( + *, + is_split: bool, + is_live: bool = False, + has_traces: bool = False, + has_testcases: bool = False, + has_application_steps: bool = False, + has_evaluator_steps: bool = False, +) -> bool: + if is_live or has_traces or has_testcases: + return False + if not has_application_steps or not has_evaluator_steps: + return False + return is_split + + +class EvaluationPlanner: + """Build the evaluation result tensor without knowing how steps execute.""" + + def plan( + self, + *, + run_id: UUID, + scenario_id: UUID, + source: ResolvedSourceItem, + steps: List[EvaluationStep], + repeats: Optional[int] = None, + is_split: bool = False, + is_live: bool = False, + has_traces: bool = False, + has_testcases: bool = False, + execute_custom: bool = False, + ) -> ExecutionPlan: + return self.plan_bindings( + run_id=run_id, + bindings=[ + ScenarioBinding( + scenario_id=scenario_id, + source=source, + ) + ], + steps=steps, + repeats=repeats, + is_split=is_split, + is_live=is_live, + has_traces=has_traces, + has_testcases=has_testcases, + execute_custom=execute_custom, + ) + + def plan_bindings( + self, + *, + run_id: UUID, + bindings: List[ScenarioBinding], + steps: List[EvaluationStep], + repeats: Optional[int] = None, + is_split: bool = False, + is_live: bool = False, + has_traces: bool = False, + has_testcases: bool = False, + execute_custom: bool = False, + ) -> ExecutionPlan: + repeat_indices = build_repeat_indices(repeats) + + input_steps = [step for step in steps if step.type == "input"] + application_steps = [step for step in steps if step.type == "invocation"] + evaluator_steps = [step for step in steps if step.type == "annotation"] + app_repeat_indices = self._application_repeat_indices( + repeat_indices=repeat_indices, + is_split=is_split, + is_live=is_live, + has_traces=has_traces, + has_testcases=has_testcases, + has_application_steps=bool(application_steps), + has_evaluator_steps=bool(evaluator_steps), + ) + + cells: List[PlannedCell] = [] + + for binding in bindings: + source = binding.source + + for step in input_steps: + cells.extend( + PlannedCell( + run_id=run_id, + scenario_id=binding.scenario_id, + step_key=step.key, + step_type=step.type, + origin=step.origin, + repeat_idx=repeat_idx, + status=EvaluationStatus.SUCCESS, + trace_id=source.trace_id, + span_id=source.span_id, + testcase_id=source.testcase_id, + ) + for repeat_idx in repeat_indices + ) + + for step in application_steps: + cells.extend( + self._runnable_cells( + run_id=run_id, + scenario_id=binding.scenario_id, + source=source, + step=step, + repeat_indices=app_repeat_indices, + ) + ) + + for step in evaluator_steps: + cells.extend( + self._runnable_cells( + run_id=run_id, + scenario_id=binding.scenario_id, + source=source, + step=step, + repeat_indices=repeat_indices, + execute_custom=execute_custom, + ) + ) + + return ExecutionPlan(run_id=run_id, cells=cells) + + def _application_repeat_indices( + self, + *, + repeat_indices: List[int], + is_split: bool, + is_live: bool, + has_traces: bool, + has_testcases: bool, + has_application_steps: bool, + has_evaluator_steps: bool, + ) -> List[int]: + split = effective_is_split( + is_split=is_split, + is_live=is_live, + has_traces=has_traces, + has_testcases=has_testcases, + has_application_steps=has_application_steps, + has_evaluator_steps=has_evaluator_steps, + ) + + if not has_application_steps: + return [] + if not has_evaluator_steps: + return repeat_indices + if split: + return repeat_indices + return [0] + + def _runnable_cells( + self, + *, + run_id: UUID, + scenario_id: UUID, + source: ResolvedSourceItem, + step: EvaluationStep, + repeat_indices: List[int], + execute_custom: bool = False, + ) -> List[PlannedCell]: + # Who executes an annotation step depends on its origin: + # human -> the web frontend (never executed by the runtime) + # auto -> the runtime host (backend worker, or SDK locally) + # custom -> the SDK/external client. The runtime only runs custom when + # it IS that client (execute_custom=True, set by the SDK + # evaluate() loop); the backend leaves custom alone. + manual_origins = {"human"} if execute_custom else {"human", "custom"} + is_manual_annotation = step.type == "annotation" and step.origin in ( + manual_origins + ) + status = ( + EvaluationStatus.PENDING + if is_manual_annotation + else EvaluationStatus.QUEUED + ) + + return [ + PlannedCell( + run_id=run_id, + scenario_id=scenario_id, + step_key=step.key, + step_type=step.type, + origin=step.origin, + repeat_idx=repeat_idx, + status=status, + should_execute=not is_manual_annotation, + testcase_id=source.testcase_id, + ) + for repeat_idx in repeat_indices + ] diff --git a/sdks/python/agenta/sdk/evaluations/runtime/processor.py b/sdks/python/agenta/sdk/evaluations/runtime/processor.py new file mode 100644 index 0000000000..c6a31a8427 --- /dev/null +++ b/sdks/python/agenta/sdk/evaluations/runtime/processor.py @@ -0,0 +1,736 @@ +import asyncio +from typing import Any, Awaitable, Callable, Dict, List, Mapping, Optional, Union +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + +from agenta.sdk.evaluations.runtime.executor import ( + ResultSetter, + TraceLoader, + execute_workflow_batch, +) +from agenta.sdk.evaluations.runtime.models import ( + EvaluationStep, + PlannedCell, + ResolvedSourceItem, + ResultLogRequest, + WorkflowExecutionRequest, + WorkflowExecutionResult, +) +from agenta.sdk.evaluations.runtime.planner import EvaluationPlanner +from agenta.sdk.models.evaluations import EvaluationStatus +from agenta.sdk.utils.logging import get_logger + +logger = get_logger(__name__) + +# Default concurrency for a slice when the caller passes no explicit batch_size. +# batch_size is None across both drivers by default (nothing sets +# run.data.concurrency), which would leave the engine's semaphore inert and run +# every scenario's workflows unbounded. A bounded default gives both the API and +# the SDK the same in-slice concurrency through the shared semaphore instead of +# dormant machinery. An explicit batch_size (e.g. from run.data.concurrency) +# still overrides this. +DEFAULT_BATCH_SIZE = 10 + + +class ProcessedScenario(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + scenario: Any + results: Dict[str, Any] = Field(default_factory=dict) + metrics: Optional[Any] = None + has_pending: bool = False + has_errors: bool = False + auto_results_created: bool = False + + +CreateScenario = Callable[[UUID], Awaitable[Any]] +RefreshMetrics = Callable[[UUID, Optional[UUID]], Awaitable[Any]] +PlanCellFilter = Callable[[PlannedCell], bool] +# Per-repeat upstream context seed. Either ONE slice-wide dict (every scenario +# gets the same seed — the simple/SDK case, usually None) OR an async callable +# `(scenario_id) -> {repeat_idx: ctx}` resolved lazily per scenario. The +# callable form lets a batched slice carry DIFFERENT recovered context per +# scenario (the API re-execute path) without holding every scenario's context +# in memory at once. +InitialContextByRepeat = Dict[int, Dict[str, Any]] +InitialContextSeed = Union[ + InitialContextByRepeat, + Callable[[UUID], Awaitable[InitialContextByRepeat]], +] +# Adapter boundary for persisting a scenario's terminal status. The engine +# computes the verdict (has_errors/has_pending) per scenario, so writing the +# status is a property of `process` itself — every driver (API ingest, +# API re-execute, SDK) injects its own setter rather than re-deriving status +# in a separate post-process. A plain async `(scenario, status) -> Any`. +EditScenario = Callable[[Any, EvaluationStatus], Awaitable[Any]] + + +def scenario_status( + *, + has_errors: bool, + has_pending: bool, +) -> EvaluationStatus: + """The terminal status of a single scenario from its touched cells. + + Identical ranking on every driver: any error ranks the scenario ERRORS, + else any unresolved cell ranks it PENDING, else SUCCESS. + """ + if has_errors: + return EvaluationStatus.ERRORS + if has_pending: + return EvaluationStatus.PENDING + return EvaluationStatus.SUCCESS + + +def run_status(processed: List[ProcessedScenario]) -> EvaluationStatus: + """The run status rolled up from a slice's touched scenarios. + + One shared rollup for every driver: ERRORS if any scenario errored, else + RUNNING if any is still pending (the run is not done), else SUCCESS. Drivers + apply this verdict in their own way — the SDK closes the run with it; the API + feeds it into the cross-slice floor — but the verdict itself lives here, next + to the per-scenario `scenario_status`, so it is computed in exactly one place. + """ + if any(item.has_errors for item in processed): + return EvaluationStatus.ERRORS + if any(item.has_pending for item in processed): + return EvaluationStatus.RUNNING + return EvaluationStatus.SUCCESS + + +async def process_sources( + *, + run_id: UUID, + source_items: List[ResolvedSourceItem], + steps: List[EvaluationStep], + repeats: Optional[int] = None, + create_scenario: CreateScenario, + set_results: ResultSetter, + refresh_metrics: RefreshMetrics, + runners: Mapping[str, Any], + revisions: Mapping[str, Any], + fetch_trace: Optional[TraceLoader] = None, + is_split: bool = False, + log_pending: bool = True, + refresh_metrics_without_auto_results: bool = True, + batch_size: Optional[int] = None, + max_retries: Optional[int] = None, + retry_delay: Optional[float] = None, + execute_custom: bool = False, + initial_context_by_repeat: Optional[InitialContextSeed] = None, + plan_cell_filter: Optional[PlanCellFilter] = None, + edit_scenario: Optional[EditScenario] = None, +) -> List[ProcessedScenario]: + """Process concrete source items through the SDK-owned runtime contract. + + The function is runner/persistence agnostic. SDK preview uses local + decorator runners and API result logging; backend code can move to this + shape by supplying backend DAO/workflow adapters. + + batch_size controls the maximum number of concurrent invoke_workflow calls + across all scenarios and repeats. A single asyncio.Semaphore is shared by + both the scenario-level gather and the per-step repeat batch so that peak + concurrency equals exactly batch_size regardless of how repeats are split. + When the caller passes no batch_size, `DEFAULT_BATCH_SIZE` applies so the + slice still runs bounded-concurrent rather than unbounded — the same default + for both the API and the SDK driver. + """ + effective_batch_size = batch_size or DEFAULT_BATCH_SIZE + semaphore = asyncio.Semaphore(effective_batch_size) + processed_lock = asyncio.Lock() + processed: List[ProcessedScenario] = [] + + logger.info( + "[SLICE] Starting", + run_id=str(run_id), + scenarios=len(source_items), + batch_size=effective_batch_size, + **({"max_retries": max_retries} if max_retries else {}), + **({"retry_delay": retry_delay} if retry_delay else {}), + ) + + async def _process_one(source_item: ResolvedSourceItem) -> None: + scenario = await create_scenario(run_id) + scenario_id = scenario.id + + logger.info( + "[SCENARIO] Starting", + run_id=str(run_id), + scenario_id=str(scenario_id), + **( + {"testcase_id": str(source_item.testcase_id)} + if source_item.testcase_id + else {} + ), + ) + + plan = EvaluationPlanner().plan( + run_id=run_id, + scenario_id=scenario_id, + source=source_item, + steps=steps, + repeats=repeats, + is_split=is_split, + execute_custom=execute_custom, + ) + if plan_cell_filter is not None: + plan = plan.model_copy( + update={ + "cells": [cell for cell in plan.cells if plan_cell_filter(cell)] + } + ) + # Keyed by step_key, then repeat_idx. A step with repeats>1 produces one + # result per repeat; keying by step_key alone would let later repeats + # overwrite earlier ones, so each step maps to a {repeat_idx: result} + # dict. JSON-serializable (int keys) so it survives the SDK return. + results: Dict[str, Dict[int, Any]] = {} + + def _remember(cell: PlannedCell, value: Any) -> None: + results.setdefault(cell.step_key, {})[cell.repeat_idx] = value + + context_by_repeat = _initial_context_by_repeat( + source_item=source_item, + repeats=repeats, + ) + # The seed is either a slice-wide dict (every scenario the same) or an + # async callable resolved lazily for THIS scenario — the batched-slice + # case where each scenario recovers its own upstream context. + seed = initial_context_by_repeat + if callable(seed): + seed = await seed(scenario_id) + if seed: + context_by_repeat.update(seed) + scenario_has_pending = False + scenario_has_errors = False + scenario_auto_results_created = False + + idx = 0 + while idx < len(plan.cells): + cell = plan.cells[idx] + step = _step_by_key(steps, cell.step_key) + if step is None: + idx += 1 + continue + + if cell.step_type == "input": + _remember( + cell, + await set_results.set( + ResultLogRequest( + cell=cell, + testcase_id=source_item.testcase_id, + trace_id=source_item.trace_id, + ) + ), + ) + idx += 1 + continue + + if not cell.should_execute: + scenario_has_pending = True + logger.info( + "[STEP] Pending", + run_id=str(run_id), + scenario_id=str(scenario_id), + step_key=cell.step_key, + repeat_idx=cell.repeat_idx, + ) + if log_pending: + _remember( + cell, + await set_results.set(ResultLogRequest(cell=cell)), + ) + idx += 1 + continue + + batch_cells = _next_runnable_batch( + cells=plan.cells, + start_idx=idx, + step_key=cell.step_key, + ) + runner = runners.get(cell.step_key) + revision = revisions.get(cell.step_key) + if runner is None or revision is None: + logger.info( + "[STEP] Error", + run_id=str(run_id), + scenario_id=str(scenario_id), + step_key=cell.step_key, + error=f"Missing runner or revision for {cell.step_key}", + ) + for batch_cell in batch_cells: + scenario_has_errors = True + _remember( + batch_cell, + await set_results.set( + ResultLogRequest( + cell=_failed_cell( + batch_cell, + message=( + f"Missing runner or revision for " + f"{batch_cell.step_key}" + ), + ), + error={ + "message": ( + f"Missing runner or revision for " + f"{batch_cell.step_key}" + ) + }, + ) + ), + ) + idx += len(batch_cells) + continue + + requests = [ + _build_execution_request( + cell=batch_cell, + step=step, + source_item=source_item, + revision=revision, + context_by_repeat=context_by_repeat, + ) + for batch_cell in batch_cells + ] + + executions = await _execute_with_retry( + runner=runner, + requests=requests, + semaphore=semaphore, + max_retries=max_retries, + retry_delay=retry_delay, + ) + for batch_cell, execution in zip(batch_cells, executions): + if fetch_trace and execution.trace_id and execution.trace is None: + execution.trace = await fetch_trace(str(execution.trace_id)) + if execution.outputs is None and execution.trace is not None: + execution.outputs = _extract_outputs(execution.trace) + + _remember( + batch_cell, + await set_results.set( + ResultLogRequest( + cell=batch_cell, + trace_id=execution.trace_id, + span_id=execution.span_id, + testcase_id=source_item.testcase_id, + error=execution.error, + ) + ), + ) + scenario_auto_results_created = True + step_failed = bool( + execution.error or _is_failure_status(execution.status) + ) + if step_failed: + scenario_has_errors = True + logger.info( + "[STEP] Done", + run_id=str(run_id), + scenario_id=str(scenario_id), + step_key=batch_cell.step_key, + repeat_idx=batch_cell.repeat_idx, + status=getattr(execution.status, "name", execution.status), + **( + {"trace_id": str(execution.trace_id)} + if execution.trace_id + else {} + ), + **({"error": execution.error} if step_failed else {}), + ) + + if execution.trace_id: + _remember_context( + cell=batch_cell, + context_by_repeat=context_by_repeat, + trace=execution.trace, + trace_id=str(execution.trace_id), + span_id=execution.span_id, + outputs=execution.outputs, + ) + + if len(executions) != len(batch_cells): + scenario_has_errors = True + message = ( + f"Runner for {cell.step_key} returned {len(executions)} " + f"execution(s) for {len(batch_cells)} planned cell(s)." + ) + if len(executions) < len(batch_cells): + # Fewer executions than cells: fail the unplanned-for cells so + # the mismatch is visible per cell. + for batch_cell in batch_cells[len(executions) :]: + _remember( + batch_cell, + await set_results.set( + ResultLogRequest( + cell=_failed_cell(batch_cell, message=message), + testcase_id=source_item.testcase_id, + error={"message": message}, + ) + ), + ) + scenario_auto_results_created = True + else: + # More executions than cells: the extras have no cell and were + # dropped by the zip() above. Warn with their summaries so the + # contract violation leaves an audit trail. + extra_executions = executions[len(batch_cells) :] + logger.warning( + message, + step_key=cell.step_key, + planned_cells=len(batch_cells), + returned_executions=len(executions), + dropped_executions=[ + { + "trace_id": str(execution.trace_id) + if execution.trace_id + else None, + "span_id": str(execution.span_id) + if execution.span_id + else None, + "status": str(execution.status), + "error": execution.error, + } + for execution in extra_executions + ], + ) + + idx += len(batch_cells) + + metrics = None + if refresh_metrics_without_auto_results or scenario_auto_results_created: + try: + logger.info( + "[METRICS] Refreshing", + run_id=str(run_id), + scenario_id=str(scenario_id), + scope="variational", + ) + metrics = await refresh_metrics(run_id, scenario_id) + except Exception: # pylint: disable=broad-exception-caught + # A metrics-refresh failure must not lose the result cells that + # were already written for this scenario, nor abort sibling + # scenarios in the gather. Mark the scenario errored and carry on. + logger.error( + "[SLICE] scenario metrics refresh failed", + run_id=str(run_id), + scenario_id=str(scenario_id), + exc_info=True, + ) + scenario_has_errors = True + + status = scenario_status( + has_errors=scenario_has_errors, + has_pending=scenario_has_pending, + ) + logger.info( + "[SCENARIO] Complete", + run_id=str(run_id), + scenario_id=str(scenario_id), + status=status.name, + cells=len(plan.cells), + ) + + # Per-scenario status write. The verdict is known here (this scenario's + # touched cells), so the engine owns the write via the injected adapter + # — the same path for every driver, instead of a separate post-process. + if edit_scenario is not None: + await edit_scenario(scenario, status) + + async with processed_lock: + processed.append( + ProcessedScenario( + scenario=scenario, + results=results, + metrics=metrics, + has_pending=scenario_has_pending, + has_errors=scenario_has_errors, + auto_results_created=scenario_auto_results_created, + ) + ) + + async def _guarded_process_one(source_item: ResolvedSourceItem) -> None: + # One scenario's failure (e.g. create_scenario or an unhandled error in + # the loop) must not abort the whole slice. Isolate it: log and continue + # so sibling scenarios still run and partial results survive. + try: + await _process_one(source_item) + except Exception: # pylint: disable=broad-exception-caught + logger.error( + "[SLICE] scenario processing failed", + run_id=str(run_id), + step_key=source_item.step_key, + exc_info=True, + ) + + await asyncio.gather(*(_guarded_process_one(item) for item in source_items)) + + has_errors = any(item.has_errors for item in processed) + logger.info( + "[SLICE] Complete", + run_id=str(run_id), + processed=len(processed), + **({"has_errors": has_errors} if has_errors else {}), + ) + + if processed and ( + refresh_metrics_without_auto_results + or any(item.auto_results_created for item in processed) + ): + try: + logger.info( + "[METRICS] Refreshing", + run_id=str(run_id), + scope="global", + ) + await refresh_metrics(run_id, None) + except Exception: # pylint: disable=broad-exception-caught + # The run-level rollup is best-effort: every scenario already + # refreshed its own metrics above, and the result cells are + # persisted. A failure here must not discard the processed list the + # caller uses to finalize run status. + logger.error( + "[SLICE] run-level metrics refresh failed", + run_id=str(run_id), + exc_info=True, + ) + + return processed + + +async def _execute_with_retry( + *, + runner: Any, + requests: List[WorkflowExecutionRequest], + semaphore: Optional[asyncio.Semaphore], + max_retries: Optional[int], + retry_delay: Optional[float], +) -> List[WorkflowExecutionResult]: + attempts = max(1, (max_retries or 0) + 1) + delay = retry_delay or 0.0 + results: List[WorkflowExecutionResult] = await execute_workflow_batch( + runner=runner, + requests=requests, + semaphore=semaphore, + ) + for attempt in range(attempts - 1): + failed_indices = [ + i for i, r in enumerate(results) if r.error or _is_failure_status(r.status) + ] + if not failed_indices: + break + logger.warning( + "[RETRY] Retrying failed requests", + attempt=attempt + 1, + failed=len(failed_indices), + total=len(requests), + delay=delay, + ) + if delay > 0: + await asyncio.sleep(delay) + retried = await execute_workflow_batch( + runner=runner, + requests=[requests[i] for i in failed_indices], + semaphore=semaphore, + ) + for idx, result in zip(failed_indices, retried): + results[idx] = result + return results + + +def _is_failure_status(status: EvaluationStatus) -> bool: + # `WorkflowExecutionResult.status` is typed `EvaluationStatus`, so compare + # members directly rather than matching `str(status)` against literals. + return status in (EvaluationStatus.FAILURE, EvaluationStatus.ERRORS) + + +def _step_by_key( + steps: List[EvaluationStep], + step_key: str, +) -> Optional[EvaluationStep]: + for step in steps: + if step.key == step_key: + return step + return None + + +def _initial_context_by_repeat( + *, + source_item: ResolvedSourceItem, + repeats: Optional[int], +) -> Dict[int, Dict[str, Any]]: + if not source_item.trace and not source_item.trace_id: + return {} + + trace = source_item.trace + trace_id = source_item.trace_id or _get_trace_id(trace) + root_span = _extract_root_span(trace) + span_id = source_item.span_id or _get_span_id(root_span) + outputs = source_item.outputs or _extract_outputs(trace) + if not trace_id: + return {} + + context = { + "trace": trace, + "trace_id": str(trace_id), + "span_id": span_id, + "outputs": outputs, + } + count = repeats or 1 + return {repeat_idx: context for repeat_idx in range(max(count, 1))} + + +def _next_runnable_batch( + *, + cells: List[PlannedCell], + start_idx: int, + step_key: str, +) -> List[PlannedCell]: + batch = [] + for cell in cells[start_idx:]: + if not cell.should_execute or cell.step_key != step_key: + break + batch.append(cell) + return batch + + +def _build_execution_request( + *, + cell: PlannedCell, + step: EvaluationStep, + source_item: ResolvedSourceItem, + revision: Any, + context_by_repeat: Dict[int, Dict[str, Any]], +) -> WorkflowExecutionRequest: + upstream = _upstream_for_cell( + cell=cell, + context_by_repeat=context_by_repeat, + ) + return WorkflowExecutionRequest( + step=step, + cell=cell, + source=source_item, + revision=_dump_revision(revision), + parameters=_revision_parameters(revision), + references={ + **(source_item.references or {}), + **(step.references or {}), + }, + links=upstream.get("links"), + upstream_trace=upstream.get("trace"), + upstream_outputs=upstream.get("outputs"), + ) + + +def _failed_cell(cell: PlannedCell, *, message: str) -> PlannedCell: + return cell.model_copy( + update={ + "status": EvaluationStatus.FAILURE, + "error": {"message": message}, + } + ) + + +def _dump_revision(revision: Any) -> Any: + if hasattr(revision, "model_dump"): + return revision.model_dump(mode="json", exclude_none=True) + return revision + + +def _revision_parameters(revision: Any) -> Optional[Any]: + data = getattr(revision, "data", None) + return getattr(data, "parameters", None) if data else None + + +def _upstream_for_cell( + *, + cell: PlannedCell, + context_by_repeat: Dict[int, Dict[str, Any]], +) -> Dict[str, Any]: + context = context_by_repeat.get(cell.repeat_idx) or context_by_repeat.get(0) or {} + if not context: + return {} + + trace_id = context.get("trace_id") + span_id = context.get("span_id") + links = ( + { + "invocation": { + "trace_id": trace_id, + "span_id": span_id, + } + } + if trace_id and span_id + else None + ) + return { + "links": links, + "trace": context.get("trace"), + "outputs": context.get("outputs"), + } + + +def _remember_context( + *, + cell: PlannedCell, + context_by_repeat: Dict[int, Dict[str, Any]], + trace: Optional[Any], + trace_id: str, + span_id: Optional[str], + outputs: Optional[Any], +) -> None: + context = { + "trace": trace, + "trace_id": trace_id, + "span_id": span_id, + "outputs": outputs, + } + context_by_repeat[cell.repeat_idx] = context + if cell.step_type == "invocation" and 0 not in context_by_repeat: + context_by_repeat[0] = context + + +def _extract_outputs(trace: Any) -> Optional[Any]: + root_span = _extract_root_span(trace) + if root_span is None: + return None + attributes = ( + root_span.get("attributes", {}) + if isinstance(root_span, dict) + else getattr(root_span, "attributes", {}) + ) + if hasattr(attributes, "model_dump"): + attributes = attributes.model_dump(mode="json", exclude_none=True) + return attributes.get("ag", {}).get("data", {}).get("outputs") + + +def _extract_root_span(trace: Any) -> Optional[Any]: + spans = ( + trace.get("spans") if isinstance(trace, dict) else getattr(trace, "spans", None) + ) + if not spans: + return None + root_span = next(iter(spans.values()), None) if isinstance(spans, dict) else None + if isinstance(root_span, list): + return None + return root_span + + +def _get_trace_id(trace: Any) -> Optional[str]: + if isinstance(trace, dict): + return trace.get("trace_id") + trace_id = getattr(trace, "trace_id", None) + return str(trace_id) if trace_id else None + + +def _get_span_id(root_span: Any) -> Optional[str]: + if root_span is None: + return None + span_id = ( + root_span.get("span_id") + if isinstance(root_span, dict) + else getattr(root_span, "span_id", None) + ) + return str(span_id) if span_id else None diff --git a/sdks/python/agenta/sdk/evaluations/runtime/topology.py b/sdks/python/agenta/sdk/evaluations/runtime/topology.py new file mode 100644 index 0000000000..d06d08512e --- /dev/null +++ b/sdks/python/agenta/sdk/evaluations/runtime/topology.py @@ -0,0 +1,145 @@ +from typing import Iterable, List, Optional + +from agenta.sdk.evaluations.runtime.models import EvaluationStep, TopologyDecision + + +def _has_reference(step: EvaluationStep, token: str) -> bool: + if any(token in str(key).lower() for key in step.references.keys()): + return True + return token in step.key.lower() + + +def _input_family(step: EvaluationStep) -> Optional[str]: + if _has_reference(step, "query"): + return "query" + if _has_reference(step, "testset"): + return "testset" + if _has_reference(step, "trace"): + return "trace" + if _has_reference(step, "testcase"): + return "testcase" + return None + + +def _steps_of_type( + steps: Iterable[EvaluationStep], step_type: str +) -> List[EvaluationStep]: + return [step for step in steps if step.type == step_type] + + +def classify_steps_topology( + *, + steps: List[EvaluationStep], + is_live: bool = False, + has_queries: bool = False, + has_testsets: bool = False, + has_traces: bool = False, + has_testcases: bool = False, + has_evaluators: bool = False, +) -> TopologyDecision: + input_steps = _steps_of_type(steps, "input") + application_steps = _steps_of_type(steps, "invocation") + evaluator_steps = _steps_of_type(steps, "annotation") + + input_families = { + family for family in (_input_family(step) for step in input_steps) if family + } + has_queries = has_queries or "query" in input_families + has_testsets = has_testsets or "testset" in input_families + has_traces = has_traces or "trace" in input_families + has_testcases = has_testcases or "testcase" in input_families + has_applications = bool(application_steps) + has_evaluators = has_evaluators or bool(evaluator_steps) + + if has_queries and has_testsets: + return TopologyDecision( + status="not_planned", + label="mixed query and testset sources", + reason="mixed query and testset source families in one run are not planned", + ) + + if is_live and has_testsets: + return TopologyDecision( + status="not_planned", + label="live testset evaluation", + reason="live testset evaluation is not a meaningful product shape", + ) + + if len(application_steps) > 1: + return TopologyDecision( + status="not_planned", + label="multiple application steps", + reason="A/B application comparisons should use separate evaluations", + ) + + if is_live and has_queries and has_evaluators and not has_applications: + return TopologyDecision( + status="supported", + label="live query -> evaluator", + reason="live query evaluator runs keep scheduler/windowing behavior", + dispatch="live_query", + ) + + if has_evaluators and not has_applications: + if has_testcases: + return TopologyDecision( + status="supported", + label="direct testcases -> evaluator", + reason="direct testcase batches are worker-dispatched", + dispatch="queue_testcases", + ) + if has_traces: + return TopologyDecision( + status="supported", + label="direct traces -> evaluator", + reason="direct trace batches are worker-dispatched", + dispatch="queue_traces", + ) + + if has_queries and has_applications: + return TopologyDecision( + status="potential", + label="query -> application", + reason=( + "query traces can seed application calls, but source trace links must " + "not be attached as application links because that would classify the " + "new application traces as annotations" + ), + ) + + if has_testsets and has_evaluators and not has_applications: + return TopologyDecision( + status="potential", + label="testset -> evaluator", + reason="non-queue testcase-only evaluator execution needs an explicit evaluator contract", + ) + + if has_queries and has_evaluators and not has_applications: + return TopologyDecision( + status="supported", + label="batch query -> evaluator", + reason="batch query evaluator runs are worker-dispatched", + dispatch="batch_query", + ) + + if has_testsets and has_applications and has_evaluators: + return TopologyDecision( + status="supported", + label="testset -> application -> evaluator", + reason="batch testset evaluation is worker-dispatched", + dispatch="batch_testset", + ) + + if has_testsets and has_applications and not has_evaluators and not has_queries: + return TopologyDecision( + status="supported", + label="testset -> application", + reason="batch inference / batch invocation is worker-dispatched", + dispatch="batch_invocation", + ) + + return TopologyDecision( + status="unsupported", + label="unsupported evaluation topology", + reason="no current worker dispatch path matches this evaluation graph", + ) diff --git a/sdks/python/agenta/sdk/evaluations/scenarios.py b/sdks/python/agenta/sdk/evaluations/scenarios.py index 2b3b6e5ab1..00be347c01 100644 --- a/sdks/python/agenta/sdk/evaluations/scenarios.py +++ b/sdks/python/agenta/sdk/evaluations/scenarios.py @@ -1,4 +1,5 @@ -from typing import Optional, Dict, Any +from datetime import datetime +from typing import Optional, Dict, Any, List from uuid import UUID from agenta.sdk.utils.client import authed_api @@ -7,6 +8,42 @@ # TODO: ADD TYPES +async def aadd( + *, + run_id: UUID, + count: int, + timestamp: Optional[datetime] = None, +) -> List[EvaluationScenario]: + """Bulk-mint `count` skeleton scenarios for a run (the `add_scenarios` op). + + Mirrors `POST /simple/evaluations/{id}/scenarios/add`: skeleton rows with no + cells, returned in order so the caller can bind result cells to their ids. + `timestamp` buckets them on the temporal axis (live runs); None otherwise. + """ + if count <= 0: + return [] + + payload: Dict[str, Any] = {"count": count} + if timestamp is not None: + payload["timestamp"] = timestamp.isoformat() + + response = authed_api()( + method="POST", + endpoint=f"/simple/evaluations/{run_id}/scenarios/add", + json=payload, + ) + + try: + response.raise_for_status() + except: + print(response.text) + raise + + response = response.json() + + return [EvaluationScenario(**s) for s in response.get("scenarios", [])] + + async def acreate( *, run_id: UUID, @@ -46,3 +83,54 @@ async def acreate( scenario = EvaluationScenario(**response["scenarios"][0]) return scenario + + +async def aedit_scenario( + *, + scenario_id: UUID, + status: str, + tags: Optional[Dict[str, Any]] = None, + meta: Optional[Dict[str, Any]] = None, +) -> Optional[EvaluationScenario]: + """Edit a single scenario (status, and optionally tags/meta). + + Mirrors `PATCH /evaluations/scenarios/{scenario_id}` (operation + `edit_scenario`); the body's `scenario.id` must match the path id. Used by + the SDK evaluate loop's `edit_scenario` adapter to flip each scenario to its + computed SUCCESS/ERRORS/PENDING status after its cells are written. + + Carries `tags`/`meta` like the API's `APIScenarioEditor`, and tolerates a + run closed mid-flight: the API returns 409 (EvaluationClosedException) for an + edit against a locked run — closing is a lock, not a failure, so we return + None rather than raising, matching the API adapter's + `except EvaluationClosedConflict`. + """ + scenario: Dict[str, Any] = dict( + id=str(scenario_id), + status=status, + ) + if tags is not None: + scenario["tags"] = tags + if meta is not None: + scenario["meta"] = meta + + response = authed_api()( + method="PATCH", + endpoint=f"/evaluations/scenarios/{scenario_id}", + json=dict(scenario=scenario), + ) + + # Run closed (locked) mid-flight -> 409. Skip the write, don't raise. + if response.status_code == 409: + return None + + try: + response.raise_for_status() + except Exception: + print(response.text) + raise + + response = response.json() + + scenario_data = response.get("scenario") + return EvaluationScenario(**scenario_data) if scenario_data else None diff --git a/sdks/python/agenta/sdk/litellm/mocks/__init__.py b/sdks/python/agenta/sdk/litellm/mocks/__init__.py index 59f3947c8b..ea7418aa4d 100644 --- a/sdks/python/agenta/sdk/litellm/mocks/__init__.py +++ b/sdks/python/agenta/sdk/litellm/mocks/__init__.py @@ -78,11 +78,34 @@ def capital_mock_response(*args, **kwargs) -> MockResponseModel: ) +@instrument() +def echo_mock_response(*args, **kwargs) -> MockResponseModel: + # Returns the last user message content verbatim — deterministic, useful for + # asserting prompt plumbing without an LLM. + messages = kwargs.get("messages") or [] + content = "" + for message in reversed(messages): + if isinstance(message, dict) and message.get("role") == "user": + content = str(message.get("content", "")) + break + return MockResponseModel( + choices=[MockChoiceModel(message=MockMessageModel(content=content))], + ) + + +@instrument() +def error_mock_response(*args, **kwargs) -> MockResponseModel: + # Raises to exercise the LLM failure path deterministically. + raise RuntimeError("mock LLM error") + + MOCKS: dict[str, Callable[..., MockResponseModel]] = { "hello": hello_mock_response, "chat": chat_mock_response, "delay": delay_mock_response, "capital": capital_mock_response, + "echo": echo_mock_response, + "error": error_mock_response, } CAPITALS = { diff --git a/sdks/python/agenta/sdk/managers/applications.py b/sdks/python/agenta/sdk/managers/applications.py index f4b2ac91af..72fb59dc41 100644 --- a/sdks/python/agenta/sdk/managers/applications.py +++ b/sdks/python/agenta/sdk/managers/applications.py @@ -162,7 +162,7 @@ async def aupsert( req = await application_workflow.inspect() - simple_application_flags = SimpleApplicationFlags(**req.flags) + simple_application_flags = SimpleApplicationFlags(**(req.flags or {})) simple_application_data = SimpleApplicationData( **( @@ -264,36 +264,6 @@ async def aupsert( # print("Retrieve response:", retrieve_response) if retrieve_response and retrieve_response.id and retrieve_response.application_id: - existing_application_name = None - try: - with_name = await _fetch_simple_application( - application_id=retrieve_response.application_id - ) - except ValueError as e: - print( - "[WARN]: Failed to fetch existing application for name preservation; " - f"continuing without it: {e}" - ) - with_name = None - if with_name: - existing_application_name = with_name.name - - # TEMPORARY: API simple application edit currently rejects renaming. - # Preserve the existing stored name when updating by slug/id so evaluate() - # can keep syncing configuration/data without triggering rename failures. - if ( - existing_application_name - and name is not None - and name != existing_application_name - ): - print( - "[INFO]: Renaming applications is temporarily disabled. " - f"Using existing application name '{existing_application_name}'." - ) - name = existing_application_name - elif existing_application_name and name is None: - name = existing_application_name - application_id = retrieve_response.application_id # print(" --- Updating application...", application_id) application_edit_request = SimpleApplicationEdit( diff --git a/sdks/python/agenta/sdk/managers/evaluators.py b/sdks/python/agenta/sdk/managers/evaluators.py index 7c9c9e169b..0d440cc566 100644 --- a/sdks/python/agenta/sdk/managers/evaluators.py +++ b/sdks/python/agenta/sdk/managers/evaluators.py @@ -158,7 +158,7 @@ async def aupsert( req = await evaluator_workflow.inspect() - legacy_application_flags = SimpleEvaluatorFlags(**req.flags) + legacy_application_flags = SimpleEvaluatorFlags(**(req.flags or {})) simple_evaluator_data = SimpleEvaluatorData( **( diff --git a/sdks/python/agenta/sdk/middlewares/routing/auth.py b/sdks/python/agenta/sdk/middlewares/routing/auth.py index 3c83d8bea8..6d11d89c9a 100644 --- a/sdks/python/agenta/sdk/middlewares/routing/auth.py +++ b/sdks/python/agenta/sdk/middlewares/routing/auth.py @@ -142,7 +142,7 @@ async def get_credentials( async with httpx.AsyncClient() as client: try: response = await client.get( - f"{host}/api/permissions/verify", + f"{host}/api/access/permissions/check", headers=headers, cookies=cookies, params=params, diff --git a/sdks/python/agenta/sdk/middlewares/running/vault.py b/sdks/python/agenta/sdk/middlewares/running/vault.py index 2c4fc0d719..5061b7879f 100644 --- a/sdks/python/agenta/sdk/middlewares/running/vault.py +++ b/sdks/python/agenta/sdk/middlewares/running/vault.py @@ -131,7 +131,7 @@ async def _allow_local_secrets( ): """ Verify if the user has permission to use local secrets. - Makes an API call to /api/permissions/verify to check access. + Makes an API call to /api/access/permissions/check to check access. """ try: if not _AUTH_ENABLED: @@ -174,7 +174,7 @@ async def _allow_local_secrets( async with httpx.AsyncClient() as client: try: response = await client.get( - f"{host}/api/permissions/verify", + f"{host}/api/access/permissions/check", headers=headers, params=params, timeout=30.0, @@ -329,7 +329,7 @@ async def get_secrets( try: async with httpx.AsyncClient() as client: response = await client.get( - f"{api_url}/vault/v1/secrets/", + f"{api_url}/secrets/", headers=headers, ) diff --git a/sdks/python/agenta/sdk/models/evaluations.py b/sdks/python/agenta/sdk/models/evaluations.py index 267f996fab..94479ee24c 100644 --- a/sdks/python/agenta/sdk/models/evaluations.py +++ b/sdks/python/agenta/sdk/models/evaluations.py @@ -42,6 +42,9 @@ class EvaluationRunFlags(BaseModel): is_live: bool = False # Indicates if the run has live queries is_active: bool = False # Indicates if the run is currently active is_closed: bool = False # Indicates if the run is modifiable + is_queue: bool = False # Indicates this run belongs to an annotation queue + is_cached: bool = False # Indicates the run should reuse traces by hash + is_split: bool = False # Indicates repeats fan out at the application step # has_queries: bool = False # Indicates if the run has queries has_testsets: bool = False # Indicates if the run has testsets @@ -86,9 +89,10 @@ class EvaluationResult(BaseModel): run_id: UUID scenario_id: UUID step_key: str + repeat_idx: Optional[int] = 0 testcase_id: Optional[UUID] = None - trace_id: Optional[UUID] = None + trace_id: Optional[str] = None error: Optional[dict] = None flags: Optional[Dict[str, Any]] = None diff --git a/sdks/python/oss/tests/legacy/new_tests/vault_router/conftest.py b/sdks/python/oss/tests/legacy/new_tests/vault_router/conftest.py index aef2c39a57..ec4abb182f 100644 --- a/sdks/python/oss/tests/legacy/new_tests/vault_router/conftest.py +++ b/sdks/python/oss/tests/legacy/new_tests/vault_router/conftest.py @@ -6,7 +6,7 @@ AGENTA_HOST = os.environ.get("AGENTA_HOST", "http://localhost") -API_BASE_URL = f"{AGENTA_HOST}/api/vault/v1/" +API_BASE_URL = f"{AGENTA_HOST}/api/" @pytest.fixture diff --git a/sdks/python/oss/tests/pytest/acceptance/evaluations/test_evaluate_flow.py b/sdks/python/oss/tests/pytest/acceptance/evaluations/test_evaluate_flow.py new file mode 100644 index 0000000000..b8cc5d8f24 --- /dev/null +++ b/sdks/python/oss/tests/pytest/acceptance/evaluations/test_evaluate_flow.py @@ -0,0 +1,182 @@ +""" +End-to-end acceptance tests for the high-level evaluate() entrypoint +(`agenta.sdk.evaluations.aevaluate`). + +aevaluate() is a CLIENT-SIDE orchestrator: it upserts the testset/application/ +evaluator, creates a run, builds the input/invocation/annotation step graph, +executes the slice IN-PROCESS via the SDK local runners, computes metrics, and +closes the run — returning {run, scenarios, metrics}. No worker is involved. + +These run against the live API (acceptance) using deterministic, LLM-free +handlers, so they exercise the real upsert + run + local-execution path. + +Coverage: + - local-callable mode (handlers passed directly) across the config matrix: + single auto evaluator, repeats, multiple evaluators; + - saved-workflow mode once: the application is the saved agenta:custom:mock:v0 + workflow referenced by revision id. + +We do not re-run the whole matrix through both modes — the matrix runs through +local-callable mode; saved-workflow mode gets a single wiring test. +""" + +from uuid import uuid4 + +import pytest + +import agenta as ag +from agenta.sdk.evaluations import aevaluate +from agenta.sdk.managers import testsets, applications + +pytestmark = [pytest.mark.acceptance, pytest.mark.asyncio] + +MOCK_URI = "agenta:custom:mock:v0" + + +@ag.workflow(uri=MOCK_URI, parameters={"key": "echo", "kwargs": {}}) +def mock_app(): + """Saved-workflow handle pointing at the deterministic agenta:custom:mock:v0 + workflow (echo selector). Registered as an application via its URI.""" + + +# --- deterministic, LLM-free local handlers -------------------------------- + + +@ag.application() +def echo_app(topic: str = "") -> dict: + """A trivial deterministic application: echoes the input.""" + return {"answer": topic} + + +@ag.evaluator() +def pass_evaluator(answer: str = "", **kwargs) -> dict: + """A deterministic evaluator that always passes.""" + return {"score": 1.0, "success": True} + + +@ag.evaluator() +def length_evaluator(answer: str = "", **kwargs) -> dict: + """A deterministic evaluator scoring by output length.""" + return {"score": float(len(answer or "")), "success": True} + + +async def _make_testset(): + name = f"sdk-eval-flow-{uuid4().hex[:8]}" + rev = await testsets.aupsert( + name=name, + data=[ + {"topic": "alpha"}, + {"topic": "beta"}, + ], + ) + assert rev is not None and rev.testset_id + return rev + + +def _assert_eval_result(result, *, expected_scenarios): + assert result is not None, "aevaluate returned None" + assert set(result.keys()) == {"run", "scenarios", "metrics"} + assert result["run"] is not None and result["run"].id + assert len(result["scenarios"]) == expected_scenarios + + +def _metrics_data(result): + m = result["metrics"] + return getattr(m, "data", None) or {} + + +def _assert_evaluator_metrics_present(result): + # The evaluator must actually be EXECUTED by the SDK runtime (custom origin), + # so its outputs land in the run metrics. Before the custom-execution fix the + # evaluator step was skipped (logged pending, trace_id=None) and produced no + # metrics — this assertion guards that regression. + data = _metrics_data(result) + evaluator_steps = {k: v for k, v in data.items() if k.startswith("evaluator-")} + assert evaluator_steps, f"no evaluator metrics in run metrics: {list(data.keys())}" + # at least one evaluator step exposes its scored output + assert any( + any("ag.data.outputs.score" in path for path in v) + for v in evaluator_steps.values() + ), f"evaluator metrics present but no score output: {evaluator_steps}" + + +class TestEvaluateLocalCallable: + async def test_basic_testset_app_auto_evaluator(self, agenta_init): + rev = await _make_testset() + result = await aevaluate( + name="sdk-eval-basic", + testsets={str(rev.id): "custom"}, + applications=[echo_app], + evaluators=[pass_evaluator], + ) + # 2 testcases x 1 repeat = 2 scenarios + _assert_eval_result(result, expected_scenarios=2) + # the custom (SDK-run) evaluator must actually execute and yield metrics + _assert_evaluator_metrics_present(result) + + async def test_with_repeats(self, agenta_init): + rev = await _make_testset() + result = await aevaluate( + name="sdk-eval-repeats", + testsets={str(rev.id): "custom"}, + applications=[echo_app], + evaluators=[pass_evaluator], + repeats=2, + ) + # repeats fan out at the scenario/cell level; still one scenario per + # testcase row (2), each carrying repeated cells. + _assert_eval_result(result, expected_scenarios=2) + + async def test_multiple_evaluators(self, agenta_init): + rev = await _make_testset() + result = await aevaluate( + name="sdk-eval-multi", + testsets={str(rev.id): "custom"}, + applications=[echo_app], + evaluators=[pass_evaluator, length_evaluator], + ) + _assert_eval_result(result, expected_scenarios=2) + _assert_evaluator_metrics_present(result) + + async def test_specs_dict_equivalent_to_kwargs(self, agenta_init): + rev = await _make_testset() + result = await aevaluate( + name="sdk-eval-specs", + specs={ + "testsets": {str(rev.id): "custom"}, + "applications": [echo_app], + "evaluators": [pass_evaluator], + }, + ) + _assert_eval_result(result, expected_scenarios=2) + + async def test_missing_evaluators_raises(self, agenta_init): + rev = await _make_testset() + with pytest.raises(ValueError, match="missing evaluators"): + await aevaluate( + name="sdk-eval-bad", + testsets={str(rev.id): "custom"}, + applications=[echo_app], + ) + + +class TestEvaluateSavedWorkflow: + async def test_saved_mock_workflow_application(self, agenta_init): + # Saved-workflow mode: register the agenta:custom:mock:v0 workflow as an + # application revision, then reference it by revision id in aevaluate. + rev = await _make_testset() + + app_revision_id = await applications.aupsert( + application_slug=f"mock-app-{uuid4().hex[:8]}", + name="Mock App", + handler=mock_app, + ) + assert app_revision_id is not None + + result = await aevaluate( + name="sdk-eval-saved", + testsets={str(rev.id): "custom"}, + applications={str(app_revision_id): "custom"}, + evaluators=[pass_evaluator], + ) + _assert_eval_result(result, expected_scenarios=2) diff --git a/sdks/python/oss/tests/pytest/acceptance/integrations/test_vault_secrets.py b/sdks/python/oss/tests/pytest/acceptance/integrations/test_vault_secrets.py index d99a8db3fa..8d5cfc8a7a 100644 --- a/sdks/python/oss/tests/pytest/acceptance/integrations/test_vault_secrets.py +++ b/sdks/python/oss/tests/pytest/acceptance/integrations/test_vault_secrets.py @@ -2,7 +2,7 @@ Integration tests for Vault/Secrets functionality. These tests verify: -1. Permissions verification via access_control.verify_permissions() +1. Permissions verification via access.verify_permissions() 2. Secrets CRUD via secrets.list_secrets(), create_secret(), read_secret(), delete_secret() The vault middleware uses these endpoints during workflow execution to: @@ -13,6 +13,7 @@ import time import pytest +import requests import agenta as ag from agenta.client.types import ( @@ -53,26 +54,28 @@ def _wait_for_secret_ids( class TestAccessControlPermissions: """Test access control permission verification.""" - @pytest.mark.xfail( - reason=( - "/permissions router is mounted with include_in_schema=False, " - "so Fern does not generate ag.api.access_control. Either flip the " - "schema flag and regenerate the client, or rewrite this test " - "against the raw HTTP endpoint." - ), - strict=True, - ) - def test_verify_permissions_for_local_secrets(self, agenta_init): + @staticmethod + def _check_permissions(*, e2e_account): + response = requests.get( + f"{e2e_account['api_url']}/access/permissions/check", + headers={"Authorization": e2e_account["credentials"]}, + params={ + "action": "view_secret", + "resource_type": "local_secrets", + }, + timeout=30, + ) + response.raise_for_status() + return response.json() + + def test_verify_permissions_for_local_secrets(self, agenta_init, e2e_account): """ - Test that verify_permissions works for local_secrets resource. + Test that the permission check endpoint works for local_secrets resource. This is the same call the vault middleware makes to check if a user can use local (env var) secrets during workflow execution. """ - result = ag.api.access_control.verify_permissions( - action="view_secret", - resource_type="local_secrets", - ) + result = self._check_permissions(e2e_account=e2e_account) # The response should indicate the permission effect assert result is not None @@ -81,23 +84,13 @@ def test_verify_permissions_for_local_secrets(self, agenta_init): # Effect should be "allow" or "deny" assert result["effect"] in ("allow", "deny") - @pytest.mark.xfail( - reason=( - "/permissions router is mounted with include_in_schema=False, " - "so Fern does not generate ag.api.access_control. Either flip the " - "schema flag and regenerate the client, or rewrite this test " - "against the raw HTTP endpoint." - ), - strict=True, - ) - def test_verify_permissions_returns_allow_for_valid_user(self, agenta_init): + def test_verify_permissions_returns_allow_for_valid_user( + self, agenta_init, e2e_account + ): """ Test that a valid API key gets 'allow' effect for view_secret. """ - result = ag.api.access_control.verify_permissions( - action="view_secret", - resource_type="local_secrets", - ) + result = self._check_permissions(e2e_account=e2e_account) assert result is not None # A valid API key should have permission to view secrets diff --git a/sdks/python/oss/tests/pytest/acceptance/workflows/test_apps_shared_manager.py b/sdks/python/oss/tests/pytest/acceptance/workflows/test_apps_shared_manager.py index 5b6f820c33..68ef2c8623 100644 --- a/sdks/python/oss/tests/pytest/acceptance/workflows/test_apps_shared_manager.py +++ b/sdks/python/oss/tests/pytest/acceptance/workflows/test_apps_shared_manager.py @@ -141,15 +141,12 @@ def test_list_apps_contains_created_app(self, agenta_init, test_app): f"Created app {test_app['app_id']} should be in the list" ) - @pytest.mark.xfail(reason="TEMPORARY: Renaming applications is disabled in the API") def test_update_app(self, agenta_init, test_app): """Test updating an app via AppManager.update().""" new_slug = generate_unique_slug("updated") _result = AppManager.update(app_id=test_app["app_id"], app_slug=new_slug) - # update() may return None or the updated app - # The important thing is it doesn't raise an exception assert _result is None or hasattr(_result, "app_id") def test_delete_app(self, agenta_init): @@ -240,13 +237,11 @@ async def test_alist_apps(self, agenta_init): assert_not_none(result, "alist() should return a response") assert isinstance(result, list), "alist() should return a list" - @pytest.mark.xfail(reason="TEMPORARY: Renaming applications is disabled in the API") async def test_aupdate_app(self, agenta_init, test_app): """Test updating an app via AppManager.aupdate().""" new_slug = generate_unique_slug("async-updated") _result = await AppManager.aupdate(app_id=test_app["app_id"], app_slug=new_slug) - # Update may return None or the updated app assert _result is None or hasattr(_result, "app_id") async def test_adelete_app(self, agenta_init): diff --git a/sdks/python/oss/tests/pytest/integration/__init__.py b/sdks/python/oss/tests/pytest/integration/__init__.py new file mode 100644 index 0000000000..a26504824f --- /dev/null +++ b/sdks/python/oss/tests/pytest/integration/__init__.py @@ -0,0 +1 @@ +# Integration tests package diff --git a/sdks/python/oss/tests/pytest/integration/conftest.py b/sdks/python/oss/tests/pytest/integration/conftest.py new file mode 100644 index 0000000000..202af977f8 --- /dev/null +++ b/sdks/python/oss/tests/pytest/integration/conftest.py @@ -0,0 +1,2 @@ +# Integration tests wire multiple SDK modules together with the network +# boundary mocked. No live services required (unlike acceptance/). diff --git a/sdks/python/oss/tests/pytest/integration/test_evaluate_orchestration.py b/sdks/python/oss/tests/pytest/integration/test_evaluate_orchestration.py new file mode 100644 index 0000000000..64b4022ed5 --- /dev/null +++ b/sdks/python/oss/tests/pytest/integration/test_evaluate_orchestration.py @@ -0,0 +1,232 @@ +""" +Integration-level tests for the aevaluate() orchestration. + +These exercise the real SDK orchestration in agenta.sdk.evaluations.preview.evaluate +(spec parsing, entity retrieval, step-graph construction, runner wiring, result +assembly) with the network boundary and the slice processor mocked. They verify +that aevaluate(): + + - builds the input/invocation/annotation step graph with the right keys + origins, + - wires a local application runner and (for auto evaluators) a local evaluator + runner, but NOT a runner for human evaluators, + - forwards repeats and the resolved source_items to the processor, + - assembles {run, scenarios, metrics} from the processed scenarios. + +The runtime processor itself is unit-tested separately (test_evaluations_runtime); +here it is mocked so these tests stay deterministic and free of execution timing. +""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +import pytest + +import agenta.sdk.evaluations.preview.evaluate as ev +from agenta.sdk.evaluations.runtime.adapters import ( + SDKWorkflowRunner, +) + +pytestmark = pytest.mark.integration + +MOD = "agenta.sdk.evaluations.preview.evaluate" + + +def run(coro): + return asyncio.run(coro) + + +def _testcase(data): + return SimpleNamespace( + id=uuid4(), + data=data, + model_dump=lambda **kw: {"data": data}, + ) + + +def _testset_revision(*, slug="t1", testcases): + return SimpleNamespace( + id=uuid4(), + slug=slug, + version="1", + testset_id=uuid4(), + testset_variant_id=uuid4(), + data=SimpleNamespace(testcases=testcases), + ) + + +def _application_revision(*, slug="app1"): + return SimpleNamespace( + id=uuid4(), + slug=slug, + version="1", + application_id=uuid4(), + application_variant_id=uuid4(), + data=SimpleNamespace(), + ) + + +def _evaluator_revision(*, slug="ev1"): + return SimpleNamespace( + id=uuid4(), + slug=slug, + version="1", + evaluator_id=uuid4(), + evaluator_variant_id=uuid4(), + data=SimpleNamespace(), + ) + + +def _patch_io(*, testset_revision, application_revision, evaluator_revision, captured): + """Patch every network/IO boundary aevaluate() uses, and capture the + process_sources kwargs. Returns a context-manager list. + + The flow runs process_sources ONCE per testset over all source items — + `captured["process_kwargs"]` holds the most recent call and + `captured["process_calls"]` holds every call's kwargs. + """ + run_obj = SimpleNamespace(id=uuid4()) + captured["process_calls"] = [] + + async def fake_process(**kwargs): + captured["process_kwargs"] = kwargs + captured["process_calls"].append(kwargs) + # Return one processed scenario per source item. + return [ + SimpleNamespace( + scenario=SimpleNamespace(id=uuid4()), + results={}, + metrics={"score": 1.0}, + has_errors=False, + has_pending=False, + ) + for _ in kwargs["source_items"] + ] + + async def fake_add_scenarios(*, run_id, count, timestamp=None): + return [SimpleNamespace(id=uuid4()) for _ in range(count)] + + # aevaluate queries the GLOBAL aggregate metric at end-of-run; the SDK + # client (aquery_global) asks the API for that single row and returns it. + global_metric = SimpleNamespace( + scenario_id=None, timestamp=None, data={"score": 1.0} + ) + + return [ + patch(f"{MOD}.acreate_run", AsyncMock(return_value=run_obj)), + patch(f"{MOD}.aclose_run", AsyncMock(return_value=run_obj)), + patch(f"{MOD}.aget_url", AsyncMock(return_value="http://x/run")), + patch(f"{MOD}.aadd_scenarios", fake_add_scenarios), + patch(f"{MOD}.apopulate_slice", AsyncMock(return_value=[])), + patch(f"{MOD}.arefresh", AsyncMock(return_value=None)), + patch(f"{MOD}.aedit_scenario", AsyncMock(return_value=None)), + patch(f"{MOD}.aquery_metrics", AsyncMock(return_value=global_metric)), + patch(f"{MOD}.aretrieve_testset", AsyncMock(return_value=testset_revision)), + patch( + f"{MOD}.aretrieve_application", + AsyncMock(return_value=application_revision), + ), + patch( + f"{MOD}.aretrieve_evaluator", + AsyncMock(return_value=evaluator_revision), + ), + patch(f"{MOD}.process_sources", fake_process), + ] + + +def _evaluate(*, evaluators, repeats=None, testcases=None): + captured = {} + tsr = _testset_revision( + testcases=testcases or [_testcase({"q": "a"}), _testcase({"q": "b"})] + ) + appr = _application_revision() + evr = _evaluator_revision() + + patches = _patch_io( + testset_revision=tsr, + application_revision=appr, + evaluator_revision=evr, + captured=captured, + ) + for p in patches: + p.start() + try: + result = run( + ev.aevaluate( + testsets={str(tsr.id): "custom"}, + applications={str(appr.id): "custom"}, + evaluators=evaluators, + repeats=repeats, + ) + ) + finally: + for p in patches: + p.stop() + return result, captured, (tsr, appr, evr) + + +class TestAevaluateOrchestration: + def test_builds_step_graph_and_returns_run_scenarios_metrics(self): + _, captured, (tsr, appr, evr) = _evaluate(evaluators={str(uuid4()): "auto"}) + + steps = captured["process_kwargs"]["steps"] + kinds = [(s.key, s.type, s.origin) for s in steps] + + # input (testset) + invocation (application) + annotation (evaluator) + types = [t for _, t, _ in kinds] + assert "input" in types + assert "invocation" in types + assert "annotation" in types + + # input step key derives from the testset slug, invocation from app slug + assert any(k.startswith("testset-") and t == "input" for k, t, _ in kinds) + assert any( + k == f"application-{appr.slug}" and t == "invocation" for k, t, _ in kinds + ) + assert any( + k == f"evaluator-{evr.slug}" and t == "annotation" for k, t, _ in kinds + ) + + def test_auto_evaluator_gets_local_runner(self): + _, captured, (_tsr, appr, evr) = _evaluate(evaluators={str(uuid4()): "auto"}) + runners = captured["process_kwargs"]["runners"] + + # one shared SDKWorkflowRunner is wired into both runnable step kinds + # (it branches on step type internally). + assert isinstance(runners[f"application-{appr.slug}"], SDKWorkflowRunner) + # auto evaluator -> local runner is wired + assert isinstance(runners[f"evaluator-{evr.slug}"], SDKWorkflowRunner) + + def test_human_evaluator_gets_no_runner(self): + _, captured, (_tsr, _appr, evr) = _evaluate(evaluators={str(uuid4()): "human"}) + runners = captured["process_kwargs"]["runners"] + + # human annotation has no auto runner (it is filled in by a person) + assert f"evaluator-{evr.slug}" not in runners + # but the annotation step is still in the graph + steps = captured["process_kwargs"]["steps"] + assert any( + s.key == f"evaluator-{evr.slug}" and s.origin == "human" for s in steps + ) + + def test_forwards_repeats_and_source_items(self): + tcs = [_testcase({"q": "a"}), _testcase({"q": "b"}), _testcase({"q": "c"})] + _, captured, _ = _evaluate( + evaluators={str(uuid4()): "auto"}, repeats=4, testcases=tcs + ) + calls = captured["process_calls"] + # single-slice: process_sources is called ONCE for the testset, carrying + # all three source items, with repeats forwarded. + assert len(calls) == 1 + assert calls[0]["repeats"] == 4 + assert len(calls[0]["source_items"]) == 3 + + def test_assembles_result_payload(self): + result, captured, _ = _evaluate(evaluators={str(uuid4()): "auto"}) + assert set(result.keys()) == {"run", "scenarios", "metrics"} + # one processed scenario per source item (2 default testcases) + assert len(result["scenarios"]) == 2 + # metrics is the GLOBAL aggregate row queried at end-of-run (not the + # per-scenario one); its data carries the run's headline score. + assert result["metrics"].data == {"score": 1.0} diff --git a/sdks/python/oss/tests/pytest/unit/test_auto_ai_critique_v0_runtime.py b/sdks/python/oss/tests/pytest/unit/test_auto_ai_critique_v0_runtime.py index f3d2affe24..7b23bf32b0 100644 --- a/sdks/python/oss/tests/pytest/unit/test_auto_ai_critique_v0_runtime.py +++ b/sdks/python/oss/tests/pytest/unit/test_auto_ai_critique_v0_runtime.py @@ -44,7 +44,23 @@ def _noop_aws_credentials(_provider_settings): @pytest.fixture -def mocked_llm_call(): +def initialized_sdk(): + """Initialize the SDK so the litellm callback isn't configured pre-init. + + The handlers lazily configure litellm on first use, which calls + ``litellm_handler()`` — and that emits a ``RuntimeWarning`` when ``ag.tracing`` + is unset. ``ag.init()`` runs offline (no network) and sets ``ag.tracing``, + keeping these tests warning-free. Scoped to the LLM-touching tests via + ``mocked_llm_call`` rather than applied autouse. + """ + import agenta as ag + + ag.init() + yield + + +@pytest.fixture +def mocked_llm_call(initialized_sdk): """Mock the LLM call boundary without touching the network.""" captured: dict = {} diff --git a/sdks/python/oss/tests/pytest/unit/test_evaluate_specs.py b/sdks/python/oss/tests/pytest/unit/test_evaluate_specs.py new file mode 100644 index 0000000000..449b9bfb60 --- /dev/null +++ b/sdks/python/oss/tests/pytest/unit/test_evaluate_specs.py @@ -0,0 +1,190 @@ +""" +Unit tests for the evaluate() spec parsing/normalization layer. + +These cover the pure, API-free internals of `agenta.sdk.evaluations.preview.evaluate`: + + - `_parse_evaluate_kwargs`: merges kwargs with `specs`, applies kwargs-win + precedence, carries `repeats`, and validates the three required step groups. + - `_normalize_step_id`: coerces UUID-compatible ids to canonical str, drops + invalid ones. + - `_normalize_target_steps`: builds {step_id: origin}, rejecting invalid ids + and non-{custom,human,auto} origins. + +`_parse_evaluate_kwargs` is async but does no I/O, so it runs via asyncio.run. +""" + +import asyncio +from uuid import uuid4 + +import pytest + +from agenta.sdk.evaluations.preview.evaluate import ( + EvaluateSpecs, + _normalize_step_id, + _normalize_target_steps, + _parse_evaluate_kwargs, +) + + +def run(coro): + return asyncio.run(coro) + + +def _str_keys(steps): + # Target = Dict[UUID, Origin] coerces string keys to UUID objects; compare by + # canonical string so the assertions are independent of key representation. + return {str(k): v for k, v in (steps or {}).items()} + + +# --- _normalize_step_id ---------------------------------------------------- + + +def test_normalize_step_id_passes_through_uuid_object(): + u = uuid4() + assert _normalize_step_id(u) == str(u) + + +def test_normalize_step_id_canonicalizes_uuid_string(): + u = uuid4() + assert _normalize_step_id(str(u)) == str(u) + + +def test_normalize_step_id_returns_none_for_none(): + assert _normalize_step_id(None) is None + + +def test_normalize_step_id_returns_none_for_invalid(): + assert _normalize_step_id("not-a-uuid") is None + + +# --- _normalize_target_steps ---------------------------------------------- + + +def test_normalize_target_steps_keeps_valid_entries(): + a, b = uuid4(), uuid4() + steps = {str(a): "human", str(b): "auto"} + out = _normalize_target_steps(steps=steps, step_name="evaluators") + assert out == {str(a): "human", str(b): "auto"} + + +def test_normalize_target_steps_rejects_invalid_id(): + with pytest.raises(ValueError, match="invalid"): + _normalize_target_steps(steps={"not-a-uuid": "auto"}, step_name="evaluators") + + +def test_normalize_target_steps_rejects_invalid_origin(): + with pytest.raises(ValueError, match="invalid"): + _normalize_target_steps(steps={str(uuid4()): "robot"}, step_name="evaluators") + + +def test_normalize_target_steps_rejects_empty(): + with pytest.raises(ValueError, match="missing or invalid"): + _normalize_target_steps(steps={}, step_name="evaluators") + + +def test_normalize_target_steps_rejects_non_dict(): + with pytest.raises(ValueError, match="missing or invalid"): + _normalize_target_steps(steps=["not", "a", "dict"], step_name="evaluators") + + +# --- _parse_evaluate_kwargs ------------------------------------------------ + + +def _ids(): + return str(uuid4()), str(uuid4()), str(uuid4()) + + +def test_parse_kwargs_builds_data_from_direct_kwargs(): + ts, app, ev = _ids() + data = run( + _parse_evaluate_kwargs( + testsets={ts: "custom"}, + applications={app: "custom"}, + evaluators={ev: "auto"}, + repeats=3, + ) + ) + assert _str_keys(data.testset_steps) == {ts: "custom"} + assert _str_keys(data.application_steps) == {app: "custom"} + assert _str_keys(data.evaluator_steps) == {ev: "auto"} + assert data.repeats == 3 + + +def test_parse_kwargs_builds_data_from_specs_dict(): + ts, app, ev = _ids() + data = run( + _parse_evaluate_kwargs( + specs={ + "testsets": {ts: "custom"}, + "applications": {app: "custom"}, + "evaluators": {ev: "human"}, + "repeats": 2, + } + ) + ) + assert _str_keys(data.testset_steps) == {ts: "custom"} + assert _str_keys(data.evaluator_steps) == {ev: "human"} + assert data.repeats == 2 + + +def test_parse_kwargs_accepts_evaluatespecs_instance(): + ts, app, ev = _ids() + data = run( + _parse_evaluate_kwargs( + specs=EvaluateSpecs( + testsets={ts: "custom"}, + applications={app: "custom"}, + evaluators={ev: "auto"}, + ) + ) + ) + assert _str_keys(data.application_steps) == {app: "custom"} + + +def test_parse_kwargs_direct_kwargs_win_over_specs(): + ts, app, ev = _ids() + other_ts = str(uuid4()) + data = run( + _parse_evaluate_kwargs( + testsets={ts: "custom"}, + specs={ + "testsets": {other_ts: "custom"}, + "applications": {app: "custom"}, + "evaluators": {ev: "auto"}, + }, + ) + ) + # direct testsets kwarg wins; applications/evaluators fall back to specs + assert _str_keys(data.testset_steps) == {ts: "custom"} + assert _str_keys(data.application_steps) == {app: "custom"} + + +@pytest.mark.parametrize( + "missing", + ["testsets", "applications", "evaluators"], +) +def test_parse_kwargs_requires_each_step_group(missing): + ts, app, ev = _ids() + kwargs = { + "testsets": {ts: "custom"}, + "applications": {app: "custom"}, + "evaluators": {ev: "auto"}, + } + kwargs.pop(missing) + with pytest.raises(ValueError, match=f"missing {missing}"): + run(_parse_evaluate_kwargs(**kwargs)) + + +def test_parse_kwargs_ignores_non_spec_object(): + ts, app, ev = _ids() + # A specs value that is neither dict nor EvaluateSpecs is dropped, so the + # direct kwargs must still satisfy the required groups. + data = run( + _parse_evaluate_kwargs( + testsets={ts: "custom"}, + applications={app: "custom"}, + evaluators={ev: "auto"}, + specs="not-a-spec", + ) + ) + assert _str_keys(data.testset_steps) == {ts: "custom"} diff --git a/sdks/python/oss/tests/pytest/unit/test_evaluations_runtime.py b/sdks/python/oss/tests/pytest/unit/test_evaluations_runtime.py new file mode 100644 index 0000000000..a7f717c400 --- /dev/null +++ b/sdks/python/oss/tests/pytest/unit/test_evaluations_runtime.py @@ -0,0 +1,1452 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest + +import agenta.sdk.evaluations.preview.evaluate as preview_evaluate +import agenta.sdk.evaluations.runtime.adapters as runtime_adapters +from agenta.sdk.evaluations.runtime.executor import execute_workflow_batch +from agenta.sdk.evaluations.runtime.models import ( + EvaluationStep, + PlannedCell, + ResolvedSourceItem, + ResultLogRequest, + ScenarioBinding, +) +from agenta.sdk.evaluations.runtime.planner import EvaluationPlanner +from agenta.sdk.evaluations.runtime.processor import ( + process_sources, +) +from agenta.sdk.evaluations.runtime.topology import classify_steps_topology +from agenta.sdk.evaluations.runtime.models import WorkflowExecutionResult +from agenta.sdk.models.evaluations import EvaluationStatus + + +def test_sdk_runtime_planner_matches_split_repeat_rules(): + run_id = uuid4() + scenario_id = uuid4() + plan = EvaluationPlanner().plan( + run_id=run_id, + scenario_id=scenario_id, + source=ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + ), + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="application-main", type="invocation"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + EvaluationStep(key="evaluator-human", type="annotation", origin="human"), + ], + repeats=3, + is_split=False, + ) + + assert [ + cell.repeat_idx for cell in plan.cells if cell.step_key == "application-main" + ] == [0] + assert [ + cell.status for cell in plan.cells if cell.step_key == "evaluator-human" + ] == [ + EvaluationStatus.PENDING, + EvaluationStatus.PENDING, + EvaluationStatus.PENDING, + ] + assert {(cell.step_key, cell.repeat_idx) for cell in plan.executable_cells} == { + ("application-main", 0), + ("evaluator-auto", 0), + ("evaluator-auto", 1), + ("evaluator-auto", 2), + } + + +def test_sdk_runtime_planner_handles_multiple_scenario_bindings(): + run_id = uuid4() + first_scenario_id = uuid4() + second_scenario_id = uuid4() + + plan = EvaluationPlanner().plan_bindings( + run_id=run_id, + bindings=[ + ScenarioBinding( + scenario_id=first_scenario_id, + source=ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + ), + ), + ScenarioBinding( + scenario_id=second_scenario_id, + source=ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + ), + ), + ], + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="application-main", type="invocation"), + ], + repeats=2, + ) + + assert [cell.scenario_id for cell in plan.cells] == [ + first_scenario_id, + first_scenario_id, + first_scenario_id, + first_scenario_id, + second_scenario_id, + second_scenario_id, + second_scenario_id, + second_scenario_id, + ] + assert [ + (cell.step_key, cell.repeat_idx) + for cell in plan.cells + if cell.scenario_id == first_scenario_id + ] == [ + ("testset-main", 0), + ("testset-main", 1), + ("application-main", 0), + ("application-main", 1), + ] + + +def test_sdk_runtime_topology_classifier_matches_batch_inference_shape(): + decision = classify_steps_topology( + steps=[ + EvaluationStep( + key="testset-main", + type="input", + references={"testset_revision": {"id": str(uuid4())}}, + ), + EvaluationStep( + key="application-main", + type="invocation", + references={"application_revision": {"id": str(uuid4())}}, + ), + ], + ) + + assert decision.status == "supported" + assert decision.dispatch == "batch_invocation" + + +def test_sdk_runtime_topology_classifier_distinguishes_direct_testcases_from_testsets(): + decision = classify_steps_topology( + steps=[ + EvaluationStep(key="testcases", type="input"), + EvaluationStep(key="evaluator-human", type="annotation", origin="human"), + ], + has_testcases=True, + has_evaluators=True, + ) + + assert decision.status == "supported" + assert decision.dispatch == "queue_testcases" + + +def test_sdk_runtime_topology_classifier_keeps_deferred_query_to_application_shape(): + decision = classify_steps_topology( + steps=[ + EvaluationStep( + key="query-main", + type="input", + references={"query_revision": {"id": str(uuid4())}}, + ), + EvaluationStep( + key="application-main", + type="invocation", + references={"application_revision": {"id": str(uuid4())}}, + ), + ], + ) + + assert decision.status == "potential" + + +@pytest.mark.asyncio +async def test_sdk_workflow_batch_falls_back_to_single_execute(): + calls = [] + + class SingleRunner: + async def execute(self, request): + calls.append(request.cell.repeat_idx) + return WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id=f"trace-{request.cell.repeat_idx}", + ) + + requests = [ + SimpleNamespace( + cell=SimpleNamespace(repeat_idx=0), + ), + SimpleNamespace( + cell=SimpleNamespace(repeat_idx=1), + ), + ] + + results = await execute_workflow_batch( + runner=SingleRunner(), + requests=requests, + ) + + assert calls == [0, 1] + assert [result.trace_id for result in results] == ["trace-0", "trace-1"] + + +@pytest.mark.asyncio +async def test_sdk_source_slice_batches_runnable_cells(): + run_id = uuid4() + scenario_id = uuid4() + logged = [] + + class BatchRunner: + def __init__(self): + self.requests = [] + + async def execute_batch(self, requests, semaphore=None): + self.requests.append(requests) + return [ + WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id=f"trace-{request.cell.repeat_idx}", + span_id=f"span-{request.cell.repeat_idx}", + ) + for request in requests + ] + + class Logger: + async def set(self, request): + logged.append((request.cell.step_key, request.cell.repeat_idx)) + return SimpleNamespace(id=uuid4()) + + async def create_scenario(run_id): + return SimpleNamespace(id=scenario_id) + + async def refresh_metrics(run_id, scenario_id): + return SimpleNamespace(id=uuid4()) + + runner = BatchRunner() + + processed = await process_sources( + run_id=run_id, + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"prompt": "hello"}, + ) + ], + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=3, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={"evaluator-auto": runner}, + revisions={"evaluator-auto": {"id": "revision"}}, + ) + + assert len(runner.requests) == 1 + assert [request.cell.repeat_idx for request in runner.requests[0]] == [0, 1, 2] + assert logged == [ + ("testset-main", 0), + ("testset-main", 1), + ("testset-main", 2), + ("evaluator-auto", 0), + ("evaluator-auto", 1), + ("evaluator-auto", 2), + ] + + # ProcessedScenario.results retains EVERY repeat per step (keyed by + # repeat_idx), not just the last one — repeats>1 must not collapse to one + # result per step. Regression guard for UEL-016. + assert len(processed) == 1 + results = processed[0].results + assert set(results["testset-main"].keys()) == {0, 1, 2} + assert set(results["evaluator-auto"].keys()) == {0, 1, 2} + + +@pytest.mark.asyncio +async def test_sdk_source_slice_isolates_one_scenario_failure(): + """One scenario's create_scenario failure must not abort the siblings. + + Regression guard for UEL-023: the per-scenario seams were unguarded under + asyncio.gather(return_exceptions=False), so a single failing scenario crashed + the whole slice. Now each scenario is isolated. + """ + run_id = uuid4() + good_scenario_id = uuid4() + + class Runner: + async def execute_batch(self, requests, semaphore=None): + return [ + WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id=f"trace-{req.cell.repeat_idx}", + ) + for req in requests + ] + + class Logger: + async def set(self, request): + return SimpleNamespace(id=uuid4()) + + calls = {"n": 0} + + async def create_scenario(run_id): + # First scenario blows up; the second succeeds. + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("boom: scenario factory failed") + return SimpleNamespace(id=good_scenario_id) + + async def refresh_metrics(run_id, scenario_id): + return SimpleNamespace(id=uuid4()) + + source_items = [ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"prompt": "a"}, + ), + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"prompt": "b"}, + ), + ] + + processed = await process_sources( + run_id=run_id, + source_items=source_items, + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=1, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={"evaluator-auto": Runner()}, + revisions={"evaluator-auto": {"id": "revision"}}, + ) + + # The failed scenario is dropped (not raised); the healthy one still ran. + assert len(processed) == 1 + assert processed[0].scenario.id == good_scenario_id + + +@pytest.mark.asyncio +async def test_sdk_source_slice_marks_short_runner_batch_as_error(): + run_id = uuid4() + scenario_id = uuid4() + logged = [] + + class ShortRunner: + async def execute_batch(self, requests, semaphore=None): + return [ + WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id="trace-0", + span_id="span-0", + ) + ] + + class Logger: + async def set(self, request): + logged.append(request) + return SimpleNamespace(id=uuid4()) + + async def create_scenario(run_id): + return SimpleNamespace(id=scenario_id) + + async def refresh_metrics(run_id, scenario_id): + return SimpleNamespace(id=uuid4()) + + processed = await process_sources( + run_id=run_id, + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"prompt": "hello"}, + ) + ], + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=2, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={"evaluator-auto": ShortRunner()}, + revisions={"evaluator-auto": {"id": "revision"}}, + ) + + assert processed[0].has_errors is True + failed_log = logged[-1] + assert failed_log.cell.step_key == "evaluator-auto" + assert failed_log.cell.repeat_idx == 1 + assert failed_log.cell.status == EvaluationStatus.FAILURE + assert failed_log.error == { + "message": ( + "Runner for evaluator-auto returned 1 execution(s) for 2 planned cell(s)." + ) + } + + +@pytest.mark.asyncio +async def test_sdk_source_slice_handles_over_count_runner_batch(): + # Runner returns more executions than planned cells: the planned cells are + # logged from the first executions, the extras have no cell and are dropped + # (with a structured warning), and the scenario is flagged as having errors. + run_id = uuid4() + scenario_id = uuid4() + logged = [] + + class LongRunner: + async def execute_batch(self, requests, semaphore=None): + # 3 executions for 2 planned cells (repeats=2). + return [ + WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id=f"trace-{i}", + span_id=f"span-{i}", + ) + for i in range(3) + ] + + class Logger: + async def set(self, request): + logged.append(request) + return SimpleNamespace(id=uuid4()) + + async def create_scenario(run_id): + return SimpleNamespace(id=scenario_id) + + async def refresh_metrics(run_id, scenario_id): + return SimpleNamespace(id=uuid4()) + + processed = await process_sources( + run_id=run_id, + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"prompt": "hello"}, + ) + ], + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=2, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={"evaluator-auto": LongRunner()}, + revisions={"evaluator-auto": {"id": "revision"}}, + ) + + assert processed[0].has_errors is True + # Both planned evaluator cells were logged from the first two executions; the + # third (extra) execution is dropped, not logged as a cell. + evaluator_logs = [ + entry for entry in logged if entry.cell.step_key == "evaluator-auto" + ] + assert len(evaluator_logs) == 2 + assert {entry.cell.repeat_idx for entry in evaluator_logs} == {0, 1} + + +@pytest.mark.asyncio +async def test_sdk_source_slice_marks_missing_runner_as_error(): + run_id = uuid4() + scenario_id = uuid4() + logged = [] + + class Logger: + async def set(self, request): + logged.append(request) + return SimpleNamespace(id=uuid4()) + + async def create_scenario(run_id): + return SimpleNamespace(id=scenario_id) + + async def refresh_metrics(run_id, scenario_id): + return SimpleNamespace(id=uuid4()) + + processed = await process_sources( + run_id=run_id, + source_items=[ + ResolvedSourceItem( + kind="trace", + step_key="query-main", + trace_id="query-trace", + ) + ], + steps=[ + EvaluationStep(key="query-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=1, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={}, + revisions={}, + ) + + assert processed[0].has_errors is True + assert [(item.cell.step_key, item.error) for item in logged] == [ + ("query-main", None), + ( + "evaluator-auto", + {"message": "Missing runner or revision for evaluator-auto"}, + ), + ] + + +@pytest.mark.asyncio +async def test_sdk_source_slice_can_defer_manual_results_without_metric_refresh(): + run_id = uuid4() + scenario_id = uuid4() + logged = [] + + class Logger: + async def set(self, request): + logged.append(request.cell.step_key) + return SimpleNamespace(id=uuid4()) + + refresh_metrics = pytest.fail + + async def create_scenario(run_id): + return SimpleNamespace(id=scenario_id) + + processed = await process_sources( + run_id=run_id, + source_items=[ + ResolvedSourceItem( + kind="trace", + step_key="query-main", + trace_id="query-trace", + ) + ], + steps=[ + EvaluationStep(key="query-main", type="input"), + EvaluationStep(key="evaluator-human", type="annotation", origin="human"), + ], + repeats=1, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={}, + revisions={}, + log_pending=False, + refresh_metrics_without_auto_results=False, + ) + + assert processed[0].has_pending is True + assert processed[0].auto_results_created is False + assert logged == ["query-main"] + + +@pytest.mark.asyncio +async def test_sdk_source_slice_links_evaluators_to_application_traces(): + run_id = uuid4() + scenario_id = uuid4() + evaluator_requests = [] + + class ApplicationRunner: + async def execute_batch(self, requests, semaphore=None): + return [ + WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id="app-trace", + span_id="app-span", + outputs={"answer": "hello"}, + trace={ + "trace_id": "app-trace", + "spans": { + "root": { + "span_id": "app-span", + "attributes": { + "ag": { + "data": { + "outputs": {"answer": "hello"}, + } + } + }, + } + }, + }, + ) + for _ in requests + ] + + class EvaluatorRunner: + async def execute_batch(self, requests, semaphore=None): + evaluator_requests.extend(requests) + return [ + WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id=f"eval-trace-{request.cell.repeat_idx}", + ) + for request in requests + ] + + class Logger: + async def set(self, request): + return SimpleNamespace(id=uuid4()) + + async def create_scenario(run_id): + return SimpleNamespace(id=scenario_id) + + async def refresh_metrics(run_id, scenario_id): + return SimpleNamespace(id=uuid4()) + + await process_sources( + run_id=run_id, + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"prompt": "hello"}, + ) + ], + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="application-main", type="invocation"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=2, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={ + "application-main": ApplicationRunner(), + "evaluator-auto": EvaluatorRunner(), + }, + revisions={ + "application-main": {"id": "application-revision"}, + "evaluator-auto": {"id": "evaluator-revision"}, + }, + is_split=False, + ) + + assert [request.cell.repeat_idx for request in evaluator_requests] == [0, 1] + assert [request.links for request in evaluator_requests] == [ + {"invocation": {"trace_id": "app-trace", "span_id": "app-span"}}, + {"invocation": {"trace_id": "app-trace", "span_id": "app-span"}}, + ] + assert [request.upstream_outputs for request in evaluator_requests] == [ + {"answer": "hello"}, + {"answer": "hello"}, + ] + + +@pytest.mark.asyncio +async def test_sdk_result_setter_writes_populate_ready_cell_live(): + """SDKResultSetter writes each cell LIVE via populate as the engine produces it. + + It shapes the cell into a populate-ready dict — preserving repeat_idx and the + cell's bound trace/testcase — writes it immediately (one populate call per + cell), and returns that dict as the engine's remembered value. + """ + run_id = uuid4() + scenario_id = uuid4() + testcase_id = uuid4() + cell = PlannedCell( + run_id=run_id, + scenario_id=scenario_id, + step_key="evaluator-auto", + step_type="annotation", + origin="auto", + repeat_idx=2, + status=EvaluationStatus.SUCCESS, + testcase_id=testcase_id, + ) + + populate_calls = [] + + async def fake_populate(*, results): + populate_calls.append(results) + return results + + setter = runtime_adapters.SDKResultSetter(populate=fake_populate) + returned = await setter.set( + ResultLogRequest( + cell=cell, + trace_id="trace-repeat", + ) + ) + + expected = { + "run_id": str(run_id), + "scenario_id": str(scenario_id), + "step_key": "evaluator-auto", + "repeat_idx": 2, + "status": "success", + "trace_id": "trace-repeat", + "testcase_id": str(testcase_id), + "error": None, + } + assert returned == expected + # written live: one populate call carrying exactly this cell. + assert populate_calls == [[expected]] + + +@pytest.mark.asyncio +async def test_sdk_preview_evaluate_logs_repeat_aware_results(monkeypatch): + run_id = uuid4() + scenario_id = uuid4() + testset_id = uuid4() + testset_variant_id = uuid4() + testset_revision_id = uuid4() + application_revision_id = uuid4() + evaluator_revision_id = uuid4() + testcase_id = uuid4() + + testcase = SimpleNamespace( + id=testcase_id, + data={"prompt": "hello"}, + model_dump=lambda **kwargs: { + "id": str(testcase_id), + "data": {"prompt": "hello"}, + }, + ) + testset_revision = SimpleNamespace( + id=testset_revision_id, + testset_id=testset_id, + testset_variant_id=testset_variant_id, + slug="main", + version="1", + data=SimpleNamespace(testcases=[testcase]), + ) + application_revision = SimpleNamespace( + id=application_revision_id, + application_id=uuid4(), + application_variant_id=uuid4(), + slug="app", + version="1", + data=SimpleNamespace(parameters={"temperature": 0}), + model_dump=lambda **kwargs: {"id": str(application_revision_id)}, + ) + evaluator_revision = SimpleNamespace( + id=evaluator_revision_id, + evaluator_id=uuid4(), + evaluator_variant_id=uuid4(), + slug="eval", + version="1", + data=SimpleNamespace(parameters={"threshold": 1}), + model_dump=lambda **kwargs: {"id": str(evaluator_revision_id)}, + ) + + async def fake_retrieve_testset(**kwargs): + return testset_revision + + async def fake_retrieve_application(**kwargs): + return application_revision + + async def fake_retrieve_evaluator(**kwargs): + return evaluator_revision + + async def fake_create_run(**kwargs): + return SimpleNamespace(id=run_id) + + async def fake_add_scenarios(*, run_id, count, timestamp=None): + # bulk-mint: one scenario per testcase, in order. + return [SimpleNamespace(id=scenario_id) for _ in range(count)] + + populated_cells = [] + + async def fake_populate_slice(*, results): + # the single bulk populate the SDK does after local execution. + populated_cells.extend(results) + return [SimpleNamespace(id=uuid4()) for _ in results] + + refresh_calls = [] + edit_status_calls = [] + + async def fake_refresh(run_id, scenario_id=None): + # arefresh(run_id, scenario_id): per-scenario (variational) when + # scenario_id is set, global when it is None. + refresh_calls.append((run_id, scenario_id)) + + async def fake_edit_scenario(*, scenario_id, status, tags=None, meta=None): + edit_status_calls.append((scenario_id, status)) + + async def fake_invoke_application(**kwargs): + return SimpleNamespace( + data=SimpleNamespace(), + trace_id="app-trace", + span_id="app-span", + ) + + evaluator_trace_ids = iter(["eval-trace-0", "eval-trace-1"]) + + async def fake_invoke_evaluator(**kwargs): + return SimpleNamespace( + data=SimpleNamespace(), + trace_id=next(evaluator_trace_ids), + span_id="eval-span", + ) + + async def fake_afetch_trace(trace_id, **kwargs): + return { + "spans": { + "root": { + "attributes": { + "ag": { + "data": { + "inputs": {"prompt": "hello"}, + "outputs": {"answer": trace_id}, + } + } + } + } + } + } + + async def fake_close_run(**kwargs): + return SimpleNamespace(id=run_id) + + async def fake_get_url(**kwargs): + return "" + + monkeypatch.setattr(preview_evaluate, "aretrieve_testset", fake_retrieve_testset) + monkeypatch.setattr( + preview_evaluate, "aretrieve_application", fake_retrieve_application + ) + monkeypatch.setattr( + preview_evaluate, "aretrieve_evaluator", fake_retrieve_evaluator + ) + monkeypatch.setattr(preview_evaluate, "acreate_run", fake_create_run) + # the SDK mirrors the API: bulk add_scenarios -> ONE slice over all scenarios + # with live per-cell populate, inline per-scenario + global metric refresh, + # and per-scenario status writes. + monkeypatch.setattr(preview_evaluate, "aadd_scenarios", fake_add_scenarios) + monkeypatch.setattr(preview_evaluate, "apopulate_slice", fake_populate_slice) + monkeypatch.setattr(preview_evaluate, "arefresh", fake_refresh) + monkeypatch.setattr(preview_evaluate, "aedit_scenario", fake_edit_scenario) + monkeypatch.setattr( + preview_evaluate, + "aquery_metrics", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr(runtime_adapters, "invoke_application", fake_invoke_application) + monkeypatch.setattr(runtime_adapters, "invoke_evaluator", fake_invoke_evaluator) + monkeypatch.setattr(preview_evaluate, "afetch_trace", fake_afetch_trace) + monkeypatch.setattr(preview_evaluate, "aclose_run", fake_close_run) + monkeypatch.setattr(preview_evaluate, "aget_url", fake_get_url) + + result = await preview_evaluate.aevaluate( + testsets={testset_revision_id: "custom"}, + applications={application_revision_id: "custom"}, + evaluators={evaluator_revision_id: "auto"}, + repeats=2, + ) + + assert result["run"].id == run_id + # cells written live (one cell per populate call), repeat-aware, in plan + # order. populated_cells accumulates them across the per-cell writes. + assert [(cell["step_key"], cell["repeat_idx"]) for cell in populated_cells] == [ + ("testset-main", 0), + ("testset-main", 1), + ("application-app", 0), + ("evaluator-eval", 0), + ("evaluator-eval", 1), + ] + assert [ + cell["trace_id"] + for cell in populated_cells + if cell["step_key"] == "evaluator-eval" + ] == ["eval-trace-0", "eval-trace-1"] + # metrics refreshed inline for the scenario (variational) AND once globally, + # mirroring the API engine. + assert (run_id, scenario_id) in refresh_calls + assert (run_id, None) in refresh_calls + + +@pytest.mark.asyncio +async def test_sdk_preview_evaluate_processes_all_scenarios_in_one_slice(monkeypatch): + """Two testcases -> ONE slice over both scenarios (the design's + `process_slice(all scenarios)`). + + Locks in the single-slice boundary: one bulk populate carrying every + scenario's cells, one refresh scoped to all scenario ids, and a status write + per scenario. An outer per-scenario loop (one populate/refresh each) would + leave the engine's concurrency inert, so this guards that regression. + """ + run_id = uuid4() + scenario_a, scenario_b = uuid4(), uuid4() + testset_revision_id = uuid4() + application_revision_id = uuid4() + evaluator_revision_id = uuid4() + tc_a, tc_b = uuid4(), uuid4() + + def _testcase(tcid, prompt): + return SimpleNamespace( + id=tcid, + data={"prompt": prompt}, + model_dump=lambda **kwargs: {"id": str(tcid), "data": {"prompt": prompt}}, + ) + + testset_revision = SimpleNamespace( + id=testset_revision_id, + testset_id=uuid4(), + testset_variant_id=uuid4(), + slug="main", + version="1", + data=SimpleNamespace(testcases=[_testcase(tc_a, "a"), _testcase(tc_b, "b")]), + ) + application_revision = SimpleNamespace( + id=application_revision_id, + application_id=uuid4(), + application_variant_id=uuid4(), + slug="app", + version="1", + data=SimpleNamespace(parameters={}), + model_dump=lambda **kwargs: {"id": str(application_revision_id)}, + ) + evaluator_revision = SimpleNamespace( + id=evaluator_revision_id, + evaluator_id=uuid4(), + evaluator_variant_id=uuid4(), + slug="eval", + version="1", + data=SimpleNamespace(parameters={}), + model_dump=lambda **kwargs: {"id": str(evaluator_revision_id)}, + ) + + # one scenario id per testcase, in order. + minted_ids = iter([scenario_a, scenario_b]) + + async def fake_add_scenarios(*, run_id, count, timestamp=None): + return [SimpleNamespace(id=next(minted_ids)) for _ in range(count)] + + # cells are written live (one cell per populate call); collect every cell. + populated_cells = [] + + async def fake_populate_slice(*, results): + populated_cells.extend(results) + return [SimpleNamespace(id=uuid4()) for _ in results] + + refresh_calls = [] + edit_status_calls = [] + + async def fake_refresh(run_id, scenario_id=None): + refresh_calls.append((run_id, scenario_id)) + + async def fake_edit_scenario(*, scenario_id, status, tags=None, meta=None): + edit_status_calls.append((scenario_id, status)) + + async def fake_invoke_application(**kwargs): + return SimpleNamespace( + data=SimpleNamespace(), trace_id="app-trace", span_id="app-span" + ) + + async def fake_invoke_evaluator(**kwargs): + return SimpleNamespace( + data=SimpleNamespace(), trace_id="eval-trace", span_id="eval-span" + ) + + async def fake_afetch_trace(trace_id, **kwargs): + return {"spans": {"root": {"attributes": {"ag": {"data": {}}}}}} + + monkeypatch.setattr( + preview_evaluate, "aretrieve_testset", lambda **k: _async(testset_revision) + ) + monkeypatch.setattr( + preview_evaluate, + "aretrieve_application", + lambda **k: _async(application_revision), + ) + monkeypatch.setattr( + preview_evaluate, "aretrieve_evaluator", lambda **k: _async(evaluator_revision) + ) + monkeypatch.setattr( + preview_evaluate, "acreate_run", lambda **k: _async(SimpleNamespace(id=run_id)) + ) + monkeypatch.setattr(preview_evaluate, "aadd_scenarios", fake_add_scenarios) + monkeypatch.setattr(preview_evaluate, "apopulate_slice", fake_populate_slice) + monkeypatch.setattr(preview_evaluate, "arefresh", fake_refresh) + monkeypatch.setattr(preview_evaluate, "aedit_scenario", fake_edit_scenario) + monkeypatch.setattr( + preview_evaluate, + "aquery_metrics", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr(runtime_adapters, "invoke_application", fake_invoke_application) + monkeypatch.setattr(runtime_adapters, "invoke_evaluator", fake_invoke_evaluator) + monkeypatch.setattr(preview_evaluate, "afetch_trace", fake_afetch_trace) + monkeypatch.setattr( + preview_evaluate, "aclose_run", lambda **k: _async(SimpleNamespace(id=run_id)) + ) + monkeypatch.setattr(preview_evaluate, "aget_url", lambda **k: _async("")) + + await preview_evaluate.aevaluate( + testsets={testset_revision_id: "custom"}, + applications={application_revision_id: "custom"}, + evaluators={evaluator_revision_id: "auto"}, + repeats=1, + ) + + # TWO scenarios processed in ONE slice: every cell written live, carrying + # both scenarios' ids. + assert {c["scenario_id"] for c in populated_cells} == { + str(scenario_a), + str(scenario_b), + } + # metrics refreshed inline per scenario (variational) AND once globally, + # mirroring the API engine. + assert (run_id, scenario_a) in refresh_calls + assert (run_id, scenario_b) in refresh_calls + assert (run_id, None) in refresh_calls + # each scenario got its status written. + assert {sid for sid, _status in edit_status_calls} == {scenario_a, scenario_b} + + +async def _async(value): + return value + + +@pytest.mark.asyncio +async def test_sdk_workflow_runner_execute_batch_is_concurrent_bounded(): + """SDKWorkflowRunner.execute_batch runs concurrently, bounded by the + semaphore — same shape as APIWorkflowRunner (not the old sequential loop).""" + import asyncio + + runner = runtime_adapters.SDKWorkflowRunner() + + in_flight = 0 + peak = 0 + + async def fake_execute(request): + nonlocal in_flight, peak + in_flight += 1 + peak = max(peak, in_flight) + await asyncio.sleep(0) + in_flight -= 1 + return WorkflowExecutionResult(status=EvaluationStatus.SUCCESS, trace_id="t") + + runner.execute = fake_execute # type: ignore[method-assign] + + requests = [SimpleNamespace(idx=i) for i in range(6)] + semaphore = asyncio.Semaphore(2) + + results = await runner.execute_batch(requests, semaphore=semaphore) + + assert len(results) == 6 + assert peak <= 2 # honored the semaphore + + +# --------------------------------------------------------------------------- +# Concurrency and retry +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_sdk_source_slice_runs_scenarios_concurrently_up_to_batch_size(): + """batch_size=2 with 4 scenarios: at most 2 invoke_workflow calls in flight at once.""" + import asyncio + + run_id = uuid4() + in_flight = 0 + peak = 0 + + class ConcurrentRunner: + async def execute_batch(self, requests, semaphore=None): + results = [] + for request in requests: + + async def _one(req): + nonlocal in_flight, peak + in_flight += 1 + peak = max(peak, in_flight) + await asyncio.sleep(0) + in_flight -= 1 + return WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id=f"trace-{req.cell.repeat_idx}", + ) + + if semaphore is not None: + async with semaphore: + results.append(await _one(request)) + else: + results.append(await _one(request)) + return results + + class Logger: + async def set(self, request): + return SimpleNamespace(id=uuid4()) + + scenarios_created = [] + + async def create_scenario(run_id): + sid = uuid4() + scenarios_created.append(sid) + return SimpleNamespace(id=sid) + + async def refresh_metrics(run_id, scenario_id): + return None + + source_items = [ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"x": str(i)}, + ) + for i in range(4) + ] + + await process_sources( + run_id=run_id, + source_items=source_items, + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=1, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={"evaluator-auto": ConcurrentRunner()}, + revisions={"evaluator-auto": {"id": "rev"}}, + batch_size=2, + ) + + assert len(scenarios_created) == 4 + assert peak <= 2 + + +@pytest.mark.asyncio +async def test_sdk_source_slice_semaphore_shared_across_repeats(): + """batch_size=2 with 1 scenario and 4 repeats: peak concurrency stays ≤ 2.""" + import asyncio + + run_id = uuid4() + scenario_id = uuid4() + in_flight = 0 + peak = 0 + + class ConcurrentRunner: + async def execute_batch(self, requests, semaphore=None): + async def _one(req): + nonlocal in_flight, peak + in_flight += 1 + peak = max(peak, in_flight) + await asyncio.sleep(0) + in_flight -= 1 + return WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id=f"trace-{req.cell.repeat_idx}", + ) + + if semaphore is not None: + results = [] + for req in requests: + async with semaphore: + results.append(await _one(req)) + return results + return [await _one(req) for req in requests] + + class Logger: + async def set(self, request): + return SimpleNamespace(id=uuid4()) + + async def create_scenario(run_id): + return SimpleNamespace(id=scenario_id) + + async def refresh_metrics(run_id, scenario_id): + return None + + await process_sources( + run_id=run_id, + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"x": "0"}, + ) + ], + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=4, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={"evaluator-auto": ConcurrentRunner()}, + revisions={"evaluator-auto": {"id": "rev"}}, + batch_size=2, + ) + + assert peak <= 2 + + +@pytest.mark.asyncio +async def test_sdk_source_slice_no_batch_size_runs_all_concurrently(): + """When batch_size=None the semaphore is absent and all scenarios run freely.""" + run_id = uuid4() + scenarios_created = [] + + class Runner: + async def execute_batch(self, requests, semaphore=None): + return [ + WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id="t", + ) + for _ in requests + ] + + class Logger: + async def set(self, request): + return SimpleNamespace(id=uuid4()) + + async def create_scenario(run_id): + sid = uuid4() + scenarios_created.append(sid) + return SimpleNamespace(id=sid) + + async def refresh_metrics(run_id, scenario_id): + return None + + source_items = [ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + ) + for _ in range(5) + ] + + processed = await process_sources( + run_id=run_id, + source_items=source_items, + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=1, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={"evaluator-auto": Runner()}, + revisions={"evaluator-auto": {"id": "rev"}}, + batch_size=None, + ) + + assert len(processed) == 5 + + +@pytest.mark.asyncio +async def test_sdk_source_slice_retries_failed_cells_and_succeeds(): + """max_retries=1: first attempt fails, retry succeeds; result is success.""" + run_id = uuid4() + scenario_id = uuid4() + call_count = 0 + + class FlakyRunner: + async def execute_batch(self, requests, semaphore=None): + nonlocal call_count + call_count += 1 + if call_count == 1: + return [ + WorkflowExecutionResult( + status=EvaluationStatus.FAILURE, + error={"message": "transient"}, + ) + for _ in requests + ] + return [ + WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id="recovered", + ) + for _ in requests + ] + + logged = [] + + class Logger: + async def set(self, request): + logged.append((request.cell.step_key, request.trace_id, request.error)) + return SimpleNamespace(id=uuid4()) + + async def create_scenario(run_id): + return SimpleNamespace(id=scenario_id) + + async def refresh_metrics(run_id, scenario_id): + return None + + processed = await process_sources( + run_id=run_id, + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"x": "1"}, + ) + ], + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=1, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={"evaluator-auto": FlakyRunner()}, + revisions={"evaluator-auto": {"id": "rev"}}, + max_retries=1, + ) + + assert call_count == 2 + assert processed[0].has_errors is False + eval_log = next(entry for entry in logged if entry[0] == "evaluator-auto") + assert eval_log[1] == "recovered" + assert eval_log[2] is None + + +@pytest.mark.asyncio +async def test_sdk_source_slice_exhausts_retries_and_marks_error(): + """max_retries=1 with persistent failure: result is still an error.""" + run_id = uuid4() + scenario_id = uuid4() + call_count = 0 + + class AlwaysFailRunner: + async def execute_batch(self, requests, semaphore=None): + nonlocal call_count + call_count += 1 + return [ + WorkflowExecutionResult( + status=EvaluationStatus.FAILURE, + error={"message": "always fails"}, + ) + for _ in requests + ] + + class Logger: + async def set(self, request): + return SimpleNamespace(id=uuid4()) + + async def create_scenario(run_id): + return SimpleNamespace(id=scenario_id) + + async def refresh_metrics(run_id, scenario_id): + return None + + processed = await process_sources( + run_id=run_id, + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"x": "1"}, + ) + ], + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=1, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={"evaluator-auto": AlwaysFailRunner()}, + revisions={"evaluator-auto": {"id": "rev"}}, + max_retries=1, + ) + + assert call_count == 2 + assert processed[0].has_errors is True + + +@pytest.mark.asyncio +async def test_sdk_source_slice_retries_only_failed_cells_in_batch(): + """With repeats=2, only the failing repeat is retried, not the successful one.""" + run_id = uuid4() + scenario_id = uuid4() + attempt_by_repeat: dict = {} + + class SelectiveFlakyRunner: + async def execute_batch(self, requests, semaphore=None): + results = [] + for req in requests: + idx = req.cell.repeat_idx + attempt_by_repeat[idx] = attempt_by_repeat.get(idx, 0) + 1 + if idx == 1 and attempt_by_repeat[idx] == 1: + results.append( + WorkflowExecutionResult( + status=EvaluationStatus.FAILURE, + error={"message": "fail repeat 1 first time"}, + ) + ) + else: + results.append( + WorkflowExecutionResult( + status=EvaluationStatus.SUCCESS, + trace_id=f"trace-{idx}", + ) + ) + return results + + class Logger: + async def set(self, request): + return SimpleNamespace(id=uuid4()) + + async def create_scenario(run_id): + return SimpleNamespace(id=scenario_id) + + async def refresh_metrics(run_id, scenario_id): + return None + + processed = await process_sources( + run_id=run_id, + source_items=[ + ResolvedSourceItem( + kind="testcase", + step_key="testset-main", + testcase_id=uuid4(), + inputs={"x": "1"}, + ) + ], + steps=[ + EvaluationStep(key="testset-main", type="input"), + EvaluationStep(key="evaluator-auto", type="annotation", origin="auto"), + ], + repeats=2, + create_scenario=create_scenario, + set_results=Logger(), + refresh_metrics=refresh_metrics, + runners={"evaluator-auto": SelectiveFlakyRunner()}, + revisions={"evaluator-auto": {"id": "rev"}}, + max_retries=1, + ) + + assert processed[0].has_errors is False + assert attempt_by_repeat[0] == 1 + assert attempt_by_repeat[1] == 2 diff --git a/sdks/python/oss/tests/pytest/utils/test_mock_v0.py b/sdks/python/oss/tests/pytest/utils/test_mock_v0.py new file mode 100644 index 0000000000..e67ddb41f7 --- /dev/null +++ b/sdks/python/oss/tests/pytest/utils/test_mock_v0.py @@ -0,0 +1,147 @@ +""" +Unit tests for the mock_v0 workflow (agenta:custom:mock:v0). + +mock_v0 is a deterministic, LLM-free, sandbox-free workflow used by evaluation +flow tests. It stands in for BOTH an application (invocation step, returns +outputs) and an evaluator (annotation step, returns {"score", "success"}). The +behavior is selected by name via parameters = {"key": ..., "kwargs": {...}}. + +async handlers are called via asyncio.run() so no pytest-asyncio marker is +needed. The @instrument() decorator is bypassed via __wrapped__. +""" + +import asyncio + +import pytest + +from agenta.sdk.workflows.errors import ( + MockV0Error, + InvalidConfigurationParameterV0Error, + MissingConfigurationParameterV0Error, +) +from agenta.sdk.workflows.handlers import mock_v0, MOCK_V0_BEHAVIORS + +_mock_v0 = mock_v0.__wrapped__ + + +def run(coro): + return asyncio.run(coro) + + +def call(key=None, *, kwargs=None, inputs=None, outputs=None, trace=None): + params = {} + if key is not None: + params["key"] = key + if kwargs is not None: + params["kwargs"] = kwargs + return run(_mock_v0(parameters=params, inputs=inputs, outputs=outputs, trace=trace)) + + +# --- parameter validation -------------------------------------------------- + + +def test_missing_key_raises(): + with pytest.raises(MissingConfigurationParameterV0Error): + call() + + +def test_unknown_key_raises(): + with pytest.raises(InvalidConfigurationParameterV0Error): + call("not-a-real-selector") + + +def test_non_dict_kwargs_raises(): + with pytest.raises(InvalidConfigurationParameterV0Error): + run(_mock_v0(parameters={"key": "pass", "kwargs": ["not", "a", "dict"]})) + + +# --- app-role selectors ---------------------------------------------------- + + +def test_echo_returns_inputs_verbatim(): + assert call("echo", inputs={"input": "hello"}) == {"input": "hello"} + + +def test_echo_empty_inputs(): + assert call("echo") == {} + + +def test_static_returns_kwargs_output(): + assert call("static", kwargs={"output": {"answer": "world"}}) == {"answer": "world"} + + +# --- evaluator-role selectors ---------------------------------------------- + + +def test_pass_returns_success(): + assert call("pass") == {"score": 1.0, "success": True} + + +def test_fail_returns_failure(): + assert call("fail") == {"score": 0.0, "success": False} + + +def test_score_above_threshold_succeeds(): + assert call("score", kwargs={"score": 0.8, "threshold": 0.5}) == { + "score": 0.8, + "success": True, + } + + +def test_score_below_threshold_fails(): + assert call("score", kwargs={"score": 0.2, "threshold": 0.5}) == { + "score": 0.2, + "success": False, + } + + +def test_score_default_threshold(): + # default threshold is 0.5 + assert call("score", kwargs={"score": 0.5})["success"] is True + assert call("score", kwargs={"score": 0.49})["success"] is False + + +# --- shared selectors ------------------------------------------------------ + + +def test_error_raises_mock_error(): + with pytest.raises(MockV0Error): + call("error") + + +def test_error_custom_message(): + with pytest.raises(MockV0Error) as exc: + call("error", kwargs={"message": "boom"}) + assert "boom" in str(exc.value) + + +def test_delay_sleeps_then_defers(): + # delay is fast here; just assert it defers to the named behavior + assert call("delay", kwargs={"seconds": 0.01, "then": "pass"}) == { + "score": 1.0, + "success": True, + } + + +def test_delay_default_then_is_pass(): + assert call("delay", kwargs={"seconds": 0.01})["success"] is True + + +def test_delay_recursion_guard(): + # then="delay" must not recurse into another sleep; it falls back to pass + assert call("delay", kwargs={"seconds": 0.01, "then": "delay"})["success"] is True + + +# --- registry -------------------------------------------------------------- + + +def test_registry_covers_documented_selectors(): + assert set(MOCK_V0_BEHAVIORS) == { + "echo", + "static", + "pass", + "fail", + "score", + "error", + "delay", + } diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 28235adf41..29dc125c39 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "mystace>=1,<2", "orjson>=3,<4", "python-jsonpath>=2,<3", - "daytona>=0.183,<0.184", + "daytona>=0.184,<0.185", ] [project.urls] diff --git a/sdks/python/pytest.ini b/sdks/python/pytest.ini index 9f1886b274..8152f5a6d4 100644 --- a/sdks/python/pytest.ini +++ b/sdks/python/pytest.ini @@ -5,6 +5,11 @@ testpaths = ee/tests/pytest addopts = -ra -n auto --self-contained-html asyncio_mode = auto +filterwarnings = + # TestsetRevisionData is a Pydantic model, not a test class; pytest's + # default `Test*` collection rule picks it up and warns. Silenced here + # (test-side config) rather than tagging the model in source. + ignore:cannot collect test class 'TestsetRevisionData':pytest.PytestCollectionWarning markers = coverage_smoke: breadth over depth coverage_full: breadth and depth diff --git a/sdks/python/uv.lock b/sdks/python/uv.lock index 0c8416db1d..e39ef44480 100644 --- a/sdks/python/uv.lock +++ b/sdks/python/uv.lock @@ -46,7 +46,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "agenta-client", editable = "../../clients/python" }, - { name = "daytona", specifier = ">=0.183,<0.184" }, + { name = "daytona", specifier = ">=0.184,<0.185" }, { name = "fastapi", specifier = ">=0.136,<0.137" }, { name = "httpx", specifier = ">=0.28,<0.29" }, { name = "jinja2", specifier = ">=3,<4" }, @@ -267,30 +267,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.43.19" +version = "1.43.21" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/97/da/229987ebb70daf5928f959aa8f4dd77dfcf425e6b0e7ff03aaef61ccc333/boto3-1.43.19.tar.gz", hash = "sha256:8b84704719dd3960ac12a8f37d9ff5adb853715baa9742f84fdbe2de0305c4cb", size = 113225, upload-time = "2026-06-01T19:33:06.514Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/f2/0ef88b6584285002a8a8000e34340f56e82681ad2b8a76ea8bd3ecaa5cb9/boto3-1.43.21.tar.gz", hash = "sha256:6dfeb70bf4f9a3514b91c7199f475f71f939199d62f9c63cd555b033fb283f89", size = 113157, upload-time = "2026-06-03T07:09:23.263Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/cc/77097be39d83068f864767b710cee0d8f9cd61331de816dd2675a596c328/boto3-1.43.19-py3-none-any.whl", hash = "sha256:ec6825193b75fbb6bfbf12181e4960d00ad2f404343586765394ce620e63783c", size = 140535, upload-time = "2026-06-01T19:33:03.758Z" }, + { url = "https://files.pythonhosted.org/packages/01/ea/5352950cbee9d1e8392e5396ddbc6defc982414e2abc004b501139bce13c/boto3-1.43.21-py3-none-any.whl", hash = "sha256:8bb863b32dabe5baa4f2e3701778c259243ba117e4811a595411819958c4fb1b", size = 140534, upload-time = "2026-06-03T07:09:21.18Z" }, ] [[package]] name = "botocore" -version = "1.43.19" +version = "1.43.21" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e2/a7/298986789785b74a954e2347114993be7e6b070417159125a6865f2687b6/botocore-1.43.19.tar.gz", hash = "sha256:18ac2fdd76c89b940707eb10493ff58678adad337d03215caec2d408ccd43cc0", size = 15435441, upload-time = "2026-06-01T19:32:53.126Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/97/d9d26cebf0a8533105e183d8438931c0b196e52484cd5bf00e8443ac1b2d/botocore-1.43.21.tar.gz", hash = "sha256:17604607efe28894e947401379e569cc8f0fe2d69337ece98bd0c82d1bcfaf92", size = 15451979, upload-time = "2026-06-03T07:09:11.489Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/75/fe4d45bdd08afd66f3d5273db58f3d8a29365e52ce3a0851f7f5e5900943/botocore-1.43.19-py3-none-any.whl", hash = "sha256:99dbdccbf748974750601e805cecc9362a85d11fee89d6d58cd3f4ff302e6ff9", size = 15117709, upload-time = "2026-06-01T19:32:47.871Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c7/8c60049357e96d663980d66b98a44cb3626a7e5eaca66480b97826eb5379/botocore-1.43.21-py3-none-any.whl", hash = "sha256:f021ba3e844c36031106fc531ec90259ef005ba5d04f691df53bc4ecbd08f0dd", size = 15135303, upload-time = "2026-06-03T07:09:06.115Z" }, ] [[package]] @@ -382,7 +382,7 @@ wheels = [ [[package]] name = "daytona" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, @@ -406,14 +406,14 @@ dependencies = [ { name = "urllib3" }, { name = "wsproto" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c8/dd/3914a2ae3dc4ecf3fd861eff221f4222e48ba4e5530ff7e585d4c4aa7965/daytona-0.183.0.tar.gz", hash = "sha256:2f0020537712ae67693fd2341d3008d464ada06dcd8e89baa6183846271e2f76", size = 141636, upload-time = "2026-05-29T10:59:50.565Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/6d/482ed38e868e5d6904961bf8c0317a94981bdd2b165f2611acb4f7094357/daytona-0.184.0.tar.gz", hash = "sha256:38a8de4a2daf34cb5940590db4d0f816ee8354d9919ca8ab1051df4a38f2107c", size = 141946, upload-time = "2026-06-03T14:46:56.231Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/ae/b23f7e5381ca5d66f2be0f802ebef0110b5e21022946fa7b9d6e873ec6ad/daytona-0.183.0-py3-none-any.whl", hash = "sha256:89af1bcfc9b37c580329fcad01167bcfab786330e2d0ae74ad3fae8cb40397d2", size = 172192, upload-time = "2026-05-29T10:59:48.838Z" }, + { url = "https://files.pythonhosted.org/packages/10/da/5ff417bbf0ca7d480ee88c310372fec3ab039a66878c27ecab97203d00c0/daytona-0.184.0-py3-none-any.whl", hash = "sha256:630df33c62aebae2ae34679eff79198806014567850c98dcd8d45accdcc5437a", size = 172586, upload-time = "2026-06-03T14:46:54.467Z" }, ] [[package]] name = "daytona-api-client" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -421,14 +421,14 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/40/a5090268e734f804489b8ca3dcdbd4de7c392b1b4d4b02a60c5d85c7198f/daytona_api_client-0.183.0.tar.gz", hash = "sha256:659fff84a6461d1f51986fb814bf6e3fbc51ac203e6502eb6ae6f26029d69814", size = 147714, upload-time = "2026-05-29T10:59:01.709Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/63/cdb3ea6dd35fd2091a7e1f988ce1d3194d9bec1accfa6dbbf63ad424e671/daytona_api_client-0.184.0.tar.gz", hash = "sha256:3cdb111cf2a21be13cc648e444162c094d45f80d1eeb638b671fc854be307c23", size = 148346, upload-time = "2026-06-03T14:46:09.888Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/70/6a/8890e773ecf1cd9c1b70ddb0fda57e4941f5a8a44ced1c91574253797f63/daytona_api_client-0.183.0-py3-none-any.whl", hash = "sha256:c319ae3b4666ec15402cacb8ec3aaffbf55802c0dbc33f6826ef07afceae5085", size = 406274, upload-time = "2026-05-29T10:59:00.427Z" }, + { url = "https://files.pythonhosted.org/packages/88/65/44993d1bb3cb1f5f294ac97bb3246346c14bf7ba3e79cdd4e92c7df0c6df/daytona_api_client-0.184.0-py3-none-any.whl", hash = "sha256:8fec5062d757fb533e82e9bf295fc1625ae294b4a154c24081bbd2bf30bcb5dd", size = 407600, upload-time = "2026-06-03T14:46:08.086Z" }, ] [[package]] name = "daytona-api-client-async" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -437,14 +437,14 @@ dependencies = [ { name = "python-dateutil" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/90/aa/c9de46d48ee45228566a53bb699bc4bdab693dc8ce300516ce334bfec82f/daytona_api_client_async-0.183.0.tar.gz", hash = "sha256:4f5b397e375635fe3614c342cf34faa236d5a0918c53588d7059b5061d881407", size = 148190, upload-time = "2026-05-29T10:59:14.315Z" } +sdist = { url = "https://files.pythonhosted.org/packages/df/2f/caba484cbac693d269b417df73237f117882e2e28d1af4c3878e568bf148/daytona_api_client_async-0.184.0.tar.gz", hash = "sha256:a3cb0ebe5eec2c824b7848d4189a035a13cd84de1e0083c3f0c033f5445f3b71", size = 149131, upload-time = "2026-06-03T14:46:19.64Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/71/db/dde2e2478faf010b048af24227e5ff8b4c635d8104ace26c66ed1f29dd49/daytona_api_client_async-0.183.0-py3-none-any.whl", hash = "sha256:8be2b0bac344893294a8ffd8df7a4703427ec48376f6948975aca85f09dba541", size = 409552, upload-time = "2026-05-29T10:59:12.964Z" }, + { url = "https://files.pythonhosted.org/packages/8e/fc/c6e304db0bb382a0842cda913474ba42060acbf4f6cc91d8708f7997e337/daytona_api_client_async-0.184.0-py3-none-any.whl", hash = "sha256:19f268e6fb4d4190e7a4ea2ad3f4600bf14c25ab8f4997390d8d07d4a281de61", size = 410889, upload-time = "2026-06-03T14:46:18.051Z" }, ] [[package]] name = "daytona-toolbox-api-client" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -452,14 +452,14 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ac/ee/1c8b9785c4028a0932f655e131e44417380a1e3d6ff4eb58b4e10b9aac18/daytona_toolbox_api_client-0.183.0.tar.gz", hash = "sha256:7cf4eb8ab36ec488f436695822b0e4c06e82d12d8fb2fe28e3239151dd43c4b7", size = 77732, upload-time = "2026-05-29T10:58:40.866Z" } +sdist = { url = "https://files.pythonhosted.org/packages/26/8f/78b0c06e91065e573ce1f2f4c6b7941066be939de46128a7e97a30b2e4a7/daytona_toolbox_api_client-0.184.0.tar.gz", hash = "sha256:c325a050ca45e7832c7d05127260634ed0742d8bc75b2d9de65847153fba70e6", size = 77893, upload-time = "2026-06-03T14:45:52.776Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/3a/904bbf95eac89036775051410dbaf55414a73a1e6d074e1486ca29ee3dbb/daytona_toolbox_api_client-0.183.0-py3-none-any.whl", hash = "sha256:bc0c2fb986f50cad23791afa181c04f27d975b59301d7ce59ce519c1f518483c", size = 205626, upload-time = "2026-05-29T10:58:39.381Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f9/311c8a6d8fdbb48793c1d5504fa212b0325047887e1c8c8dc2560687cff6/daytona_toolbox_api_client-0.184.0-py3-none-any.whl", hash = "sha256:05b6b179018247bf3e7de667b049b0d7bc0e5e83f58b498e0371988e61b3041e", size = 205757, upload-time = "2026-06-03T14:45:51.343Z" }, ] [[package]] name = "daytona-toolbox-api-client-async" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -468,9 +468,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/0f/85e2ad6511ae228e1c423944d6d90ba3ace2153685ba8aa3b4035c5ccc6f/daytona_toolbox_api_client_async-0.183.0.tar.gz", hash = "sha256:31984e9fc5c440efbb5d23426f187046fa419c075c42d8691b127304692c4e6b", size = 71347, upload-time = "2026-05-29T10:59:03.772Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/cc/d10617b9bf1eae7f61870d0f98d5f22e6a56b0777ee9643427be5d8ec488/daytona_toolbox_api_client_async-0.184.0.tar.gz", hash = "sha256:495526bbd8de1d1ce2d86ecdb561efd5c4e7e86007abf4bc550ecc0c5efaf2e9", size = 71490, upload-time = "2026-06-03T14:46:20.57Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/23/66/af01390a47ba626454fd2ba44766282ffb7174ecc60c1742ab4859bd074f/daytona_toolbox_api_client_async-0.183.0-py3-none-any.whl", hash = "sha256:54a25297cfaf68568d85cecfd5c07da2e7fd862c8ab445df97b76a360951f757", size = 203902, upload-time = "2026-05-29T10:59:02.046Z" }, + { url = "https://files.pythonhosted.org/packages/db/71/d6e98539e8a855bc63a55ce60f554f0b0063e9275424fea0476eb46f3a2a/daytona_toolbox_api_client_async-0.184.0-py3-none-any.whl", hash = "sha256:b971bc3475c0f658c38a86de40a5d4d31394bf60ad4b4c7f8b7693a5a56647b1", size = 204036, upload-time = "2026-06-03T14:46:18.771Z" }, ] [[package]] @@ -762,11 +762,11 @@ wheels = [ [[package]] name = "idna" -version = "3.17" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/28/99c51f664567218d824af024c0251650fb27e4ca066df188dab0769c5b91/idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f", size = 196048, upload-time = "2026-05-28T14:32:38.55Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/a7/f76514cc40ad6234098ecdebda08732d75964776c51a42845b7da10649e2/idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c", size = 65316, upload-time = "2026-05-28T14:32:37.035Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -1375,7 +1375,7 @@ wheels = [ [[package]] name = "posthog" -version = "7.16.3" +version = "7.16.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -1383,9 +1383,9 @@ dependencies = [ { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/20/d4/6e1c24823c515683cb6f74bd1dcc3fbe55f60a27743c1694bdc61c5db5d0/posthog-7.16.3.tar.gz", hash = "sha256:ff8972813c836ae4fcb634b499cf06d643bc3c4f49931218fe142e7aa8a39810", size = 226535, upload-time = "2026-06-01T13:24:07.781Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/af/10ba67866a1246e608fccf0d186c63b55054feb66c0fc9d77be458aac9c0/posthog-7.16.4.tar.gz", hash = "sha256:a2baa6595c7089865350826932867ed31355e27307ae4774bc88d42536678848", size = 228616, upload-time = "2026-06-03T13:12:16.006Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/0b/562afe6f037ab3dde06d2bbe33f1de9f8516ac22f882ad263e4e0bfb665b/posthog-7.16.3-py3-none-any.whl", hash = "sha256:f0cbaf25ac06211b87c0a43500673fa2d8de86eb1edb75b8bf688f1b884878ce", size = 264300, upload-time = "2026-06-01T13:24:06.074Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b2/4c9659e06074b473216dc69360fcf7ddba9134e622d16ef10bb45da5cc91/posthog-7.16.4-py3-none-any.whl", hash = "sha256:92ede0237aafb7b90cdb730dacf828cf915676366a69522a76d300fa580f0da0", size = 267073, upload-time = "2026-06-03T13:12:14.415Z" }, ] [[package]] diff --git a/services/entrypoints/main.py b/services/entrypoints/main.py index 919698c5ae..72cc291dfb 100644 --- a/services/entrypoints/main.py +++ b/services/entrypoints/main.py @@ -38,6 +38,7 @@ builtin_match_app, custom_code_app, custom_hook_app, + custom_mock_app, custom_config_app, ) from oss.src.chat import chat_v0_app @@ -144,6 +145,7 @@ async def health(): app.mount("/config/v0", custom_config_app) app.mount("/code/v0", custom_code_app) app.mount("/hook/v0", custom_hook_app) +app.mount("/mock/v0", custom_mock_app) app.mount("/match/v0", builtin_match_app) app.mount("/llm/v0", builtin_llm_app) app.mount("/auto_exact_match/v0", builtin_auto_exact_match_app) @@ -177,4 +179,5 @@ async def health(): port=8080, reload=True, reload_dirs=[".", "/sdk"], + reload_excludes=["**/tests/**", "/sdk/tests"], ) diff --git a/services/oss/src/managed.py b/services/oss/src/managed.py index d2a8081715..8ad8226175 100644 --- a/services/oss/src/managed.py +++ b/services/oss/src/managed.py @@ -24,6 +24,7 @@ json_multi_field_match_v0, llm_v0, match_v0, + mock_v0, config_v0, ) @@ -53,6 +54,10 @@ def _create_managed_service( hook_v0, uri="agenta:custom:hook:v0", ) +custom_mock_app = _create_managed_service( + mock_v0, + uri="agenta:custom:mock:v0", +) builtin_match_app = _create_managed_service( match_v0, uri="agenta:builtin:match:v0", diff --git a/services/oss/tests/pytest/acceptance/test_catalog_to_observability.py b/services/oss/tests/pytest/acceptance/test_catalog_to_observability.py index c5c9645ceb..2783057af1 100644 --- a/services/oss/tests/pytest/acceptance/test_catalog_to_observability.py +++ b/services/oss/tests/pytest/acceptance/test_catalog_to_observability.py @@ -34,7 +34,7 @@ # --------------------------------------------------------------------------- _POLL_INTERVAL = 0.5 # seconds between trace-fetch retries -_POLL_TIMEOUT = 20.0 # maximum seconds to wait for a trace to appear +_POLL_TIMEOUT = 30.0 # maximum seconds to wait for a trace to appear def _uid() -> str: diff --git a/services/uv.lock b/services/uv.lock index aeb5e2e243..4246db4e5a 100644 --- a/services/uv.lock +++ b/services/uv.lock @@ -33,7 +33,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "agenta-client", editable = "../clients/python" }, - { name = "daytona", specifier = ">=0.183,<0.184" }, + { name = "daytona", specifier = ">=0.184,<0.185" }, { name = "fastapi", specifier = ">=0.136,<0.137" }, { name = "httpx", specifier = ">=0.28,<0.29" }, { name = "jinja2", specifier = ">=3,<4" }, @@ -245,30 +245,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.43.19" +version = "1.43.21" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/97/da/229987ebb70daf5928f959aa8f4dd77dfcf425e6b0e7ff03aaef61ccc333/boto3-1.43.19.tar.gz", hash = "sha256:8b84704719dd3960ac12a8f37d9ff5adb853715baa9742f84fdbe2de0305c4cb", size = 113225, upload-time = "2026-06-01T19:33:06.514Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/f2/0ef88b6584285002a8a8000e34340f56e82681ad2b8a76ea8bd3ecaa5cb9/boto3-1.43.21.tar.gz", hash = "sha256:6dfeb70bf4f9a3514b91c7199f475f71f939199d62f9c63cd555b033fb283f89", size = 113157, upload-time = "2026-06-03T07:09:23.263Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/cc/77097be39d83068f864767b710cee0d8f9cd61331de816dd2675a596c328/boto3-1.43.19-py3-none-any.whl", hash = "sha256:ec6825193b75fbb6bfbf12181e4960d00ad2f404343586765394ce620e63783c", size = 140535, upload-time = "2026-06-01T19:33:03.758Z" }, + { url = "https://files.pythonhosted.org/packages/01/ea/5352950cbee9d1e8392e5396ddbc6defc982414e2abc004b501139bce13c/boto3-1.43.21-py3-none-any.whl", hash = "sha256:8bb863b32dabe5baa4f2e3701778c259243ba117e4811a595411819958c4fb1b", size = 140534, upload-time = "2026-06-03T07:09:21.18Z" }, ] [[package]] name = "botocore" -version = "1.43.19" +version = "1.43.21" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e2/a7/298986789785b74a954e2347114993be7e6b070417159125a6865f2687b6/botocore-1.43.19.tar.gz", hash = "sha256:18ac2fdd76c89b940707eb10493ff58678adad337d03215caec2d408ccd43cc0", size = 15435441, upload-time = "2026-06-01T19:32:53.126Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/97/d9d26cebf0a8533105e183d8438931c0b196e52484cd5bf00e8443ac1b2d/botocore-1.43.21.tar.gz", hash = "sha256:17604607efe28894e947401379e569cc8f0fe2d69337ece98bd0c82d1bcfaf92", size = 15451979, upload-time = "2026-06-03T07:09:11.489Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/75/fe4d45bdd08afd66f3d5273db58f3d8a29365e52ce3a0851f7f5e5900943/botocore-1.43.19-py3-none-any.whl", hash = "sha256:99dbdccbf748974750601e805cecc9362a85d11fee89d6d58cd3f4ff302e6ff9", size = 15117709, upload-time = "2026-06-01T19:32:47.871Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c7/8c60049357e96d663980d66b98a44cb3626a7e5eaca66480b97826eb5379/botocore-1.43.21-py3-none-any.whl", hash = "sha256:f021ba3e844c36031106fc531ec90259ef005ba5d04f691df53bc4ecbd08f0dd", size = 15135303, upload-time = "2026-06-03T07:09:06.115Z" }, ] [[package]] @@ -453,7 +453,7 @@ wheels = [ [[package]] name = "daytona" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, @@ -477,14 +477,14 @@ dependencies = [ { name = "urllib3" }, { name = "wsproto" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c8/dd/3914a2ae3dc4ecf3fd861eff221f4222e48ba4e5530ff7e585d4c4aa7965/daytona-0.183.0.tar.gz", hash = "sha256:2f0020537712ae67693fd2341d3008d464ada06dcd8e89baa6183846271e2f76", size = 141636, upload-time = "2026-05-29T10:59:50.565Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/6d/482ed38e868e5d6904961bf8c0317a94981bdd2b165f2611acb4f7094357/daytona-0.184.0.tar.gz", hash = "sha256:38a8de4a2daf34cb5940590db4d0f816ee8354d9919ca8ab1051df4a38f2107c", size = 141946, upload-time = "2026-06-03T14:46:56.231Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/ae/b23f7e5381ca5d66f2be0f802ebef0110b5e21022946fa7b9d6e873ec6ad/daytona-0.183.0-py3-none-any.whl", hash = "sha256:89af1bcfc9b37c580329fcad01167bcfab786330e2d0ae74ad3fae8cb40397d2", size = 172192, upload-time = "2026-05-29T10:59:48.838Z" }, + { url = "https://files.pythonhosted.org/packages/10/da/5ff417bbf0ca7d480ee88c310372fec3ab039a66878c27ecab97203d00c0/daytona-0.184.0-py3-none-any.whl", hash = "sha256:630df33c62aebae2ae34679eff79198806014567850c98dcd8d45accdcc5437a", size = 172586, upload-time = "2026-06-03T14:46:54.467Z" }, ] [[package]] name = "daytona-api-client" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -492,14 +492,14 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/40/a5090268e734f804489b8ca3dcdbd4de7c392b1b4d4b02a60c5d85c7198f/daytona_api_client-0.183.0.tar.gz", hash = "sha256:659fff84a6461d1f51986fb814bf6e3fbc51ac203e6502eb6ae6f26029d69814", size = 147714, upload-time = "2026-05-29T10:59:01.709Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/63/cdb3ea6dd35fd2091a7e1f988ce1d3194d9bec1accfa6dbbf63ad424e671/daytona_api_client-0.184.0.tar.gz", hash = "sha256:3cdb111cf2a21be13cc648e444162c094d45f80d1eeb638b671fc854be307c23", size = 148346, upload-time = "2026-06-03T14:46:09.888Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/70/6a/8890e773ecf1cd9c1b70ddb0fda57e4941f5a8a44ced1c91574253797f63/daytona_api_client-0.183.0-py3-none-any.whl", hash = "sha256:c319ae3b4666ec15402cacb8ec3aaffbf55802c0dbc33f6826ef07afceae5085", size = 406274, upload-time = "2026-05-29T10:59:00.427Z" }, + { url = "https://files.pythonhosted.org/packages/88/65/44993d1bb3cb1f5f294ac97bb3246346c14bf7ba3e79cdd4e92c7df0c6df/daytona_api_client-0.184.0-py3-none-any.whl", hash = "sha256:8fec5062d757fb533e82e9bf295fc1625ae294b4a154c24081bbd2bf30bcb5dd", size = 407600, upload-time = "2026-06-03T14:46:08.086Z" }, ] [[package]] name = "daytona-api-client-async" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -508,14 +508,14 @@ dependencies = [ { name = "python-dateutil" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/90/aa/c9de46d48ee45228566a53bb699bc4bdab693dc8ce300516ce334bfec82f/daytona_api_client_async-0.183.0.tar.gz", hash = "sha256:4f5b397e375635fe3614c342cf34faa236d5a0918c53588d7059b5061d881407", size = 148190, upload-time = "2026-05-29T10:59:14.315Z" } +sdist = { url = "https://files.pythonhosted.org/packages/df/2f/caba484cbac693d269b417df73237f117882e2e28d1af4c3878e568bf148/daytona_api_client_async-0.184.0.tar.gz", hash = "sha256:a3cb0ebe5eec2c824b7848d4189a035a13cd84de1e0083c3f0c033f5445f3b71", size = 149131, upload-time = "2026-06-03T14:46:19.64Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/71/db/dde2e2478faf010b048af24227e5ff8b4c635d8104ace26c66ed1f29dd49/daytona_api_client_async-0.183.0-py3-none-any.whl", hash = "sha256:8be2b0bac344893294a8ffd8df7a4703427ec48376f6948975aca85f09dba541", size = 409552, upload-time = "2026-05-29T10:59:12.964Z" }, + { url = "https://files.pythonhosted.org/packages/8e/fc/c6e304db0bb382a0842cda913474ba42060acbf4f6cc91d8708f7997e337/daytona_api_client_async-0.184.0-py3-none-any.whl", hash = "sha256:19f268e6fb4d4190e7a4ea2ad3f4600bf14c25ab8f4997390d8d07d4a281de61", size = 410889, upload-time = "2026-06-03T14:46:18.051Z" }, ] [[package]] name = "daytona-toolbox-api-client" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -523,14 +523,14 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ac/ee/1c8b9785c4028a0932f655e131e44417380a1e3d6ff4eb58b4e10b9aac18/daytona_toolbox_api_client-0.183.0.tar.gz", hash = "sha256:7cf4eb8ab36ec488f436695822b0e4c06e82d12d8fb2fe28e3239151dd43c4b7", size = 77732, upload-time = "2026-05-29T10:58:40.866Z" } +sdist = { url = "https://files.pythonhosted.org/packages/26/8f/78b0c06e91065e573ce1f2f4c6b7941066be939de46128a7e97a30b2e4a7/daytona_toolbox_api_client-0.184.0.tar.gz", hash = "sha256:c325a050ca45e7832c7d05127260634ed0742d8bc75b2d9de65847153fba70e6", size = 77893, upload-time = "2026-06-03T14:45:52.776Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/3a/904bbf95eac89036775051410dbaf55414a73a1e6d074e1486ca29ee3dbb/daytona_toolbox_api_client-0.183.0-py3-none-any.whl", hash = "sha256:bc0c2fb986f50cad23791afa181c04f27d975b59301d7ce59ce519c1f518483c", size = 205626, upload-time = "2026-05-29T10:58:39.381Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f9/311c8a6d8fdbb48793c1d5504fa212b0325047887e1c8c8dc2560687cff6/daytona_toolbox_api_client-0.184.0-py3-none-any.whl", hash = "sha256:05b6b179018247bf3e7de667b049b0d7bc0e5e83f58b498e0371988e61b3041e", size = 205757, upload-time = "2026-06-03T14:45:51.343Z" }, ] [[package]] name = "daytona-toolbox-api-client-async" -version = "0.183.0" +version = "0.184.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -539,9 +539,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/0f/85e2ad6511ae228e1c423944d6d90ba3ace2153685ba8aa3b4035c5ccc6f/daytona_toolbox_api_client_async-0.183.0.tar.gz", hash = "sha256:31984e9fc5c440efbb5d23426f187046fa419c075c42d8691b127304692c4e6b", size = 71347, upload-time = "2026-05-29T10:59:03.772Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/cc/d10617b9bf1eae7f61870d0f98d5f22e6a56b0777ee9643427be5d8ec488/daytona_toolbox_api_client_async-0.184.0.tar.gz", hash = "sha256:495526bbd8de1d1ce2d86ecdb561efd5c4e7e86007abf4bc550ecc0c5efaf2e9", size = 71490, upload-time = "2026-06-03T14:46:20.57Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/23/66/af01390a47ba626454fd2ba44766282ffb7174ecc60c1742ab4859bd074f/daytona_toolbox_api_client_async-0.183.0-py3-none-any.whl", hash = "sha256:54a25297cfaf68568d85cecfd5c07da2e7fd862c8ab445df97b76a360951f757", size = 203902, upload-time = "2026-05-29T10:59:02.046Z" }, + { url = "https://files.pythonhosted.org/packages/db/71/d6e98539e8a855bc63a55ce60f554f0b0063e9275424fea0476eb46f3a2a/daytona_toolbox_api_client_async-0.184.0-py3-none-any.whl", hash = "sha256:b971bc3475c0f658c38a86de40a5d4d31394bf60ad4b4c7f8b7693a5a56647b1", size = 204036, upload-time = "2026-06-03T14:46:18.771Z" }, ] [[package]] @@ -733,7 +733,7 @@ wheels = [ [[package]] name = "google-api-core" -version = "2.30.3" +version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-auth" }, @@ -742,9 +742,9 @@ dependencies = [ { name = "protobuf" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001, upload-time = "2026-04-10T00:41:28.035Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/22/155cadf1d49272a9cf48f3168c0f3874fa13397297e611a5ea00cd093880/google_api_core-2.31.0.tar.gz", hash = "sha256:2be84ee0f584c48e6bde1b36766e23348b361fb7e55e56135fc76ce1c397f9c2", size = 176492, upload-time = "2026-06-03T14:52:17.257Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274, upload-time = "2026-04-09T22:57:16.198Z" }, + { url = "https://files.pythonhosted.org/packages/86/40/9bdbb60b03a332bd45acb8703da08bbc27d991d35286b62e42acc86d243a/google_api_core-2.31.0-py3-none-any.whl", hash = "sha256:ef79fb3784c71cbac89cbd03301ba0c8fb8ad2aa95d7f9204dd9628f7adf59ab", size = 173102, upload-time = "2026-06-03T14:51:26.729Z" }, ] [package.optional-dependencies] @@ -1143,11 +1143,11 @@ wheels = [ [[package]] name = "idna" -version = "3.17" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/28/99c51f664567218d824af024c0251650fb27e4ca066df188dab0769c5b91/idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f", size = 196048, upload-time = "2026-05-28T14:32:38.55Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/a7/f76514cc40ad6234098ecdebda08732d75964776c51a42845b7da10649e2/idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c", size = 65316, upload-time = "2026-05-28T14:32:37.035Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] diff --git a/web/_reference/agenta-sdk/src/vault.ts b/web/_reference/agenta-sdk/src/vault.ts index b592e52523..cc5580e7d2 100644 --- a/web/_reference/agenta-sdk/src/vault.ts +++ b/web/_reference/agenta-sdk/src/vault.ts @@ -17,19 +17,19 @@ export class Vault { /** * List all secrets for the current project. * - * GET /vault/v1/secrets/ + * GET /secrets/ */ async list(): Promise { - return this.client.request("GET", "/vault/v1/secrets/", {legacy: true}) + return this.client.request("GET", "/secrets/", {legacy: true}) } /** * Get a single secret by ID. * - * GET /vault/v1/secrets/{secretId} + * GET /secrets/{secretId} */ async get(secretId: string): Promise { - return this.client.request("GET", `/vault/v1/secrets/${secretId}`, { + return this.client.request("GET", `/secrets/${secretId}`, { legacy: true, }) } @@ -37,10 +37,10 @@ export class Vault { /** * Create a new secret. * - * POST /vault/v1/secrets/ + * POST /secrets/ */ async create(request: CreateSecretRequest): Promise { - return this.client.request("POST", "/vault/v1/secrets/", { + return this.client.request("POST", "/secrets/", { body: request, legacy: true, }) @@ -49,10 +49,10 @@ export class Vault { /** * Update an existing secret. * - * PUT /vault/v1/secrets/{secretId} + * PUT /secrets/{secretId} */ async update(secretId: string, request: UpdateSecretRequest): Promise { - return this.client.request("PUT", `/vault/v1/secrets/${secretId}`, { + return this.client.request("PUT", `/secrets/${secretId}`, { body: request, legacy: true, }) @@ -61,11 +61,11 @@ export class Vault { /** * Delete a secret. * - * DELETE /vault/v1/secrets/{secretId} + * DELETE /secrets/{secretId} * * Returns void (204 No Content). */ async delete(secretId: string): Promise { - await this.client.requestRaw("DELETE", `/vault/v1/secrets/${secretId}`, {legacy: true}) + await this.client.requestRaw("DELETE", `/secrets/${secretId}`, {legacy: true}) } } diff --git a/web/ee/package.json b/web/ee/package.json index 2d33bb2ad3..d0022df036 100644 --- a/web/ee/package.json +++ b/web/ee/package.json @@ -43,7 +43,6 @@ "@types/react-dom": "^19.0.4", "@types/react-resizable": "^3.0.7", "@types/react-window": "^1.8.8", - "@types/recharts": "^2.0.1", "antd": "^6.1.3", "autoprefixer": "10.4.20", "axios": "1.16.0", diff --git a/web/oss/src/components/pages/evaluations/NewEvaluation/Components/AdvancedSettings.tsx b/web/oss/src/components/pages/evaluations/NewEvaluation/Components/AdvancedSettings.tsx index 4edb5403b0..c825e9af04 100644 --- a/web/oss/src/components/pages/evaluations/NewEvaluation/Components/AdvancedSettings.tsx +++ b/web/oss/src/components/pages/evaluations/NewEvaluation/Components/AdvancedSettings.tsx @@ -1,14 +1,26 @@ import {memo, useCallback, useMemo} from "react" import {QuestionCircleOutlined} from "@ant-design/icons" -import {Button, Col, Flex, Form, Input, InputNumber, Row, Tooltip, Typography} from "antd" +import {Button, Flex, Form, InputNumber, Tooltip, Typography} from "antd" import deepEqual from "fast-deep-equal" import {DEFAULT_ADVANCE_SETTINGS} from "../assets/constants" -import {AdvancedSettingsProps} from "../types" +import {AdvancedSettingsProps, EvaluationConcurrencySettings} from "../types" + +const FIELD_LABELS: Record = { + batch_size: "Batch Size", + max_retries: "Max Retries", + retry_delay: "Retry Delay (s)", +} + +const FIELD_TOOLTIPS: Record = { + batch_size: "Maximum number of concurrent invocations", + max_retries: "How many times to retry a failed invocation", + retry_delay: "Seconds to wait before retrying a failed invocation", +} const AdvancedSettings = ({advanceSettings, setAdvanceSettings}: AdvancedSettingsProps) => { - const handleChange = (key: string, value: any) => { + const handleChange = (key: keyof EvaluationConcurrencySettings, value: number | null) => { setAdvanceSettings((prev) => ({ ...prev, [key]: value, @@ -24,17 +36,14 @@ const AdvancedSettings = ({advanceSettings, setAdvanceSettings}: AdvancedSetting [advanceSettings], ) - const {correct_answer_column, ...rateLimitConfig} = advanceSettings - return (
- Rate Limit Configuration + Concurrency {isAdvancedSettingsChanged && (