fix: require Flow production release evidence

Author: abrichr

.github/workflows/quickstart-lifecycle.yml

@@ -71,4 +71,4 @@ jobs:
             runs/lifecycle/artifacts/**/REPORT.md
             runs/lifecycle/artifacts/**/report.json
             runs/lifecycle/artifacts/**/patch.json
-          if-no-files-found: warn
+          if-no-files-found: error

public-artifacts.json

@@ -113,7 +113,7 @@
     },
     {
       "path": ".github/workflows/quickstart-lifecycle.yml",
-      "sha256": "34edc8580648b5aeeb3cbd1860681567694a6b5c9ea337b6bb504509ea644b07"
+      "sha256": "cdc9b801276e410eda81cb044320c6afc7b11907e986e2aa36bb8ef3af610b46"
     },
     {
       "path": ".github/workflows/release-health.yml",

pyproject.toml

@@ -17,7 +17,13 @@ authors = [{ name = "OpenAdapt.AI" }]
 dependencies = [
     "pydantic>=2.5",
     "numpy>=1.26",
-    "opencv-python-headless>=4.9",
+    # rapidocr-onnxruntime requires opencv-python by distribution name. Python
+    # packaging has no provider/alias mechanism through which the headless
+    # distribution can satisfy that requirement. Declaring both installs two
+    # distributions that own the same cv2 package. Use the one provider that
+    # RapidOCR's published metadata requires, and enforce this in the clean
+    # wheel lifecycle on every supported OS.
+    "opencv-python>=4.9",
     "pillow>=10.0",
     "rapidocr-onnxruntime>=1.3",
     # GPU-less runners call the on-prem VLM appliance over HTTP

scripts/check_release_ci.py

@@ -24,6 +24,13 @@
         "test-matrix (macos-latest, 3.12)",
     }
 )
+EXPECTED_CLEAN_MACHINE_JOBS = frozenset(
+    {
+        "lifecycle (ubuntu-latest)",
+        "lifecycle (macos-latest)",
+        "lifecycle (windows-latest)",
+    }
+)
 PER_PAGE = 100
 MAX_PAGES = 100
 GITHUB_API_VERSION = "2022-11-28"
@@ -48,6 +55,12 @@ class Qualification:
     job_names: frozenset[str]
 
 
+@dataclass(frozen=True)
+class ProductionQualification:
+    full_matrix: Qualification
+    clean_machine: Qualification
+
+
 class GitHubJSONFetcher:
     """Small authenticated GitHub REST reader with fail-closed decoding."""
 
@@ -207,6 +220,103 @@ def require_exact_full_matrix(
     )
 
 
+def require_exact_clean_machine(
+    fetch_json: JSONFetcher,
+    *,
+    repository: str,
+    sha: str,
+) -> Qualification:
+    """Require the three-OS clean-wheel Browser lifecycle on the exact SHA."""
+
+    if not _REPOSITORY_RE.fullmatch(repository):
+        raise QualificationError(f"invalid GitHub repository: {repository!r}")
+    if not _SHA_RE.fullmatch(sha):
+        raise QualificationError(f"invalid Git commit SHA: {sha!r}")
+
+    runs = _paginate(
+        fetch_json,
+        f"/repos/{repository}/actions/workflows/quickstart-lifecycle.yml/runs",
+        "workflow_runs",
+        {"head_sha": sha, "event": "workflow_dispatch"},
+    )
+    exact_runs = [
+        run
+        for run in runs
+        if run.get("head_sha") == sha and run.get("event") == "workflow_dispatch"
+    ]
+    if not exact_runs:
+        raise QualificationPending(
+            f"no exact-SHA workflow_dispatch clean-machine run exists for {sha}"
+        )
+    latest = max(
+        exact_runs,
+        key=lambda run: (str(run.get("created_at", "")), int(run.get("id", 0))),
+    )
+    run_id = latest.get("id")
+    if not isinstance(run_id, int) or run_id <= 0:
+        raise QualificationError("clean-machine qualification run has an invalid id")
+    status = latest.get("status")
+    conclusion = latest.get("conclusion")
+    if status != "completed":
+        raise QualificationPending(
+            f"exact-SHA clean-machine run {run_id} is {status!r}"
+        )
+    if conclusion != "success":
+        raise QualificationError(
+            f"exact-SHA clean-machine run {run_id} concluded {conclusion!r}"
+        )
+
+    jobs = _paginate(
+        fetch_json,
+        f"/repos/{repository}/actions/runs/{run_id}/jobs",
+        "jobs",
+        {"filter": "latest"},
+    )
+    lifecycle_jobs = [
+        job
+        for job in jobs
+        if isinstance(job.get("name"), str)
+        and str(job["name"]).startswith("lifecycle")
+    ]
+    counts = Counter(str(job.get("name")) for job in lifecycle_jobs)
+    expected_counts = Counter({name: 1 for name in EXPECTED_CLEAN_MACHINE_JOBS})
+    if counts != expected_counts:
+        raise QualificationError(
+            "exact-SHA clean-machine job set/count mismatch: "
+            f"expected={dict(sorted(expected_counts.items()))}, "
+            f"observed={dict(sorted(counts.items()))}"
+        )
+    non_success = {
+        str(job["name"]): job.get("conclusion")
+        for job in lifecycle_jobs
+        if job.get("conclusion") != "success"
+    }
+    if non_success:
+        raise QualificationError(
+            "exact-SHA clean-machine run has non-success jobs: "
+            f"{dict(sorted(non_success.items()))}"
+        )
+    return Qualification(run_id=run_id, sha=sha, job_names=frozenset(counts))
+
+
+def require_production_qualification(
+    fetch_json: JSONFetcher,
+    *,
+    repository: str,
+    sha: str,
+) -> ProductionQualification:
+    """Require both code-level and clean-wheel product qualification."""
+
+    return ProductionQualification(
+        full_matrix=require_exact_full_matrix(
+            fetch_json, repository=repository, sha=sha
+        ),
+        clean_machine=require_exact_clean_machine(
+            fetch_json, repository=repository, sha=sha
+        ),
+    )
+
+
 def _parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description="Require an exact-SHA dispatched CI full-matrix qualification."
@@ -232,7 +342,7 @@ def main(argv: list[str] | None = None) -> int:
     deadline = time.monotonic() + args.wait_seconds
     while True:
         try:
-            qualification = require_exact_full_matrix(
+            qualification = require_production_qualification(
                 fetch_json,
                 repository=args.repository,
                 sha=args.sha,
@@ -249,8 +359,11 @@ def main(argv: list[str] | None = None) -> int:
             return 1
         print(
             "Release qualification passed: "
-            f"sha={qualification.sha} run_id={qualification.run_id} "
-            f"matrix_jobs={len(qualification.job_names)}"
+            f"sha={qualification.full_matrix.sha} "
+            f"matrix_run_id={qualification.full_matrix.run_id} "
+            f"clean_machine_run_id={qualification.clean_machine.run_id} "
+            f"matrix_jobs={len(qualification.full_matrix.job_names)} "
+            f"clean_machine_jobs={len(qualification.clean_machine.job_names)}"
         )
         return 0
 

scripts/quickstart_lifecycle.py

@@ -100,6 +100,38 @@ def _console_script(root: Path) -> Path:
     )
 
 
+def _inspect_opencv_provider(
+    python: Path,
+    *,
+    cwd: Path,
+    env: dict[str, str],
+    log: Path,
+) -> str:
+    """Require one reviewed distribution to own the installed ``cv2`` package."""
+
+    probe = _run(
+        [
+            str(python),
+            "-c",
+            (
+                "import importlib.metadata as m, json; "
+                "names={d.metadata['Name'].lower() for d in m.distributions() "
+                "if d.metadata.get('Name')}; "
+                "providers=sorted(names & {'opencv-python','opencv-python-headless',"
+                "'opencv-contrib-python','opencv-contrib-python-headless'}); "
+                "import cv2; "
+                "print(json.dumps({'providers':providers,'cv2_version':cv2.__version__})); "
+                "assert providers == ['opencv-python'], providers"
+            ),
+        ],
+        cwd=cwd,
+        env=env,
+        log=log,
+    )
+    payload = json.loads(probe.stdout.splitlines()[-1])
+    return str(payload["providers"][0])
+
+
 def _load_report(path: Path) -> dict:
     if not path.is_file():
         raise AssertionError(f"missing machine-readable run report: {path}")
@@ -290,14 +322,20 @@ def run_lifecycle(
             log=logs / "01-install.log",
         )
         installed = True
+        summary["opencv_provider"] = _inspect_opencv_provider(
+            python,
+            cwd=artifacts,
+            env=env,
+            log=logs / "02-opencv-provider.log",
+        )
         console = _console_script(venv_dir)
         if not console.is_file():
             raise AssertionError(f"console entry point was not installed: {console}")
         _run(
             [str(console), "--help"],
             cwd=artifacts,
             env=env,
-            log=logs / "02-cli-help.log",
+            log=logs / "03-cli-help.log",
         )
 
         # Linux needs host libraries that the ordinary unprivileged first-run
@@ -312,7 +350,7 @@ def run_lifecycle(
                 browser_command,
                 cwd=artifacts,
                 env=env,
-                log=logs / "03-browser-install.log",
+                log=logs / "04-browser-install.log",
             )
 
         cli = [str(python), "-m", "openadapt_flow"]
@@ -322,7 +360,7 @@ def run_lifecycle(
             [*cli, "demo-record", "--out", str(recording)],
             cwd=artifacts,
             env=env,
-            log=logs / "04-record.log",
+            log=logs / "05-record.log",
         )
         _run(
             [
@@ -336,7 +374,7 @@ def run_lifecycle(
             ],
             cwd=artifacts,
             env=env,
-            log=logs / "05-compile.log",
+            log=logs / "06-compile.log",
         )
 
         # The bundled tutorial is deliberately not production-certified. The
@@ -349,20 +387,20 @@ def run_lifecycle(
             [*cli, "lint", str(bundle), "--strict"],
             cwd=artifacts,
             env=env,
-            log=logs / "06-strict-lint-expected-refusal.log",
+            log=logs / "07-strict-lint-expected-refusal.log",
             expected=1,
         )
         _run(
             [*cli, "certify", str(bundle), "--policy", "permissive"],
             cwd=artifacts,
             env=env,
-            log=logs / "07-certify-permissive.log",
+            log=logs / "08-certify-permissive.log",
         )
         _run(
             [*cli, "certify", str(bundle), "--policy", "clinical-write"],
             cwd=artifacts,
             env=env,
-            log=logs / "08-certify-clinical-expected-refusal.log",
+            log=logs / "09-certify-clinical-expected-refusal.log",
             expected=2,
         )
         _run(
@@ -375,7 +413,7 @@ def run_lifecycle(
             ],
             cwd=artifacts,
             env=env,
-            log=logs / "09-replay-baseline.log",
+            log=logs / "10-replay-baseline.log",
         )
         _run(
             [
@@ -391,15 +429,15 @@ def run_lifecycle(
             ],
             cwd=artifacts,
             env=env,
-            log=logs / "10-replay-drift.log",
+            log=logs / "11-replay-drift.log",
         )
         # The COMPOSED free path. Every command above passed while this loop
         # was broken; only running it end to end catches that.
         _run(
             [*cli, "tutorial", "--out", str(artifacts / "tutorial")],
             cwd=artifacts,
             env=env,
-            log=logs / "11-tutorial-verified.log",
+            log=logs / "12-tutorial-verified.log",
         )
         summary.update(_inspect_artifacts(artifacts))
     finally:
@@ -408,7 +446,7 @@ def run_lifecycle(
                 [str(python), "-m", "pip", "uninstall", "-y", "openadapt-flow"],
                 cwd=artifacts,
                 env=env,
-                log=logs / "12-uninstall.log",
+                log=logs / "13-uninstall.log",
             )
             probe = _run(
                 [
@@ -421,7 +459,7 @@ def run_lifecycle(
                 ],
                 cwd=artifacts,
                 env=env,
-                log=logs / "13-uninstall-probe.log",
+                log=logs / "14-uninstall-probe.log",
             )
             summary["uninstall_verified"] = probe.returncode == 0
         (work_dir / "summary.json").write_text(