External Validation Matrix #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: External Validation Matrix | |
| on: | |
| schedule: | |
| - cron: '0 6 * * 1' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| checks: write | |
| jobs: | |
| matrix_consumer: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| scenario: | |
| - check_strict_block | |
| - check_advisory_allow | |
| - mvp_manifest_allow | |
| - external_pilot_advisory | |
| - check_strict_emit_check | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install package | |
| run: pip install -e '.[dev]' | |
| - name: Load scenario | |
| id: scenario | |
| run: | | |
| python - <<'PY' | |
| import json | |
| from pathlib import Path | |
| scenarios = json.loads(Path("benchmarks/external_validation/scenarios.json").read_text()) | |
| target = "${{ matrix.scenario }}" | |
| match = next(s for s in scenarios["scenarios"] if s["id"] == target) | |
| def emit(name: str, value: str) -> None: | |
| with open("${GITHUB_OUTPUT}", "a", encoding="utf-8") as handle: | |
| handle.write(f"{name}={value}\n") | |
| emit("mode", match.get("mode", "advisory")) | |
| emit("expected_recommendation", match.get("expected_recommendation", "")) | |
| emit("expect_exit_code", str(match.get("expect_exit_code", 0))) | |
| emit("validate_bundle", str(match.get("validate_bundle", False)).lower()) | |
| inputs = dict(match.get("action_inputs", {})) | |
| flat_fallback = { | |
| "use-check": "true" if match.get("kind") == "action_check" else "false", | |
| "changed-files": match.get("changed_files", ""), | |
| "metadata": match.get("metadata", ""), | |
| "verification-manifest": match.get("verification_manifest", ""), | |
| "bundle-output-dir": match.get("bundle_output_dir", "external-validation-bundle"), | |
| "emit-check": "false", | |
| } | |
| for key, value in flat_fallback.items(): | |
| inputs.setdefault(key, value) | |
| for key, value in inputs.items(): | |
| emit(key.replace("-", "_"), str(value)) | |
| PY | |
| - name: Run OVK Action scenario | |
| id: ovk | |
| uses: ./ | |
| with: | |
| mode: ${{ steps.scenario.outputs.mode == 'strict' && 'strict' || 'advisory' }} | |
| use-check: ${{ steps.scenario.outputs.use_check || 'false' }} | |
| changed-files: ${{ steps.scenario.outputs.changed_files || '' }} | |
| metadata: ${{ steps.scenario.outputs.metadata || '' }} | |
| verification-manifest: ${{ steps.scenario.outputs.verification_manifest || '' }} | |
| bundle-output-dir: ${{ steps.scenario.outputs.bundle_output_dir || 'external-validation-bundle' }} | |
| emit-check: ${{ steps.scenario.outputs.emit_check || 'false' }} | |
| post-comment: 'false' | |
| continue-on-error: ${{ steps.scenario.outputs.expect_exit_code != '0' }} | |
| - name: Assert scenario recommendation and exit behavior | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| expected = "${{ steps.scenario.outputs.expected_recommendation }}" | |
| expect_exit = int("${{ steps.scenario.outputs.expect_exit_code }}") | |
| outcome = "${{ steps.ovk.outcome }}" | |
| actual_exit = 0 if outcome == "success" else 1 | |
| evidence_paths = [ | |
| Path("ovk-evidence.json"), | |
| Path("external-validation-mvp-bundle/ovk-evidence.json"), | |
| Path("external-validation-bundle/ovk-evidence.json"), | |
| ] | |
| evidence_path = next((path for path in evidence_paths if path.exists()), None) | |
| assert evidence_path is not None, "expected ovk-evidence.json artifact" | |
| recommendation = json.loads(evidence_path.read_text(encoding="utf-8"))["decision"]["merge_recommendation"] | |
| assert recommendation == expected, f"expected recommendation {expected}, got {recommendation}" | |
| assert actual_exit == (0 if expect_exit == 0 else 1), ( | |
| f"expected exit behavior {expect_exit}, step outcome was {outcome}" | |
| ) | |
| print(f"scenario ${{ matrix.scenario }} passed: {recommendation}, exit={actual_exit}") | |
| PY | |
| - name: Validate MVP bundle when required | |
| if: steps.scenario.outputs.validate_bundle == 'true' | |
| run: ovk validate-outputs external-validation-mvp-bundle | |
| forged_bundle_rejected: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install package | |
| run: pip install -e '.[dev]' | |
| - name: Assert adversarial bundle fails quality gate | |
| run: | | |
| python - <<'PY' | |
| from ovk.core.evidence_quality import build_evidence_quality_report | |
| from ovk.core.json_io import read_json_file | |
| from ovk.core.models import EvidenceBundle | |
| bundle = EvidenceBundle.model_validate(read_json_file("examples/evidence_quality/adversarial_allow_with_fail.json")) | |
| assert not build_evidence_quality_report(bundle).passed | |
| print("forged bundle correctly rejected") | |
| PY | |
| matrix_release_pin: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Document release pin consumer path | |
| run: | | |
| echo "Fork consumers should pin: uses: fraware/open-verification-kernel@v1.2.0" | |
| test -f examples/github_workflows/external_consumer.yml | |
| attestation_smoke: | |
| runs-on: ubuntu-latest | |
| env: | |
| OVK_SIGNING_KEY: external-validation-smoke-key | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install package | |
| run: pip install -e '.[dev]' | |
| - name: Strict signed bundle smoke | |
| run: | | |
| ovk release-bundle \ | |
| --lane infrastructure \ | |
| --input examples/infrastructure_exposure/input_private_sensitive_resource.json \ | |
| --output-dir external-validation-signed-bundle | |
| ovk validate-outputs external-validation-signed-bundle | |
| upload_artifacts: | |
| runs-on: ubuntu-latest | |
| needs: [matrix_consumer, forged_bundle_rejected, attestation_smoke] | |
| if: always() | |
| steps: | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: external-validation-logs | |
| path: | | |
| benchmarks/external_validation/scenarios.json | |
| if-no-files-found: ignore |