openadapt-evals/.github/workflows/complex-visual.yml at main · OpenAdaptAI/openadapt-evals · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
name: complex-visual

on:
  pull_request:
    paths:
      - benchmark/complex_visual/**
      - tests/test_complex_visual_campaign.py
      - .github/workflows/complex-visual.yml
  workflow_dispatch:

concurrency:
  group: complex-visual-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  headed-pixel-campaign:
    runs-on: ubuntu-latest
    timeout-minutes: 15

    steps:
      - name: Checkout repository
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: '3.12'

      - name: Install uv
        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
        with:
          version: "0.11.29"

      - name: Install dependencies
        run: uv sync --locked --extra dev --no-sources

      - name: Ensure local X11 fixture tools
        # The job budget is 15 minutes and the campaign itself is the point of
        # it. Without a step bound, one hung apt call consumes all 15 and the
        # real work never runs. The equivalent apt step in openadapt-capture
        # takes 19s, so 5 minutes covers three update attempts plus 30s of
        # backoff with wide margin, and leaves at least 10 minutes for the
        # dependency install and the campaign.
        timeout-minutes: 5
        run: |
          set -uo pipefail
          if ! command -v Xvfb >/dev/null || ! command -v xvfb-run >/dev/null; then
            # The hosted runner resolves its Ubuntu mirror through
            # /etc/apt/apt-mirrors.txt, which points at azure.archive.ubuntu.com.
            # That mirror fails intermittently, and each failure costs minutes of
            # apt retries. Prefer the canonical archive. Best-effort: an absent
            # or already-canonical file changes nothing.
            sudo sed -i \
              's|http://azure.archive.ubuntu.com/ubuntu|http://archive.ubuntu.com/ubuntu|g' \
              /etc/apt/apt-mirrors.txt 2>/dev/null || true
            update_ok=""
            for attempt in 1 2 3; do
              if sudo apt-get update; then
                update_ok=1
                break
              fi
              echo "::warning::apt-get update failed (attempt ${attempt}/3); retrying"
              sleep $((attempt * 10))
            done
            if [ -z "$update_ok" ]; then
              echo "::error::apt-get update failed three times; the Ubuntu mirror is unreachable"
              exit 1
            fi
            set -e
            sudo apt-get install --yes xvfb xauth
          fi
          set -e
          python -c 'import tkinter'

      - name: Create dedicated actor identity
        run: |
          sudo useradd --system --no-create-home --shell /usr/sbin/nologin openadapt-actor
          test "$(id -u openadapt-actor)" != "$(id -u)"

      - name: Prepare read-only actor runtime
        env:
          ACTOR_RUNTIME: /tmp/openadapt-actor-runtime-${{ github.run_id }}
        run: |
          sudo install -d -o root -g root -m 0755 "$ACTOR_RUNTIME/source"
          sudo cp -a .venv "$ACTOR_RUNTIME/venv"
          sudo cp -a benchmark "$ACTOR_RUNTIME/source/benchmark"
          sudo chown -R root:root "$ACTOR_RUNTIME"
          sudo chmod -R u=rwX,go=rX "$ACTOR_RUNTIME"

      - name: Run headed pixel campaign
        env:
          COMPLEX_VISUAL_ACTOR_USER: openadapt-actor
          COMPLEX_VISUAL_ACTOR_PYTHON: /tmp/openadapt-actor-runtime-${{ github.run_id }}/venv/bin/python
          COMPLEX_VISUAL_ACTOR_SOURCE: /tmp/openadapt-actor-runtime-${{ github.run_id }}/source
          COMPLEX_VISUAL_OUTPUT: /tmp/complex-visual-artifacts-${{ github.run_id }}
        run: >-
          xvfb-run --auto-servernum --server-args="-screen 0 1280x700x24 -ac"
          uv run --no-sources pytest -q
          tests/test_complex_visual_campaign.py

      - name: Upload exact campaign evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: complex-visual-${{ github.sha }}
          path: /tmp/complex-visual-artifacts-${{ github.run_id }}
          if-no-files-found: error
          retention-days: 7