SPOG Integration Tests #16
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPOG Integration Tests for dbt-databricks. | |
| # | |
| # Mirrors integration.yml's matrix (prepare-shards + 3 sharded functional | |
| # jobs) on a weekly Sunday schedule, but with the host name and http_path | |
| # wired to the SPOG vanity host + ?o= so every test path exercises SPOG | |
| # routing. PR-targeting / status-reporting logic is omitted — this is a | |
| # scheduled smoke, not a PR gate. | |
| name: SPOG Integration Tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "30 21 * * 0" # Weekly: Sunday 21:30 UTC (Monday 03:00 IST). | |
| permissions: | |
| id-token: write | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| prepare-shards: | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| env: | |
| UV_FROZEN: "1" | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Setup Python Dependencies | |
| id: deps | |
| uses: ./.github/actions/setup-python-deps | |
| - name: Setup JFrog PyPI Proxy (fallback) | |
| if: steps.deps.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/setup-jfrog-pypi | |
| - name: Set up Python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 | |
| with: | |
| version: "0.11.18" | |
| cache-local-path: ~/.cache/uv | |
| - name: Install Hatch | |
| uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # install | |
| with: | |
| version: "1.17.0" | |
| - name: Create Hatch environment | |
| run: hatch env create default | |
| - name: Collect tests and assign to shards | |
| run: | | |
| set -euo pipefail | |
| mkdir -p shard-assignments | |
| declare -A NUM_SHARDS=( | |
| [databricks_cluster]=2 | |
| [databricks_uc_cluster]=3 | |
| [databricks_uc_sql_endpoint]=3 | |
| ) | |
| # Capture full output per profile so collection failures are visible. | |
| declare -A PIDS | |
| for PROFILE in "${!NUM_SHARDS[@]}"; do | |
| hatch run pytest --collect-only -q --profile "$PROFILE" tests/functional \ | |
| > "shard-assignments/${PROFILE}-raw.log" 2>&1 & | |
| PIDS[$PROFILE]=$! | |
| done | |
| COLLECT_FAILED=0 | |
| for PROFILE in "${!NUM_SHARDS[@]}"; do | |
| RC=0 | |
| wait "${PIDS[$PROFILE]}" || RC=$? | |
| if [ "$RC" -ne 0 ]; then | |
| echo "::error::Test collection failed for profile ${PROFILE} (exit ${RC}); output below:" | |
| cat "shard-assignments/${PROFILE}-raw.log" | |
| COLLECT_FAILED=1 | |
| continue | |
| fi | |
| grep "::" "shard-assignments/${PROFILE}-raw.log" \ | |
| > "shard-assignments/${PROFILE}-collected.txt" || true | |
| done | |
| [ "$COLLECT_FAILED" -eq 0 ] || exit 1 | |
| for PROFILE in "${!NUM_SHARDS[@]}"; do | |
| python3 scripts/shard_assign.py \ | |
| --profile "$PROFILE" \ | |
| --num-shards "${NUM_SHARDS[$PROFILE]}" \ | |
| --input "shard-assignments/${PROFILE}-collected.txt" \ | |
| --output-dir shard-assignments \ | |
| --algo lpt_historical_time \ | |
| --timings .github/test_timings.json | |
| done | |
| - name: Upload shard assignments | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: shard-assignments-spog | |
| path: shard-assignments/ | |
| retention-days: 5 | |
| uc-cluster: | |
| needs: prepare-shards | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [0, 1, 2] | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| environment: azure-prod | |
| env: | |
| DBT_DATABRICKS_HOST_NAME: ${{ secrets.TEST_PECO_SPOG_HOST }} | |
| DBT_DATABRICKS_CLIENT_ID: ${{ secrets.TEST_PECO_SP_ID }} | |
| DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }} | |
| DBT_DATABRICKS_UC_INITIAL_CATALOG: peco | |
| DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test | |
| TEST_PECO_SPOG_WORKSPACE_ID: ${{ secrets.TEST_PECO_SPOG_WORKSPACE_ID }} | |
| TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }} | |
| TEST_PECO_SPOG_NATIVE: "1" | |
| UV_FROZEN: "1" | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Setup Python Dependencies | |
| id: deps | |
| uses: ./.github/actions/setup-python-deps | |
| - name: Setup JFrog PyPI Proxy (fallback) | |
| if: steps.deps.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/setup-jfrog-pypi | |
| - name: Set up python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Get http path from environment | |
| run: python .github/workflows/build_cluster_http_path.py | |
| shell: sh | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 | |
| with: | |
| version: "0.11.18" | |
| cache-local-path: ~/.cache/uv | |
| - name: Install Hatch | |
| uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # install | |
| with: | |
| version: "1.17.0" | |
| - name: Download shard assignments | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 | |
| with: | |
| name: shard-assignments-spog | |
| path: shard-assignments/ | |
| - name: Resolve test list for this shard | |
| run: | | |
| set -euo pipefail | |
| SHARD_FILE="shard-assignments/databricks_uc_cluster-shard-${{ matrix.shard }}.txt" | |
| if [ ! -s "$SHARD_FILE" ]; then | |
| echo "::error::Shard file missing or empty: $SHARD_FILE" | |
| exit 1 | |
| fi | |
| echo "SHARD_TESTS_FILE=$SHARD_FILE" >> "$GITHUB_ENV" | |
| echo "Files in shard ${{ matrix.shard }}: $(wc -l < "$SHARD_FILE")" | |
| - name: Run UC Cluster Functional Tests (shard ${{ matrix.shard }}) | |
| run: | | |
| mkdir -p logs | |
| DBT_TEST_USER=notnecessaryformosttests@example.com \ | |
| xargs -r hatch -v run pytest \ | |
| --color=yes -v \ | |
| --profile databricks_uc_cluster \ | |
| -n 10 --dist=loadfile \ | |
| --reruns 2 --reruns-delay 120 \ | |
| --junitxml=logs/junit-uc-cluster-shard-${{ matrix.shard }}.xml \ | |
| < "$SHARD_TESTS_FILE" | |
| - name: Upload UC Cluster Test Logs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: spog-uc-cluster-logs-shard-${{ matrix.shard }} | |
| path: logs/ | |
| retention-days: 14 | |
| uc-sql-endpoint: | |
| needs: prepare-shards | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [0, 1, 2] | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| environment: azure-prod | |
| env: | |
| DBT_DATABRICKS_HOST_NAME: ${{ secrets.TEST_PECO_SPOG_HOST }} | |
| DBT_DATABRICKS_CLIENT_ID: ${{ secrets.TEST_PECO_SP_ID }} | |
| DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }} | |
| DBT_DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}?o=${{ secrets.TEST_PECO_SPOG_WORKSPACE_ID }} | |
| DBT_DATABRICKS_UC_INITIAL_CATALOG: peco | |
| DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test | |
| TEST_PECO_SPOG_WORKSPACE_ID: ${{ secrets.TEST_PECO_SPOG_WORKSPACE_ID }} | |
| TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }} | |
| TEST_PECO_SPOG_NATIVE: "1" | |
| UV_FROZEN: "1" | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Setup Python Dependencies | |
| id: deps | |
| uses: ./.github/actions/setup-python-deps | |
| - name: Setup JFrog PyPI Proxy (fallback) | |
| if: steps.deps.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/setup-jfrog-pypi | |
| - name: Set up python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Get http path from environment | |
| run: python .github/workflows/build_cluster_http_path.py | |
| shell: sh | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 | |
| with: | |
| version: "0.11.18" | |
| cache-local-path: ~/.cache/uv | |
| - name: Install Hatch | |
| uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # install | |
| with: | |
| version: "1.17.0" | |
| - name: Download shard assignments | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 | |
| with: | |
| name: shard-assignments-spog | |
| path: shard-assignments/ | |
| - name: Resolve test list for this shard | |
| run: | | |
| set -euo pipefail | |
| SHARD_FILE="shard-assignments/databricks_uc_sql_endpoint-shard-${{ matrix.shard }}.txt" | |
| if [ ! -s "$SHARD_FILE" ]; then | |
| echo "::error::Shard file missing or empty: $SHARD_FILE" | |
| exit 1 | |
| fi | |
| echo "SHARD_TESTS_FILE=$SHARD_FILE" >> "$GITHUB_ENV" | |
| echo "Files in shard ${{ matrix.shard }}: $(wc -l < "$SHARD_FILE")" | |
| - name: Run Sql Endpoint Functional Tests (shard ${{ matrix.shard }}) | |
| run: | | |
| mkdir -p logs | |
| DBT_TEST_USER=notnecessaryformosttests@example.com \ | |
| xargs -r hatch -v run pytest \ | |
| --color=yes -v \ | |
| --profile databricks_uc_sql_endpoint \ | |
| -n 10 --dist=loadfile \ | |
| --reruns 2 --reruns-delay 120 \ | |
| --junitxml=logs/junit-uc-sql-endpoint-shard-${{ matrix.shard }}.xml \ | |
| < "$SHARD_TESTS_FILE" | |
| - name: Upload SQL Endpoint Test Logs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: spog-uc-sql-endpoint-logs-shard-${{ matrix.shard }} | |
| path: logs/ | |
| retention-days: 14 | |
| cluster: | |
| needs: prepare-shards | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [0, 1] | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| environment: azure-prod | |
| env: | |
| DBT_DATABRICKS_HOST_NAME: ${{ secrets.TEST_PECO_SPOG_HOST }} | |
| DBT_DATABRICKS_CLIENT_ID: ${{ secrets.TEST_PECO_SP_ID }} | |
| DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }} | |
| DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test | |
| TEST_PECO_SPOG_WORKSPACE_ID: ${{ secrets.TEST_PECO_SPOG_WORKSPACE_ID }} | |
| TEST_PECO_CLUSTER_ID: ${{ secrets.TEST_PECO_CLUSTER_ID }} | |
| TEST_PECO_SPOG_NATIVE: "1" | |
| UV_FROZEN: "1" | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Setup Python Dependencies | |
| id: deps | |
| uses: ./.github/actions/setup-python-deps | |
| - name: Setup JFrog PyPI Proxy (fallback) | |
| if: steps.deps.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/setup-jfrog-pypi | |
| - name: Set up python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Get http path from environment | |
| run: python .github/workflows/build_cluster_http_path.py | |
| shell: sh | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 | |
| with: | |
| version: "0.11.18" | |
| cache-local-path: ~/.cache/uv | |
| - name: Install Hatch | |
| uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # install | |
| with: | |
| version: "1.17.0" | |
| - name: Download shard assignments | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 | |
| with: | |
| name: shard-assignments-spog | |
| path: shard-assignments/ | |
| - name: Resolve test list for this shard | |
| run: | | |
| set -euo pipefail | |
| SHARD_FILE="shard-assignments/databricks_cluster-shard-${{ matrix.shard }}.txt" | |
| if [ ! -s "$SHARD_FILE" ]; then | |
| echo "::error::Shard file missing or empty: $SHARD_FILE" | |
| exit 1 | |
| fi | |
| echo "SHARD_TESTS_FILE=$SHARD_FILE" >> "$GITHUB_ENV" | |
| echo "Files in shard ${{ matrix.shard }}: $(wc -l < "$SHARD_FILE")" | |
| - name: Run Cluster Functional Tests (shard ${{ matrix.shard }}) | |
| run: | | |
| mkdir -p logs | |
| DBT_TEST_USER=notnecessaryformosttests@example.com \ | |
| DBT_DATABRICKS_HTTP_PATH="$DBT_DATABRICKS_CLUSTER_HTTP_PATH" \ | |
| xargs -r hatch -v run pytest \ | |
| --color=yes -v \ | |
| --profile databricks_cluster \ | |
| -n 10 --dist=loadfile \ | |
| --reruns 2 --reruns-delay 120 \ | |
| --junitxml=logs/junit-cluster-shard-${{ matrix.shard }}.xml \ | |
| < "$SHARD_TESTS_FILE" | |
| - name: Upload Cluster Test Logs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: spog-cluster-logs-shard-${{ matrix.shard }} | |
| path: logs/ | |
| retention-days: 14 |