Integration Tests (Min-Deps) #103
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Integration Tests (Min-Deps) for dbt-databricks | |
| # | |
| # Sharded functional tests against the lowest-direct dependency resolution | |
| # committed in requirements.lowest-direct.txt. Mirrors integration.yml's | |
| # fan-out (skip-if-unchanged on schedule, prepare-shards via LPT historical | |
| # timing, sharded execution per profile, gather-shards verification) but | |
| # routes every pytest invocation through the `min-deps:` hatch env. | |
| # | |
| # Triggering: | |
| # 1. On a PR (internal OR fork): a maintainer comments `/integration-test min-deps`. | |
| # The integration-trigger workflow validates the comment author and | |
| # dispatches this workflow with the PR number. The run posts a result | |
| # comment back to the PR when the matrix completes. | |
| # 2. Manually from the Actions tab (workflow_dispatch): | |
| # - One PR (e.g. "100") OR comma-separated list ("100,200,300") in pr_numbers. | |
| # - Or a git_ref for ad-hoc testing. | |
| # 3. Nightly on `main` at 19:30 UTC (2h before integration.yml's 21:30 nightly). | |
| # The `prepare` job short-circuits if the current main SHA has already had a | |
| # successful min-deps integration run, emitting an empty targets array so | |
| # the matrix jobs skip cleanly. | |
| name: Integration Tests (Min-Deps) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pr_numbers: | |
| description: "PR number(s) to test — single PR or comma-separated for batch (e.g. '100' or '100,200,300')" | |
| required: false | |
| type: string | |
| git_ref: | |
| description: "Git ref (branch/tag/commit) to test — used only when pr_numbers is empty" | |
| required: false | |
| type: string | |
| commit_sha: | |
| description: "Immutable 40-character commit SHA for a single PR — set by the integration-test trigger" | |
| required: false | |
| type: string | |
| schedule: | |
| - cron: "30 19 * * *" # 19:30 UTC, 2h before integration.yml's nightly | |
| permissions: | |
| id-token: write | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.inputs.pr_numbers || github.event.inputs.git_ref || github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| prepare: | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| outputs: | |
| targets: ${{ steps.parse.outputs.targets }} | |
| shards_cluster: ${{ steps.parse.outputs.shards_cluster }} | |
| shards_uc_cluster: ${{ steps.parse.outputs.shards_uc_cluster }} | |
| shards_uc_sql_endpoint: ${{ steps.parse.outputs.shards_uc_sql_endpoint }} | |
| count_cluster: ${{ steps.parse.outputs.count_cluster }} | |
| count_uc_cluster: ${{ steps.parse.outputs.count_uc_cluster }} | |
| count_uc_sql_endpoint: ${{ steps.parse.outputs.count_uc_sql_endpoint }} | |
| steps: | |
| - name: Parse targets | |
| id: parse | |
| shell: bash | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| INPUT_PR_NUMBERS: ${{ github.event.inputs.pr_numbers }} | |
| INPUT_GIT_REF: ${{ github.event.inputs.git_ref }} | |
| INPUT_COMMIT_SHA: ${{ github.event.inputs.commit_sha }} | |
| DEFAULT_REF: ${{ github.ref }} | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| entry() { printf '{"pr":"%s","ref":"%s"}' "$1" "$2"; } | |
| targets="[" | |
| if [[ "$EVENT_NAME" == "schedule" ]]; then | |
| # Nightly skip-if-unchanged: if this main SHA already has a green | |
| # min-deps integration run, emit empty targets so the matrix jobs skip. | |
| already_tested=$(curl -sfS \ | |
| -H "Authorization: Bearer $GH_TOKEN" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/workflows/integration-min-deps.yml/runs?branch=main&status=success&head_sha=$GITHUB_SHA" \ | |
| | jq -r '.total_count // 0') | |
| if [[ "$already_tested" -gt 0 ]]; then | |
| echo "Nightly skip: main @ $GITHUB_SHA already has $already_tested successful run(s)." | |
| else | |
| targets+=$(entry "nightly" "$DEFAULT_REF") | |
| fi | |
| elif [[ -n "${INPUT_COMMIT_SHA//[[:space:]]/}" ]]; then | |
| pr_trimmed="${INPUT_PR_NUMBERS//[[:space:]]/}" | |
| if [[ ! "$pr_trimmed" =~ ^[0-9]+$ ]]; then | |
| echo "::error::commit_sha requires exactly one numeric PR number, got pr_numbers='$INPUT_PR_NUMBERS'." | |
| exit 1 | |
| fi | |
| if [[ ! "$INPUT_COMMIT_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then | |
| echo "::error::Invalid commit_sha '$INPUT_COMMIT_SHA' — expected a 40-character hexadecimal SHA." | |
| exit 1 | |
| fi | |
| targets+=$(entry "$pr_trimmed" "$INPUT_COMMIT_SHA") | |
| elif [[ -n "${INPUT_PR_NUMBERS//[[:space:]]/}" ]]; then | |
| first=1 | |
| IFS=',' read -ra prs <<< "$INPUT_PR_NUMBERS" | |
| for pr in "${prs[@]}"; do | |
| pr_trimmed="${pr//[[:space:]]/}" | |
| [[ -z "$pr_trimmed" ]] && continue | |
| if [[ ! "$pr_trimmed" =~ ^[0-9]+$ ]]; then | |
| echo "::error::Invalid PR number '$pr_trimmed' in pr_numbers='$INPUT_PR_NUMBERS' — expected digits, comma-separated." | |
| exit 1 | |
| fi | |
| [[ $first -eq 0 ]] && targets+="," | |
| first=0 | |
| targets+=$(entry "$pr_trimmed" "refs/pull/$pr_trimmed/head") | |
| done | |
| elif [[ -n "${INPUT_GIT_REF//[[:space:]]/}" ]]; then | |
| targets+=$(entry "manual" "$INPUT_GIT_REF") | |
| else | |
| targets+=$(entry "manual" "$DEFAULT_REF") | |
| fi | |
| targets+="]" | |
| echo "targets=$targets" >> "$GITHUB_OUTPUT" | |
| echo "Parsed targets: $targets" | |
| # Shard fan-out — same shape as integration.yml. Reshape here and | |
| # matrix + max-parallel + prepare-shards NUM_SHARDS all follow. | |
| shards_cluster='[0, 1]' | |
| shards_uc_cluster='[0, 1, 2]' | |
| shards_uc_sql_endpoint='[0, 1, 2]' | |
| for profile in cluster uc_cluster uc_sql_endpoint; do | |
| arr_var="shards_${profile}" | |
| arr="${!arr_var}" | |
| echo "shards_${profile}=${arr}" >> "$GITHUB_OUTPUT" | |
| echo "count_${profile}=$(jq 'length' <<< "$arr")" >> "$GITHUB_OUTPUT" | |
| done | |
| # Collects test ids per profile under the min-deps env and partitions | |
| # files into shards. `--collect-only` imports test modules but never | |
| # hits the workspace. | |
| prepare-shards: | |
| needs: prepare | |
| if: needs.prepare.outputs.targets != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: ${{ fromJSON(needs.prepare.outputs.targets) }} | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| env: | |
| UV_FROZEN: "1" | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| ref: ${{ matrix.target.ref }} | |
| - name: Setup Python Dependencies | |
| id: deps | |
| uses: ./.github/actions/setup-python-deps | |
| - name: Setup JFrog PyPI Proxy (fallback) | |
| if: steps.deps.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/setup-jfrog-pypi | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 | |
| with: | |
| version: "0.11.18" | |
| cache-local-path: ~/.cache/uv | |
| - name: Install Hatch | |
| uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # install | |
| with: | |
| version: "1.17.0" | |
| # Build the env once, serially: otherwise the three backgrounded | |
| # `hatch run min-deps:` calls below race to create it on a cold runner. | |
| - name: Create min-deps env | |
| run: hatch env create min-deps | |
| - name: Collect tests and assign to shards | |
| run: | | |
| set -euo pipefail | |
| mkdir -p shard-assignments | |
| declare -A NUM_SHARDS=( | |
| [databricks_cluster]=${{ needs.prepare.outputs.count_cluster }} | |
| [databricks_uc_cluster]=${{ needs.prepare.outputs.count_uc_cluster }} | |
| [databricks_uc_sql_endpoint]=${{ needs.prepare.outputs.count_uc_sql_endpoint }} | |
| ) | |
| # Capture full output per profile; don't pipe through grep here, or a | |
| # failure's exit code is lost and surfaces only as "no valid nodeids". | |
| declare -A PIDS | |
| for PROFILE in "${!NUM_SHARDS[@]}"; do | |
| hatch run min-deps:pytest --collect-only -q --profile "$PROFILE" tests/functional \ | |
| > "shard-assignments/${PROFILE}-raw.log" 2>&1 & | |
| PIDS[$PROFILE]=$! | |
| done | |
| COLLECT_FAILED=0 | |
| for PROFILE in "${!NUM_SHARDS[@]}"; do | |
| RC=0 | |
| wait "${PIDS[$PROFILE]}" || RC=$? | |
| if [ "$RC" -ne 0 ]; then | |
| echo "::error::min-deps test collection failed for profile ${PROFILE} (exit ${RC}); output below:" | |
| cat "shard-assignments/${PROFILE}-raw.log" | |
| COLLECT_FAILED=1 | |
| continue | |
| fi | |
| grep "::" "shard-assignments/${PROFILE}-raw.log" \ | |
| > "shard-assignments/${PROFILE}-collected.txt" || true | |
| done | |
| [ "$COLLECT_FAILED" -eq 0 ] || exit 1 | |
| for PROFILE in "${!NUM_SHARDS[@]}"; do | |
| python3 scripts/shard_assign.py \ | |
| --profile "$PROFILE" \ | |
| --num-shards "${NUM_SHARDS[$PROFILE]}" \ | |
| --input "shard-assignments/${PROFILE}-collected.txt" \ | |
| --output-dir shard-assignments \ | |
| --algo lpt_historical_time \ | |
| --timings .github/test_timings.json | |
| done | |
| - name: Upload shard assignments | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: shard-assignments-${{ matrix.target.pr }} | |
| path: shard-assignments/ | |
| retention-days: 5 | |
| gate: | |
| needs: prepare | |
| if: needs.prepare.outputs.targets != '[]' | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| timeout-minutes: 240 | |
| steps: | |
| - name: Wait for slot | |
| uses: softprops/turnstyle@15f9da4059166900981058ba251e0b652511c68f # v3.2.2 | |
| with: | |
| same-branch-only: false | |
| poll-interval-seconds: 30 | |
| abort-after-seconds: 14400 | |
| run-uc-cluster-e2e-tests: | |
| needs: | |
| - prepare | |
| - prepare-shards | |
| - gate | |
| if: needs.prepare.outputs.targets != '[]' | |
| strategy: | |
| fail-fast: false | |
| max-parallel: ${{ fromJSON(needs.prepare.outputs.count_uc_cluster) }} | |
| matrix: | |
| target: ${{ fromJSON(needs.prepare.outputs.targets) }} | |
| shard: ${{ fromJSON(needs.prepare.outputs.shards_uc_cluster) }} | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| environment: azure-prod | |
| env: | |
| DBT_DATABRICKS_HOST_NAME: ${{ secrets.DATABRICKS_HOST }} | |
| DBT_DATABRICKS_CLIENT_ID: ${{ secrets.TEST_PECO_SP_ID }} | |
| DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }} | |
| DBT_DATABRICKS_UC_INITIAL_CATALOG: peco | |
| DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test | |
| TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }} | |
| UV_FROZEN: "1" | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| ref: ${{ matrix.target.ref }} | |
| - name: Setup Python Dependencies | |
| id: deps | |
| uses: ./.github/actions/setup-python-deps | |
| - name: Setup JFrog PyPI Proxy (fallback) | |
| if: steps.deps.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/setup-jfrog-pypi | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Get http path from environment | |
| run: python .github/workflows/build_cluster_http_path.py | |
| shell: sh | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 | |
| with: | |
| version: "0.11.18" | |
| cache-local-path: ~/.cache/uv | |
| - name: Install Hatch | |
| uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # install | |
| with: | |
| version: "1.17.0" | |
| - name: Download shard assignments | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 | |
| with: | |
| name: shard-assignments-${{ matrix.target.pr }} | |
| path: shard-assignments/ | |
| - name: Resolve test list for this shard | |
| run: | | |
| set -euo pipefail | |
| SHARD_FILE="shard-assignments/databricks_uc_cluster-shard-${{ matrix.shard }}.txt" | |
| if [ ! -s "$SHARD_FILE" ]; then | |
| echo "::error::Shard file missing or empty: $SHARD_FILE" | |
| exit 1 | |
| fi | |
| echo "SHARD_TESTS_FILE=$SHARD_FILE" >> "$GITHUB_ENV" | |
| echo "Files in shard ${{ matrix.shard }}: $(wc -l < "$SHARD_FILE")" | |
| - name: Run UC Cluster Functional Tests (shard ${{ matrix.shard }}) | |
| run: | | |
| mkdir -p logs | |
| DBT_TEST_USER=notnecessaryformosttests@example.com \ | |
| xargs -r hatch -v run min-deps:pytest \ | |
| --color=yes -v \ | |
| --profile databricks_uc_cluster \ | |
| -n 10 --dist=loadfile \ | |
| --reruns 1 --reruns-delay 120 \ | |
| --junitxml=logs/junit-shard-${{ matrix.shard }}.xml \ | |
| < "$SHARD_TESTS_FILE" | |
| - name: Upload UC Cluster Test Logs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: uc-cluster-test-logs-${{ matrix.target.pr }}-shard-${{ matrix.shard }} | |
| path: logs/ | |
| retention-days: 5 | |
| run-sqlwarehouse-e2e-tests: | |
| needs: | |
| - prepare | |
| - prepare-shards | |
| - gate | |
| if: needs.prepare.outputs.targets != '[]' | |
| strategy: | |
| fail-fast: false | |
| max-parallel: ${{ fromJSON(needs.prepare.outputs.count_uc_sql_endpoint) }} | |
| matrix: | |
| target: ${{ fromJSON(needs.prepare.outputs.targets) }} | |
| shard: ${{ fromJSON(needs.prepare.outputs.shards_uc_sql_endpoint) }} | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| environment: azure-prod | |
| env: | |
| DBT_DATABRICKS_HOST_NAME: ${{ secrets.DATABRICKS_HOST }} | |
| DBT_DATABRICKS_CLIENT_ID: ${{ secrets.TEST_PECO_SP_ID }} | |
| DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }} | |
| DBT_DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} | |
| DBT_DATABRICKS_UC_INITIAL_CATALOG: peco | |
| DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test | |
| TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }} | |
| UV_FROZEN: "1" | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| ref: ${{ matrix.target.ref }} | |
| - name: Setup Python Dependencies | |
| id: deps | |
| uses: ./.github/actions/setup-python-deps | |
| - name: Setup JFrog PyPI Proxy (fallback) | |
| if: steps.deps.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/setup-jfrog-pypi | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Get http path from environment | |
| run: python .github/workflows/build_cluster_http_path.py | |
| shell: sh | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 | |
| with: | |
| version: "0.11.18" | |
| cache-local-path: ~/.cache/uv | |
| - name: Install Hatch | |
| uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # install | |
| with: | |
| version: "1.17.0" | |
| - name: Download shard assignments | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 | |
| with: | |
| name: shard-assignments-${{ matrix.target.pr }} | |
| path: shard-assignments/ | |
| - name: Resolve test list for this shard | |
| run: | | |
| set -euo pipefail | |
| SHARD_FILE="shard-assignments/databricks_uc_sql_endpoint-shard-${{ matrix.shard }}.txt" | |
| if [ ! -s "$SHARD_FILE" ]; then | |
| echo "::error::Shard file missing or empty: $SHARD_FILE" | |
| exit 1 | |
| fi | |
| echo "SHARD_TESTS_FILE=$SHARD_FILE" >> "$GITHUB_ENV" | |
| echo "Files in shard ${{ matrix.shard }}: $(wc -l < "$SHARD_FILE")" | |
| - name: Run Sql Endpoint Functional Tests (shard ${{ matrix.shard }}) | |
| run: | | |
| mkdir -p logs | |
| DBT_TEST_USER=notnecessaryformosttests@example.com \ | |
| xargs -r hatch -v run min-deps:pytest \ | |
| --color=yes -v \ | |
| --profile databricks_uc_sql_endpoint \ | |
| -n 10 --dist=loadfile \ | |
| --reruns 1 --reruns-delay 120 \ | |
| --junitxml=logs/junit-shard-${{ matrix.shard }}.xml \ | |
| < "$SHARD_TESTS_FILE" | |
| - name: Upload SQL Endpoint Test Logs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: sql-endpoint-test-logs-${{ matrix.target.pr }}-shard-${{ matrix.shard }} | |
| path: logs/ | |
| retention-days: 5 | |
| run-cluster-e2e-tests: | |
| needs: | |
| - prepare | |
| - prepare-shards | |
| - gate | |
| if: needs.prepare.outputs.targets != '[]' | |
| strategy: | |
| fail-fast: false | |
| max-parallel: ${{ fromJSON(needs.prepare.outputs.count_cluster) }} | |
| matrix: | |
| target: ${{ fromJSON(needs.prepare.outputs.targets) }} | |
| shard: ${{ fromJSON(needs.prepare.outputs.shards_cluster) }} | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| environment: azure-prod | |
| env: | |
| DBT_DATABRICKS_HOST_NAME: ${{ secrets.DATABRICKS_HOST }} | |
| DBT_DATABRICKS_CLIENT_ID: ${{ secrets.TEST_PECO_SP_ID }} | |
| DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }} | |
| TEST_PECO_CLUSTER_ID: ${{ secrets.TEST_PECO_CLUSTER_ID }} | |
| DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test | |
| UV_FROZEN: "1" | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| ref: ${{ matrix.target.ref }} | |
| - name: Setup Python Dependencies | |
| id: deps | |
| uses: ./.github/actions/setup-python-deps | |
| - name: Setup JFrog PyPI Proxy (fallback) | |
| if: steps.deps.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/setup-jfrog-pypi | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Get http path from environment | |
| run: python .github/workflows/build_cluster_http_path.py | |
| shell: sh | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4 | |
| with: | |
| version: "0.11.18" | |
| cache-local-path: ~/.cache/uv | |
| - name: Install Hatch | |
| uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # install | |
| with: | |
| version: "1.17.0" | |
| - name: Download shard assignments | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 | |
| with: | |
| name: shard-assignments-${{ matrix.target.pr }} | |
| path: shard-assignments/ | |
| - name: Resolve test list for this shard | |
| run: | | |
| set -euo pipefail | |
| SHARD_FILE="shard-assignments/databricks_cluster-shard-${{ matrix.shard }}.txt" | |
| if [ ! -s "$SHARD_FILE" ]; then | |
| echo "::error::Shard file missing or empty: $SHARD_FILE" | |
| exit 1 | |
| fi | |
| echo "SHARD_TESTS_FILE=$SHARD_FILE" >> "$GITHUB_ENV" | |
| echo "Files in shard ${{ matrix.shard }}: $(wc -l < "$SHARD_FILE")" | |
| - name: Run Cluster Functional Tests (shard ${{ matrix.shard }}) | |
| run: | | |
| mkdir -p logs | |
| DBT_TEST_USER=notnecessaryformosttests@example.com \ | |
| DBT_DATABRICKS_HTTP_PATH="$DBT_DATABRICKS_CLUSTER_HTTP_PATH" \ | |
| xargs -r hatch -v run min-deps:pytest \ | |
| --color=yes -v \ | |
| --profile databricks_cluster \ | |
| -n 10 --dist=loadfile \ | |
| --reruns 1 --reruns-delay 120 \ | |
| --junitxml=logs/junit-shard-${{ matrix.shard }}.xml \ | |
| < "$SHARD_TESTS_FILE" | |
| - name: Upload Cluster Test Logs | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: cluster-test-logs-${{ matrix.target.pr }}-shard-${{ matrix.shard }} | |
| path: logs/ | |
| retention-days: 5 | |
| gather-shards: | |
| needs: | |
| - prepare | |
| - prepare-shards | |
| - run-cluster-e2e-tests | |
| - run-uc-cluster-e2e-tests | |
| - run-sqlwarehouse-e2e-tests | |
| if: | | |
| always() && | |
| needs.prepare-shards.result == 'success' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: ${{ fromJSON(needs.prepare.outputs.targets) }} | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| ref: ${{ matrix.target.ref }} | |
| - name: Set up python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Download shard assignments | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 | |
| with: | |
| name: shard-assignments-${{ matrix.target.pr }} | |
| path: shard-assignments/ | |
| - name: Download per-shard test logs | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 | |
| with: | |
| pattern: "*-test-logs-${{ matrix.target.pr }}-shard-*" | |
| path: shard-logs/ | |
| merge-multiple: false | |
| - name: Verify shards | |
| run: | | |
| set -euo pipefail | |
| declare -a TRIPLES=( | |
| "cluster:cluster:databricks_cluster" | |
| "uc-cluster:uc-cluster:databricks_uc_cluster" | |
| "sql-endpoint:sqlw:databricks_uc_sql_endpoint" | |
| ) | |
| ANY_FAIL=0 | |
| for triple in "${TRIPLES[@]}"; do | |
| IFS=: read -r prefix subdir profile <<< "$triple" | |
| mkdir -p "junit/$subdir" | |
| for d in shard-logs/${prefix}-test-logs-${{ matrix.target.pr }}-shard-*; do | |
| [ -d "$d" ] || continue | |
| shard=$(basename "$d" | sed -E 's/.*-shard-([0-9]+)$/\1/') | |
| cp "$d/junit-shard-${shard}.xml" "junit/${subdir}/junit-shard-${shard}.xml" | |
| done | |
| echo "::group::Verify ${subdir} shards" | |
| python3 scripts/shard_verify.py \ | |
| --manifest "shard-assignments/${profile}-manifest.json" \ | |
| --junit-dir "junit/${subdir}" || ANY_FAIL=1 | |
| echo "::endgroup::" | |
| done | |
| if [ "$ANY_FAIL" -ne 0 ]; then | |
| echo "::error::One or more shard-verification invariants failed." | |
| exit 1 | |
| fi | |
| # Posts a per-job pass/fail summary comment back to the PR when dispatched | |
| # with a single PR number (the slash-command path). Skipped for batch | |
| # dispatches and for schedule / git_ref runs. | |
| report-status: | |
| needs: | |
| - run-uc-cluster-e2e-tests | |
| - run-sqlwarehouse-e2e-tests | |
| - run-cluster-e2e-tests | |
| - gather-shards | |
| if: | | |
| always() && | |
| github.event_name == 'workflow_dispatch' && | |
| inputs.pr_numbers != '' && | |
| !contains(inputs.pr_numbers, ',') | |
| runs-on: | |
| group: databricks-protected-runner-group | |
| labels: linux-ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Post result comment | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 | |
| env: | |
| PR_NUMBER: ${{ inputs.pr_numbers }} | |
| UC_RESULT: ${{ needs.run-uc-cluster-e2e-tests.result }} | |
| SQLW_RESULT: ${{ needs.run-sqlwarehouse-e2e-tests.result }} | |
| CLUSTER_RESULT: ${{ needs.run-cluster-e2e-tests.result }} | |
| GATHER_RESULT: ${{ needs.gather-shards.result }} | |
| with: | |
| script: | | |
| const ICONS = { success: ':white_check_mark:', skipped: ':fast_forward:' }; | |
| const results = { | |
| 'UC cluster': process.env.UC_RESULT, | |
| 'SQL warehouse': process.env.SQLW_RESULT, | |
| 'All-purpose cluster': process.env.CLUSTER_RESULT, | |
| 'Shard coverage': process.env.GATHER_RESULT, | |
| }; | |
| const line = Object.entries(results) | |
| .map(([name, r]) => `${name} ${ICONS[r] || ':x:'} ${r}`) | |
| .join(' · '); | |
| const runUrl = | |
| `https://github.com/${context.repo.owner}/${context.repo.repo}` + | |
| `/actions/runs/${context.runId}`; | |
| const prNumber = process.env.PR_NUMBER.trim(); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: parseInt(prNumber, 10), | |
| body: `Min-deps integration results for PR #${prNumber} — ${line}\n\n[Run details](${runUrl}).`, | |
| }); |