Percona-Lab · nogueiraanderson · Aug 7, 2026 · Aug 7, 2026 · Aug 7, 2026 · Aug 7, 2026
@@ -0,0 +1,29 @@
+# Local composite action: install the hcloud CLI at ONE pinned version,
+# checksum-verified against the release manifest. Single source of the pin for
+# its three call sites (factory bake + prune, janitor sweep). Referenced by
+# path (uses: ./.github/actions/setup-hcloud), so no SHA pin applies.
+name: setup-hcloud
+description: Install the hcloud CLI (pinned, checksum-verified)
+inputs:
+  version:
+    description: hcloud CLI release to install (bump deliberately)
+    required: false
+    default: '1.67.0'
+runs:
+  using: composite
+  steps:
+    - name: Install hcloud CLI (pinned, checksum-verified)
+      shell: bash
+      # The input reaches the script as an env var, not inline ${{ }}, so it is
+      # data to the shell and cannot break out of the run block.
+      env:
+        HCLOUD_CLI_VERSION: ${{ inputs.version }}
+      run: |
+        set -euo pipefail
+        cd "$(mktemp -d)"
+        curl -fsSLO "https://github.com/hetznercloud/cli/releases/download/v${HCLOUD_CLI_VERSION}/hcloud-linux-amd64.tar.gz"
+        curl -fsSLO "https://github.com/hetznercloud/cli/releases/download/v${HCLOUD_CLI_VERSION}/checksums.txt"
+        grep ' hcloud-linux-amd64.tar.gz$' checksums.txt | sha256sum -c -
+        tar -xzf hcloud-linux-amd64.tar.gz hcloud
+        sudo install -m 0755 hcloud /usr/local/bin/hcloud
+        hcloud version
@@ -0,0 +1,317 @@
+# EL package-test snapshot factory (Hetzner Cloud): Rocky Linux, AlmaLinux, and
+# Oracle Linux. Bakes package-test snapshots with Packer over plain SSH (Hetzner
+# injects the build key at server create, no static keys), authenticated by a
+# project-scoped API token, and promotes each via a fresh-boot smoke test. The
+# scheduled run then self-prunes retention (keep 4 promoted per combo).
+# Sibling of ppg-ami-factory.yml (AWS). Consumers select the newest
+# role=ppg-package-test snapshot by label (see ppg/packer-hetzner/README.md).
+# Lineage roots are seeded once per combo via dispatch (just ci-seed).
+#
+# Triggers: PR validate-only (pull_request -> check job, no token), recipe change
+# (code paths only, doc edits never bake), weekly security rebake (cron, offset
+# from the AWS factory's Mon 06:00 so the two factories never rebake in the same
+# window), manual.
+# Actions are pinned to commit SHAs. The trailing "# vX.Y.Z" records the tag.
+
+name: ppg-hcloud-factory
+run-name: ppg hcloud factory (${{ inputs.env || 'prod' }}, ${{ github.event_name }})
+
+on:
+  workflow_dispatch:
+    inputs:
+      combos:
+        description: 'JSON matrix include list, e.g. [{"os":"rocky","os_major":"9","arch":"x86_64","seed":true}]; os defaults to rocky, seed to false'
+        required: false
+        default: '[{"os":"rocky","os_major":"9","arch":"x86_64"}]'
+      env:
+        description: 'prod (real factory labels) or test (isolated labels, never consumed by production)'
+        required: false
+        default: 'prod'
+        type: choice
+        options: [prod, test]
+  # Code paths only (templates, scripts, justfile, bootstrap, this workflow,
+  # the local setup-hcloud action): a README-only edit must not trigger a bake.
+  push:
+    branches: [master]
+    paths:
+      - 'ppg/packer-hetzner/*.pkr.hcl'
+      - 'ppg/packer-hetzner/smoke/**'
+      - 'ppg/packer-hetzner/scripts/**'
+      - 'ppg/packer-hetzner/justfile'
+      - 'ppg/packer-hetzner/bootstrap/**'
+      - '.github/workflows/ppg-hcloud-factory.yml'
+      - '.github/actions/setup-hcloud/**'
+  pull_request:
+    paths:
+      - 'ppg/packer-hetzner/*.pkr.hcl'
+      - 'ppg/packer-hetzner/smoke/**'
+      - 'ppg/packer-hetzner/scripts/**'
+      - 'ppg/packer-hetzner/justfile'
+      - 'ppg/packer-hetzner/bootstrap/**'
+      - '.github/workflows/ppg-hcloud-factory.yml'
+      - '.github/actions/setup-hcloud/**'
+  schedule:
+    - cron: '0 8 * * 1' # weekly Mon 08:00 UTC - absorb EL errata, offset from the AWS factory
+
+permissions:
+  contents: read # workflow-level default: read-only
+
+concurrency:
+  # PR checks stay per-ref. Everything that can bake (push, schedule, manual
+  # dispatch) serializes per factory env ACROSS refs, so two prod bakes can
+  # never interleave their promote and prune steps.
+  group: ppg-hcloud-factory-${{ github.event_name == 'pull_request' && github.ref || (github.event.inputs.env || 'prod') }}
+  cancel-in-progress: false
+
+env:
+  FACTORY_ENV: ${{ github.event.inputs.env || 'prod' }}
+  # One location for bake AND smoke so the snapshot and its boot test share a
+  # placement family (parity with the justfile's location variable).
+  BAKE_LOCATION: fsn1
+
+jobs:
+  # No-token gate: a malformed template or bad combo fails HERE, before bake
+  # spins up any server. Runs scripts/check.sh, the SAME file `just check` runs,
+  # so the CI and local gates cannot drift. The factory token secret is never
+  # exposed to this job, so a PR can never reach the Hetzner project.
+  check:
+    name: Validate templates (no token)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          persist-credentials: false
+
+      - name: Setup Packer
+        uses: hashicorp/setup-packer@3286471d6cc6756d056a0b199fea5e0becdbc189 # v3.3.0
+        with:
+          version: '1.15.4' # exact, matches the bake job
+
+      - name: fmt-check + validate (fail before any server spend)
+        working-directory: ppg/packer-hetzner
+        run: |
+          set -euo pipefail
+          bash scripts/check.sh
+
+  bake:
+    name: ${{ matrix.os || 'rocky' }} ${{ matrix.os_major }} ${{ matrix.arch }}
+    needs: check # no server launches until templates validate
+    # Explicit event allowlist (not a pull_request blocklist), so a later
+    # trigger addition (e.g. pull_request_target) can never reach the token.
+    if: contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name)
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      contents: read
+      id-token: write # OIDC token for AssumeRole
+    strategy:
+      fail-fast: false
+      max-parallel: 12
+      # Default matrix: only combos with a promoted lineage today, so no leg is
+      # guaranteed red. arm64 rows return when CAX stock allows seeding (zero
+      # stock in fsn1/nbg1/hel1 since 2026-08-05). Uncomment the OL8/OL10 rows
+      # after their base bootstrap + seed land (bootstrap/bootstrap-base.sh,
+      # then `just seed <major> x86_64 prod oraclelinux`):
+      #   {"os":"oraclelinux","os_major":"8","arch":"x86_64"},
+      #   {"os":"oraclelinux","os_major":"10","arch":"x86_64"},
+      matrix:
+        include: ${{ fromJSON(github.event.inputs.combos || '[{"os":"rocky","os_major":"8","arch":"x86_64"},{"os":"rocky","os_major":"9","arch":"x86_64"},{"os":"rocky","os_major":"10","arch":"x86_64"},{"os":"almalinux","os_major":"8","arch":"x86_64"},{"os":"almalinux","os_major":"9","arch":"x86_64"},{"os":"almalinux","os_major":"10","arch":"x86_64"},{"os":"oraclelinux","os_major":"9","arch":"x86_64"}]') }}
+    steps:
+      # All tool-setup actions run BEFORE authentication, and neither the AWS
+      # credentials nor the token enter the job env: only the steps below that
+      # explicitly wire them see them.
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          persist-credentials: false
+
+      - name: Install hcloud CLI (pinned, checksum-verified)
+        uses: ./.github/actions/setup-hcloud
+
+      - name: Setup Packer
+        uses: hashicorp/setup-packer@3286471d6cc6756d056a0b199fea5e0becdbc189 # v3.3.0
+        with:
+          # Exact version, never a constraint: setup-packer uses this string verbatim
+          # as the tool-cache path segment, so a space/'>' in a constraint like
+          # ">= 1.12.0" breaks the post-install `packer version` check. Bump deliberately.
+          version: '1.15.4'
+
+      - name: Packer init
+        working-directory: ppg/packer-hetzner
+        run: packer init .
+
+      # Master-pinned OIDC trust + SSM token: a branch-ref dispatch fails at
+      # AssumeRole, and GitHub stores no Hetzner credential.
+      - name: Configure AWS credentials (OIDC)
+        id: aws
+        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
+        with:
+          # Hardcoded on purpose; security is the master-only trust, not ARN secrecy.
+          role-to-assume: arn:aws:iam::119175775298:role/percona-ci-platform-gha-ppg-hcloud-factory
+          role-session-name: ppg-hcloud-bake-${{ github.run_id }}-${{ github.run_attempt }}
+          aws-region: us-east-1
+          role-duration-seconds: 900
+          output-credentials: true
+          output-env-credentials: false
+
+      - name: Read factory token from SSM
+        id: hcloud_token
+        env:
+          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+          AWS_REGION: us-east-1
+        run: |
+          set -euo pipefail
+          token=$(aws ssm get-parameter --name /ppg/hcloud-factory-token \
+            --with-decryption --query Parameter.Value --output text)
+          if [ -z "$token" ] || [ "$token" = "REPLACE-out-of-band" ]; then
+            echo "SSM parameter still holds the placeholder; populate it first" >&2
+            exit 1
+          fi
+          echo "::add-mask::$token"
+          echo "token=$token" >> "$GITHUB_OUTPUT"
+
+      - name: Build candidate snapshot (${{ matrix.os || 'rocky' }} ${{ matrix.os_major }} ${{ matrix.arch }}, env=${{ env.FACTORY_ENV }})
+        id: build
+        working-directory: ppg/packer-hetzner
+        # Matrix values via env, not inline ${{ }}, so they reach packer as data
+        # (quoted shell vars) and cannot break out of the run shell.
+        env:
+          HCLOUD_TOKEN: ${{ steps.hcloud_token.outputs.token }}
+          OS: ${{ matrix.os || 'rocky' }}
+          SEED: ${{ matrix.seed || false }}
+          OS_MAJOR: ${{ matrix.os_major }}
+          ARCH: ${{ matrix.arch }}
+        run: |
+          set -euo pipefail
+          rm -f manifest.json
+          packer build -color=false \
+            -var "os=${OS}" \
+            -var "seed=${SEED}" \
+            -var "os_major=${OS_MAJOR}" \
+            -var "arch=${ARCH}" \
+            -var "env=${FACTORY_ENV}" \
+            -var "location=${BAKE_LOCATION}" .
+          SNAPSHOT_ID=$(python3 -c "import json;print(json.load(open('manifest.json'))['builds'][-1]['artifact_id'])")
+          echo "snapshot_id=$SNAPSHOT_ID" >> "$GITHUB_OUTPUT"
+          echo "built candidate: $SNAPSHOT_ID"
+          {
+            echo "### ${OS} ${OS_MAJOR} ${ARCH}"
+            echo "- built: $SNAPSHOT_ID"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Smoke test (fresh boot + install)
+        working-directory: ppg/packer-hetzner
+        env:
+          HCLOUD_TOKEN: ${{ steps.hcloud_token.outputs.token }}
+          SNAPSHOT_ID: ${{ steps.build.outputs.snapshot_id }}
+          OS: ${{ matrix.os || 'rocky' }}
+          OS_MAJOR: ${{ matrix.os_major }}
+          ARCH: ${{ matrix.arch }}
+        run: |
+          set -euo pipefail
+          # Same sentinel-gated script the justfile smoke recipe runs: delete
+          # the candidate only on a genuine post-SSH boot/install failure, keep
+          # it for retry on transient infrastructure failures.
+          bash scripts/smoke-run.sh "$SNAPSHOT_ID" "$OS_MAJOR" "$ARCH" "$FACTORY_ENV" "$OS" "$BAKE_LOCATION" \
+            || { echo "- smoke: FAIL (transient failures keep the candidate, see log)" >> "$GITHUB_STEP_SUMMARY"; exit 1; }
+          echo "- smoke: pass" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Promote (env=${{ env.FACTORY_ENV }})
+        working-directory: ppg/packer-hetzner
+        env:
+          HCLOUD_TOKEN: ${{ steps.hcloud_token.outputs.token }}
+          SNAPSHOT_ID: ${{ steps.build.outputs.snapshot_id }}
+        run: |
+          set -euo pipefail
+          # Promote via the SAME retried script the justfile uses: a transient
+          # label failure must NOT delete a smoke-passed snapshot.
+          bash scripts/promote.sh "$SNAPSHOT_ID" "$FACTORY_ENV" \
+            || { echo "- promote: FAILED ($SNAPSHOT_ID left for manual promote)" >> "$GITHUB_STEP_SUMMARY"; exit 1; }
+          promoted_role=ppg-package-test
+          if [ "$FACTORY_ENV" = "test" ]; then
+            promoted_role=ppg-test-package-test
+          fi
+          echo "- promote: $promoted_role" >> "$GITHUB_STEP_SUMMARY"
+
+  # Weekly rebakes self-prune retention (keep the newest 4 promoted snapshots
+  # per combo + drop stale candidates). Schedule-only on purpose: a manual
+  # dispatch (partial matrix, test env, seeding) must never trigger deletes.
+  prune:
+    name: Retention prune (keep 4 promoted per combo)
+    needs: bake
+    if: github.event_name == 'schedule' && success()
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      id-token: write # same OIDC trust boundary as bake
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          persist-credentials: false
+
+      - name: Install hcloud CLI (pinned, checksum-verified)
+        uses: ./.github/actions/setup-hcloud
+
+      - name: Configure AWS credentials (OIDC)
+        id: aws
+        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
+        with:
+          role-to-assume: arn:aws:iam::119175775298:role/percona-ci-platform-gha-ppg-hcloud-factory
+          role-session-name: ppg-hcloud-prune-${{ github.run_id }}-${{ github.run_attempt }}
+          aws-region: us-east-1
+          role-duration-seconds: 900
+          output-credentials: true
+          output-env-credentials: false
+
+      - name: Read factory token from SSM
+        id: hcloud_token
+        env:
+          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+          AWS_REGION: us-east-1
+        run: |
+          set -euo pipefail
+          token=$(aws ssm get-parameter --name /ppg/hcloud-factory-token \
+            --with-decryption --query Parameter.Value --output text)
+          if [ -z "$token" ] || [ "$token" = "REPLACE-out-of-band" ]; then
+            echo "SSM parameter still holds the placeholder; populate it first" >&2
+            exit 1
+          fi
+          echo "::add-mask::$token"
+          echo "token=$token" >> "$GITHUB_OUTPUT"
+
+      - name: Prune (prod, apply, keep 4)
+        working-directory: ppg/packer-hetzner
+        env:
+          HCLOUD_TOKEN: ${{ steps.hcloud_token.outputs.token }}
+        run: |
+          set -euo pipefail
+          bash scripts/prune.sh prod 1 4 | tee -a "$GITHUB_STEP_SUMMARY"
+
+  notify:
+    needs: [bake, prune]
+    # failure() is true when ANY ancestor failed, including a check failure on a
+    # pull_request run where bake was skipped. Never ping the webhook for PR lint.
+    if: failure() && github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    # Job-level so the step `if` below reads it reliably. A step-scoped env can be
+    # unavailable to its own step's `if`, which would skip the alert silently.
+    env:
+      SLACK_WEBHOOK: ${{ secrets.RELEASES_CI_SLACK_WEBHOOK }}
+    steps:
+      - name: Slack on failure
+        # Skip cleanly when the webhook secret is unset (e.g. fork tests), so a real
+        # bake failure is not masked by a second "missing webhook" failure.
+        if: env.SLACK_WEBHOOK != ''
+        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+        with:
+          webhook: ${{ env.SLACK_WEBHOOK }}
+          webhook-type: incoming-webhook
+          payload: |
+            text: "PPG hcloud factory FAILED - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"