PKG-1433: Run PPG molecule package tests on Hetzner snapshots

playbooks/create-hetzner.yml

@@ -0,0 +1,152 @@
+---
+# Hetzner Cloud create playbook for the molecule delegated driver: the hcloud
+# sibling of create.yml (EC2). Servers boot from the newest promoted factory
+# snapshot (jenkins-pipelines ppg/packer-hetzner label contract), resolved at
+# runtime by labels and never from a pinned image ID, and every run gets its
+# own throwaway ed25519 SSH key instead of a shared one.
+- name: Create
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+  vars:
+    ssh_port: 22
+    # The factory snapshots log in as root: the packer bake runs as root with
+    # ssh_clear_authorized_keys, and on server create Hetzner cloud-init
+    # injects the ssh_keys passed to the API into root's authorized_keys.
+    ssh_user: root
+    ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}"
+    # Label-safe run id: MOLECULE_RUN_ID when the caller sets one, otherwise
+    # derived from the ephemeral directory + Jenkins job/build so create and
+    # destroy compute the same value for the same run.
+    run_id: "{{ lookup('env', 'MOLECULE_RUN_ID')
+      | default(((ephemeral_directory ~ lookup('env', 'JOB_NAME') ~ lookup('env', 'BUILD_NUMBER')) | hash('sha1'))[:12], true) }}"
+    keypair_path: "{{ ephemeral_directory }}/ssh_key-{{ run_id }}"
+    ssh_key_name: "molecule-{{ run_id }}"
+    # Placement fallback order. A location without capacity is skipped.
+    hetzner_locations: "{{ (lookup('env', 'HETZNER_LOCATIONS') | default('fsn1 nbg1 hel1', true)).split() }}"
+    # Snapshot lookup fallback when a platform carries no image_selector field.
+    # May be empty: then every platform MUST declare image_selector (asserted
+    # below), so a scenario can never silently test the wrong OS.
+    image_selector_base: "{{ lookup('env', 'PPG_IMAGE_SELECTOR_BASE') }}"
+    # The hcloud API architecture enum is x86|arm. The factory labels and the
+    # platform arch field use x86_64|arm64.
+    arch_api_map:
+      x86_64: x86
+      arm64: arm
+
+  tasks:
+    - name: Assert HCLOUD_TOKEN is present (hetzner.hcloud modules read it from the environment)
+      ansible.builtin.assert:
+        that: lookup('env', 'HCLOUD_TOKEN') | length > 0
+        fail_msg: HCLOUD_TOKEN is empty; bind the hcloud-ppg-qa-token Jenkins credential
+        quiet: true
+
+    # Fail closed before any resource is created: a platform without its own
+    # image_selector and no PPG_IMAGE_SELECTOR_BASE would otherwise turn into
+    # an empty label selector that matches EVERY snapshot, and a wrong-OS
+    # green is worse than a missing-var red.
+    - name: Assert every platform has an image selector
+      ansible.builtin.assert:
+        that: (item.image_selector | default(image_selector_base, true) | length) > 0
+        fail_msg: >-
+          Platform {{ item.name }} declares no image_selector and
+          PPG_IMAGE_SELECTOR_BASE is empty; set one of them
+        quiet: true
+      loop: "{{ molecule_yml.platforms }}"
+
+    - name: Generate a per-run ed25519 keypair (no shared key on Hetzner)
+      community.crypto.openssh_keypair:
+        path: "{{ keypair_path }}"
+        type: ed25519
+        mode: "0600"
+      register: run_keypair
+
+    - name: Upload the run public key (labeled so destroy and the janitor can find it)
+      hetzner.hcloud.ssh_key:
+        name: "{{ ssh_key_name }}"
+        public_key: "{{ run_keypair.public_key }}"
+        labels:
+          ppg_run: "{{ run_id }}"
+          role: ppg-molecule-test
+        state: present
+
+    - name: Resolve promoted snapshots per platform (labels, never pinned IDs)
+      hetzner.hcloud.image_info:
+        type: snapshot
+        architecture: "{{ arch_api_map[item.arch | default('x86_64', true)] }}"
+        label_selector: "{{ (item.image_selector | default(image_selector_base, true)) ~ ',arch=' ~ (item.arch | default('x86_64', true)) }}"
+      loop: "{{ molecule_yml.platforms }}"
+      register: image_lookup
+
+    - name: Assert every platform matched at least one snapshot
+      ansible.builtin.assert:
+        that: item.hcloud_image_info | length > 0
+        fail_msg: >-
+          No promoted snapshot matches platform {{ item.item.name }}
+          (selector {{ (item.item.image_selector | default(image_selector_base, true)) ~ ',arch=' ~ (item.item.arch | default('x86_64', true)) }});
+          has the factory promoted this combo yet?
+        quiet: true
+      loop: "{{ image_lookup.results }}"
+
+    # image_info returns no created field. hcloud image IDs increase
+    # monotonically, so the highest id IS the newest snapshot (its factory_run
+    # label is the human-readable twin of that ordering).
+    - name: Pick the newest snapshot per platform
+      ansible.builtin.set_fact:
+        platform_images: "{{ platform_images | default({}) | combine({item.item.name: (item.hcloud_image_info | sort(attribute='id') | last).id}) }}"
+      loop: "{{ image_lookup.results }}"
+
+    - name: Start with an empty created-server list
+      ansible.builtin.set_fact:
+        hetzner_created: []
+
+    - name: Create molecule server(s) (per platform, with location fallback)
+      ansible.builtin.include_tasks: tasks/hetzner-server-attempt.yml
+      vars:
+        hetzner_fresh_attempt: true
+      loop: "{{ molecule_yml.platforms }}"
+      loop_control:
+        loop_var: platform
+
+    - name: Wait for SSH
+      ansible.builtin.wait_for:
+        port: "{{ ssh_port }}"
+        host: "{{ item.server.ipv4_address }}"
+        search_regex: SSH
+        delay: 10
+        timeout: 320
+      loop: "{{ hetzner_created }}"
+
+    - name: Wait for boot process to finish
+      ansible.builtin.pause:
+        minutes: 2
+
+    # Mandatory configuration for Molecule to function: with the managed
+    # delegated driver, molecule reads address/user/port/identity_file from
+    # this file to build the connection options (run_id and ssh_key_name are
+    # bookkeeping for destroy-hetzner.yml, and molecule ignores unknown keys).
+    - name: Populate instance config dict
+      ansible.builtin.set_fact:
+        instance_conf_dict:
+          instance: "{{ item.platform.name }}"
+          address: "{{ item.server.ipv4_address }}"
+          user: "{{ item.platform.ssh_user | default(ssh_user, true) }}"
+          port: "{{ ssh_port }}"
+          identity_file: "{{ keypair_path }}"
+          connection: ssh
+          instance_id: "{{ item.server.id }}"
+          run_id: "{{ run_id }}"
+          ssh_key_name: "{{ ssh_key_name }}"
+      loop: "{{ hetzner_created }}"
+      register: instance_config_dict
+
+    - name: Convert instance config dict to a list
+      ansible.builtin.set_fact:
+        instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}"
+
+    - name: Dump instance config
+      ansible.builtin.copy:
+        content: "{{ instance_conf | to_json | from_json | to_yaml }}"
+        dest: "{{ molecule_instance_config }}"
+        mode: "0600"

playbooks/destroy-hetzner.yml

@@ -0,0 +1,84 @@
+---
+# Hetzner Cloud destroy playbook for the molecule delegated driver: the hcloud
+# sibling of destroy.yml (EC2). Deletes by the stored server id first, then a
+# ppg_run label sweep catches servers the instance config lost track of, then
+# the run's SSH key and the local key files go. Idempotent when nothing
+# exists. The hourly janitor in jenkins-pipelines is the backstop.
+- name: Destroy
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+  vars:
+    ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}"
+    # Same derivation as create-hetzner.yml, so a destroy that finds no
+    # instance config still cleans this run's resources.
+    run_id: "{{ lookup('env', 'MOLECULE_RUN_ID')
+      | default(((ephemeral_directory ~ lookup('env', 'JOB_NAME') ~ lookup('env', 'BUILD_NUMBER')) | hash('sha1'))[:12], true) }}"
+    keypair_path: "{{ ephemeral_directory }}/ssh_key-{{ run_id }}"
+    ssh_key_name: "molecule-{{ run_id }}"
+  tasks:
+    - name: Read the instance config, tolerating a missing file
+      block:
+        - name: Populate instance config
+          ansible.builtin.set_fact:
+            instance_conf: "{{ lookup('file', molecule_instance_config) | from_yaml }}"
+            skip_instances: false
+      rescue:
+        - name: Populate instance config when file missing
+          ansible.builtin.set_fact:
+            instance_conf: []
+            skip_instances: true
+
+    # failed_when: false mirrors destroy.yml: an already-gone server or a
+    # transient API error never reddens an otherwise-green run. The label
+    # sweep below and the hourly janitor are the backstops.
+    - name: Destroy molecule instance(s) by stored id
+      hetzner.hcloud.server:
+        id: "{{ item.instance_id }}"
+        state: absent
+      loop: "{{ instance_conf }}"
+      when: not skip_instances
+      failed_when: false
+
+    # The role label rides along with ppg_run so this sweep can only ever
+    # match servers this playbook family created (same rule as the janitor:
+    # never sweep by ppg_run alone).
+    - name: Find any survivors of this run by label
+      hetzner.hcloud.server_info:
+        label_selector: "ppg_run={{ run_id }},role=ppg-molecule-test"
+      register: run_servers
+
+    - name: Destroy survivors
+      hetzner.hcloud.server:
+        id: "{{ item.id }}"
+        state: absent
+      loop: "{{ run_servers.hcloud_server_info | default([]) }}"
+      failed_when: false
+
+    # Key names from the instance config first (exact), plus this run's
+    # derived name. state=absent on a name that never existed is a no-op.
+    - name: Delete the run SSH key(s)
+      hetzner.hcloud.ssh_key:
+        name: "{{ item }}"
+        state: absent
+      loop: "{{ ((instance_conf | map(attribute='ssh_key_name', default='') | select | list) + [ssh_key_name]) | unique }}"
+
+    - name: Remove the local key files
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: absent
+      loop:
+        - "{{ keypair_path }}"
+        - "{{ keypair_path }}.pub"
+
+    # Mandatory configuration for Molecule to function.
+    - name: Populate instance config
+      ansible.builtin.set_fact:
+        instance_conf: []
+
+    - name: Dump instance config
+      ansible.builtin.copy:
+        content: "{{ instance_conf | to_json | from_json | to_yaml }}"
+        dest: "{{ molecule_instance_config }}"
+        mode: "0600"

playbooks/tasks/hetzner-server-attempt.yml

@@ -0,0 +1,56 @@
+---
+# One placement attempt for the platform in `platform`: create the server in
+# the location the cursor points at. On failure advance the cursor and recurse
+# into the next location, failing only when every location was tried.
+# create-hetzner.yml includes this once per platform with
+# hetzner_fresh_attempt=true (resets the cursor). The recursive include below
+# passes false so the cursor survives across attempts.
+- name: Reset the location cursor for {{ platform.name }}
+  ansible.builtin.set_fact:
+    hetzner_location_index: 0
+  when: hetzner_fresh_attempt | bool
+
+- name: Attempt the current fallback location for {{ platform.name }}
+  block:
+    - name: Create server in the current fallback location for {{ platform.name }}
+      hetzner.hcloud.server:
+        # RFC1123 server name: molecule-<run-id>-<platform>, sanitized and
+        # capped at 63 chars (Jenkins platform names embed JOB_NAME).
+        name: "{{ ('molecule-' ~ run_id ~ '-' ~ (platform.name | lower | regex_replace('[^a-z0-9-]', '-')))[:63] | regex_replace('-+$', '') }}"
+        # Platform value first (the pipeline sets it from the
+        # htz_server_type_* map), then the per-arch env var, then the
+        # cpx32/cax21 defaults, mirroring the AWS t3.large tier.
+        server_type: "{{ platform.server_type
+          | default(lookup('env', 'htz_server_type_' ~ (platform.arch | default('x86_64', true))), true)
+          | default({'x86_64': 'cpx32', 'arm64': 'cax21'}[platform.arch | default('x86_64', true)], true) }}"
+        image: "{{ platform_images[platform.name] }}"
+        location: "{{ hetzner_locations[hetzner_location_index | int] }}"
+        ssh_keys:
+          - "{{ ssh_key_name }}"
+        labels:
+          ppg_run: "{{ run_id }}"
+          role: ppg-molecule-test
+        user_data: "{{ platform.user_data | default(omit) }}"
+        state: started
+      register: hetzner_server_result
+
+    - name: Record created server for {{ platform.name }}
+      ansible.builtin.set_fact:
+        hetzner_created: "{{ hetzner_created + [{'platform': platform, 'server': hetzner_server_result.hcloud_server}] }}"
+  rescue:
+    - name: Advance the location cursor
+      ansible.builtin.set_fact:
+        hetzner_location_index: "{{ (hetzner_location_index | int) + 1 }}"
+
+    - name: Fail when every location was tried
+      ansible.builtin.fail:
+        msg: >-
+          Could not place {{ platform.name }} in any of
+          {{ hetzner_locations | join(', ') }}:
+          {{ ansible_failed_result.msg | default('server create failed') }}
+      when: hetzner_location_index | int >= hetzner_locations | length
+
+    - name: Retry in the next location
+      ansible.builtin.include_tasks: hetzner-server-attempt.yml
+      vars:
+        hetzner_fresh_attempt: false

ppg/pg-17/molecule/rocky-9-hetzner/molecule.yml

@@ -0,0 +1,67 @@
+---
+dependency:
+  name: galaxy
+driver:
+  # Delegated: create/destroy are the hetzner playbooks below. With
+  # managed: true molecule reads the connection details they dump into the
+  # instance config (address/user/port/identity_file).
+  name: delegated
+  options:
+    managed: true
+platforms:
+  # Boots the newest promoted factory snapshot for this combo, resolved at
+  # runtime by create-hetzner.yml from image_selector + arch (never a
+  # pinned image ID). server_type comes from the pipeline's
+  # htz_server_type_* map. When empty (local run) create-hetzner.yml falls
+  # back per arch. ssh_user is root because Hetzner injects the per-run key
+  # for root on snapshot-based creates. The cloud-config adds the rocky
+  # user the shared prepare.yml expects for the QA debug keys, sudo-capable
+  # like on the AWS AMIs.
+  - name: rocky9-hetzner-${BUILD_NUMBER}-${JOB_NAME}
+    server_type: ${HETZNER_SERVER_TYPE}
+    arch: ${HETZNER_ARCH:-x86_64}
+    image_selector: role=ppg-package-test,os=rocky,os_major=9
+    ssh_user: root
+    user_data: |
+      #cloud-config
+      users:
+        - default
+        - name: rocky
+          groups: wheel
+          shell: /bin/bash
+          sudo: ALL=(ALL) NOPASSWD:ALL
+provisioner:
+  name: ansible
+  log: True
+  env:
+    # Molecule builds its own env for ansible-playbook (ephemeral collections
+    # path + user home), dropping the caller's exported value. Pass the
+    # per-workspace collections path through so the pinned, patched
+    # hetzner.hcloud copy is the one that resolves.
+    ANSIBLE_COLLECTIONS_PATH: ${ANSIBLE_COLLECTIONS_PATH}
+  playbooks:
+    create: ../../../../playbooks/create-hetzner.yml
+    destroy: ../../../../playbooks/destroy-hetzner.yml
+    prepare: ../../../../playbooks/prepare.yml
+    cleanup: ../../playbooks/cleanup-rpm.yml
+    converge: ../../playbooks/playbook.yml
+verifier:
+  name: testinfra
+  directory: ../../../tests/tests_ppg
+  options:
+    verbose: true
+    s: true
+    junitxml: report.xml
+scenario:
+  destroy_sequence:
+    - destroy
+  cleanup_sequence:
+    - cleanup
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - verify
+    - cleanup
+    - destroy