From ab0e709c57126b42abe1cfb7014f7fc3edfa1a21 Mon Sep 17 00:00:00 2001 From: Hector Vido Date: Fri, 29 May 2026 09:55:40 -0300 Subject: [PATCH] Moved the script to generate prowjobs from GDocs to github --- hack/make-pjs.sh | 194 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 hack/make-pjs.sh diff --git a/hack/make-pjs.sh b/hack/make-pjs.sh new file mode 100644 index 0000000000000..de3e4c77d44fe --- /dev/null +++ b/hack/make-pjs.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Author: Danilo Gemoli +# Generates ProwJobs, mainly for tests on build clusters +# Usage: +# RELEASE_REPO=<...> \ +# CLUSTER=build99 \ +# HOW_MANY=5 \ +# PROWJOBS_CONFIG='ci-operator/jobs/openshift/installer/openshift-installer-main-presubmits.yaml' \ +# TYPE=e2e \ +# make_pjs.sh +# +# config for e2e: ci-operator/jobs/openshift/installer/openshift-installer-main-presubmits.yaml +# config for intranet: ci-operator/config/openshift/release/openshift-release-main__nightly-4.19.yaml' + +set -o errexit +set -o nounset +set -o pipefail + +ORG="" +REPO="" +BRANCH="" +VARIANT="" +ARCH="${ARCH:-amd64}" + +# Get the SHA of a branch +function branch_info() { + git ls-remote "https://github.com/${ORG}/${REPO}" "$BRANCH" +} + +# Generate PJs manifests and save them in a temporary directory /tmp/tests-XXXXX +function make_prowjobs() { + tests=$1 + + base_dir=$(mktemp -p /tmp -d 'tests-XXXXX') + + mkpj_img=$(awk -F '=' '/mkpj/{print $2}' "${RELEASE_REPO}/hack/images.sh") + base_sha=$(branch_info | awk '{print $1}') + echo "base_sha: $base_sha" + + for test in $tests; do + echo "generating: ${base_dir}/${test}.yaml" + refs='' + if [[ $test != periodic* ]]; then + refs='.spec.refs.pulls = [{ + author: "fake", + number: 1, + sha: (.spec.refs.base_sha) + }] |' + fi + podman run \ + --platform "linux/$ARCH" \ + --rm \ + --volume "$RELEASE_REPO:/tmp/release:z" \ + --workdir /tmp/release \ + "$mkpj_img" \ + --config-path core-services/prow/02_config/_config.yaml \ + --job-config-path ci-operator/jobs/ \ + --base-ref "$BRANCH" \ + --base-sha "$base_sha" \ + --pull-number 1 \ + --job "$test" \ + 2>/dev/null \ + | yq -ojson \ + | jq "${refs:-} + .status.state = \"triggered\" + | .spec.report = false + | .spec.cluster = \"$CLUSTER\"" \ + | yq -P \ + >"${base_dir}/${test}.yaml" + done + + echo "output dir: $base_dir" +} + +# Filter tests that have: `restrict_network_access: false` +function vpn_tests() { + ci_operator_config="$1" + + variant=$(yq -ojson "${RELEASE_REPO}/${ci_operator_config}" \ + | jq -r '.zz_generated_metadata + | if has("variant") then .variant else "" end') + + names=$(yq -ojson "${RELEASE_REPO}/${ci_operator_config}" \ + | jq -jc \ + '[ + .tests[] + | select(has("restrict_network_access") and (.restrict_network_access|not)) + | .as + ]') + + yq -ojson "${RELEASE_REPO}/ci-operator/jobs/${ORG}/${REPO}/${ORG}-${REPO}-${BRANCH}-"*.yaml \ + | jq -sr --arg variant "$variant" --argjson suffixes "$names" \ + ' .[] + | if has("periodics") then .[][] else .[][][] end + | if $variant == "" then + . + else + select(.labels.["ci-operator.openshift.io/variant"]? == $variant) + end + | .name + | . as $name + | select(any($suffixes[]; . as $suffix|$name|endswith($suffix)))' +} + +# Filter PJs whose name contains '-e2e-' +function e2e_tests() { + prowjobs_config="$1" + yq -ojson "${RELEASE_REPO}/${prowjobs_config}" \ + | jq -r --arg orgrepo "${ORG}/${REPO}" \ + 'if has("periodics") then .[][] else .[][$orgrepo][] end + | select(.name|contains("-e2e-")) + | .name' +} + +# Extract organization, repository, branch and variant from a path that matches the pattern: +# ci-operator/config/${ORG}/${REPO}/${ORG}-${REPO}-${BRANCH}__${VARIANT}.yaml +function extract_test_info_from_config() { + path="$1" + + ORG=$(awk -F '/' '{print $3}' <<<"$path") + [ -z "$ORG" ] && echo "can't extract organization" && exit 1 + + REPO=$(awk -F '/' '{print $4}' <<<"$path") + [ -z "$REPO" ] && echo "can't extract repository" && exit 1 + + filename=$(basename "$path") + + BRANCH=$(sed -E "s/${ORG}\-${REPO}\-([^_]+)(__.+)?\.yaml/\1/" <<<"$filename") + [ -z "$BRANCH" ] && echo "can't extract branch" && exit 1 + + VARIANT=$(sed -E "s/${ORG}\-${REPO}\-${BRANCH}(__(.+))?\.yaml/\2/" <<<"$filename") + + return 0 +} + +# Extract organization, repository and branch from a path that matches the pattern: +# ci-operator/jobs/${ORG}/${REPO}/${ORG}-${REPO}-${BRANCH}-(presubmits|postsubmits|periodics).yaml +function extract_test_info_from_job() { + path="$1" + + ORG=$(awk -F '/' '{print $3}' <<<"$path") + [ -z "$ORG" ] && echo "can't extract organization" && exit 1 + + REPO=$(awk -F '/' '{print $4}' <<<"$path") + [ -z "$REPO" ] && echo "can't extract repository" && exit 1 + + filename=$(basename "$path") + + BRANCH=$(sed -E "s/${ORG}\-${REPO}\-(.+)\-(presubmits|postsubmits|periodics)\.yaml/\1/" <<<"$filename") + [ -z "$BRANCH" ] && echo "can't extract branch" && exit 1 + + return 0 +} + +function dump_test_info() { + echo "org: $ORG" + echo "repo: $REPO" + echo "branch: $BRANCH" + echo "variant: $VARIANT" +} + +function entrypoint() { + [ -z "${RELEASE_REPO:-}" ] && echo "RELEASE_REPO is required" && exit 1 + [ -z "${CLUSTER:-}" ] && echo "CLUSTER is required" && exit 1 + [ -z "${HOW_MANY:-}" ] && echo "HOW_MANY is required" && exit 1 + [ -z "${TYPE:-}" ] && echo "TYPE is required" && exit 1 + + case "$TYPE" in + "e2e") + PROWJOBS_CONFIG="${PROWJOBS_CONFIG:-ci-operator/jobs/openshift/installer/openshift-installer-main-presubmits.yaml}" + extract_test_info_from_job "$PROWJOBS_CONFIG" + dump_test_info + tests=$(e2e_tests "$PROWJOBS_CONFIG" | sort -R) + [ -z "$tests" ] && echo "no e2e tests found in ${RELEASE_REPO}/${PROWJOBS_CONFIG}" && exit 1 + top_n=$(head -n "$HOW_MANY" <<<"$tests") + make_prowjobs "$top_n" + ;; + "intranet") + CI_OPERATOR_CONFIG="${CI_OPERATOR_CONFIG:-ci-operator/config/openshift/release/openshift-release-master__nightly-4.19.yaml}" + extract_test_info_from_config "$CI_OPERATOR_CONFIG" + dump_test_info + tests=$(vpn_tests "$CI_OPERATOR_CONFIG" | sort -R) + [ -z "$tests" ] && echo "no vpn tests found in ${RELEASE_REPO}/${CI_OPERATOR_CONFIG}" && exit 1 + top_n=$(head -n "$HOW_MANY" <<<"$tests") + make_prowjobs "$top_n" + ;; + *) + echo "unknown test type $TYPE" + exit 1 + ;; + esac +} + +entrypoint