diff --git a/.backports.yml b/.backports.yml new file mode 100644 index 00000000000..758430339c2 --- /dev/null +++ b/.backports.yml @@ -0,0 +1,437 @@ +# Backport branch inventory — single source of truth. +# See issue #19210 for schema documentation. +# +# Active branch logic: +# inactive if: archived == true +# OR (maintained_until != null AND maintained_until < today) +# +# Fill in maintained_until for branches with known EOL dates (YYYY-MM-DD). +# Example: security_detection_engine-8.19 -> "2027-01-15" (8.19 EOL) +# +# This file was seeded by dev/scripts/backport_bootstrap_inventory.sh and then +# manually reviewed and corrected (base_commit / base_version values for several +# branches were adjusted after the automated seed). +# Do not re-run the bootstrap — update this file directly. +backports: + - package: apm + branch: backport-apm-8.15 + base_version: "8.15.0-preview-1716438434" + base_commit: "86356203eb" + maintained_until: null + archived: true + + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false + + - package: aws + branch: backport-aws-3.13 + base_version: "3.13.4" + base_commit: "279466b250" + maintained_until: null + archived: false + + - package: aws + branch: backport-aws-2.30 + base_version: "2.30.1" + base_commit: "ba74f78535" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: aws + branch: backport-aws-2.25 + base_version: "2.25.0" + base_commit: "4c34839865" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: aws + branch: backport-aws-2.24 + base_version: "2.24.1" + base_commit: "8d52b1684b" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: aws + branch: backport-aws-1.51 + base_version: "1.51.1" + base_commit: "dd26f557c5" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + # this backport branch name did not follow the format backport--. + # it was based on the stack supported + - package: aws + branch: backport-aws-7.15.0 + base_version: "1.19.5" + base_commit: "98f007d268" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: cloud_asset_inventory + branch: backport-cloud_asset_inventory-1.4 + base_version: "1.4.0" + base_commit: "949938722b" + maintained_until: null + archived: false + + - package: cloud_asset_inventory + branch: backport-cloud_asset_inventory-1.3 + base_version: "1.3.0" + base_commit: "5deeef929e" + maintained_until: null + archived: false + + # no backport releases performed + - package: cloud_asset_inventory + branch: backport-cloud_asset_inventory-1.1 + base_version: "1.1.6" + base_commit: "42ff22b32b" + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-3.2 + base_version: "3.2.0" + base_commit: "949938722b" + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-3.1 + base_version: "3.1.2" + base_commit: "dd7ba85b55" + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-3.0 + base_version: "3.0.1" + base_commit: "ecccd7c005" + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-2.0 + base_version: "2.0.0" + base_commit: "a707cd5a4e" + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.13 + base_version: "1.13.0" + base_commit: "7a732587fb" + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.10 + base_version: "1.10.0" + base_commit: "456669c88c" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.9 + base_version: "1.9.0-preview04" + base_commit: "b1cdf5e3f9" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.8 + base_version: "1.8.0" + base_commit: "7cf56eb4d3" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.7 + base_version: "1.7.1" + base_commit: "527a51d72c" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.5 + base_version: "1.5.2" + base_commit: "6041abb5fa" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.4 + base_version: "1.4.0-preview22" + base_commit: "1fa1254163" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.3 + base_version: "1.3.0-preview7" + base_commit: "368c97f64b" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.2 + base_version: "1.2.13-preview" + base_commit: "516ff06ab5" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.1 + base_version: "1.1.1" + base_commit: "6291fa268f" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: cloud_security_posture + branch: backport-cloud_security_posture-1.0 + base_version: "1.0.5" + base_commit: "ab009e424d" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: crowdstrike + branch: backport-crowdstrike-1.52 + base_version: "1.52.1" + base_commit: "cbad38e47d" + maintained_until: null + archived: false + + # no backport releases performed + - package: crowdstrike + branch: backport-crowdstrike-1.46 + base_version: "1.46.0" + base_commit: "b7f8c57ae5" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + # no backport releases performed + - package: elastic_agent + branch: backport-elastic_agent-2.5 + base_version: "2.5.2" + base_commit: "50c274fafc" + maintained_until: null + archived: false + + # no backport releases performed - testing purposes + - package: elastic_package_registry + branch: backport-elastic_package_registry-0.2 + base_version: "0.2.0" + base_commit: "cd3c0a4974" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: gcp + branch: backport-gcp-2.22 + base_version: "2.22.1" + base_commit: "1cfed2d549" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: google_workspace + branch: backport-google_workspace-3.0 + base_version: "3.0.0" + base_commit: "e4bdd6a8cf" + maintained_until: null + archived: false + + - package: kubernetes + branch: backport-kubernetes-1.83 + base_version: "1.83.0" + base_commit: "10091758cb" + maintained_until: null + archived: false + + - package: kubernetes + branch: backport-kubernetes-1.62 + base_version: "1.62.1" + base_commit: "8a2eae8629" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: kubernetes + branch: backport-kubernetes-1.39 + base_version: "1.39.0" + base_commit: "a8d3a46dc6" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: security_detection_engine + branch: backport-security_detection_engine-9.3 + base_version: "9.3.8" + base_commit: "fd04de398f" + maintained_until: null + archived: false + + - package: security_detection_engine + branch: backport-security_detection_engine-9.2 + base_version: "9.2.4" + base_commit: "2451d2eaf6" + maintained_until: null + archived: false + + - package: security_detection_engine + branch: backport-security_detection_engine-9.1 + base_version: "9.1.8" + base_commit: "1f867af3f4" + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-9.0 + base_version: "9.0.7" + base_commit: "75a1f58879" + maintained_until: null + archived: true + + # created from backport-security_detection_engine-8.18 + - package: security_detection_engine + branch: backport-security_detection_engine-8.19 + base_version: "8.18.17" + base_commit: "67e79dbaac" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: security_detection_engine + branch: backport-security_detection_engine-8.18 + # same commit as 8.17 + base_version: "8.17.7" + base_commit: "0cd2c693e3" + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.17 + base_version: "8.17.7" + base_commit: "0cd2c693e3" + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.16 + base_version: "8.16.2" + base_commit: "550932fb5d" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.15 + base_version: "8.15.9" + base_commit: "945a941aee" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.14 + base_version: "8.14.6" + base_commit: "9080820c4b" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.13 + base_version: "8.13.6" + base_commit: "37279750b2" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.12 + base_version: "8.12.5" + base_commit: "06a16238c4" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.11 + base_version: "8.11.4" + base_commit: "ef3fed12b9" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.10 + base_version: "8.10.4-beta.1" + base_commit: "97f083b2a2" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + # backport branch created to publish a missing package version from backport-security_detection_engine-8.9 + - package: security_detection_engine + branch: backport-security_detection_engine-8.9.10 + base_version: "8.9.10" + base_commit: "efb86cb13a" + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.9 + base_version: "8.9.3" + base_commit: "6c72289e7a" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.8 + base_version: "8.8.7" + base_commit: "5e1a07f493" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.7 + base_version: "8.7.9" + base_commit: "741899cb42" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: security_detection_engine + branch: backport-security_detection_engine-8.6 + base_version: "8.6.9" + base_commit: "b1c7529a99" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: true + + - package: sql + branch: backport-sql-0.6 + base_version: "0.6.0" + base_commit: "00a4eb9813" + maintained_until: null + archived: false + + - package: synthetics + branch: backport-synthetics-1.0 + base_version: "1.0.6" + base_commit: "7bebadfcb3" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + + - package: tenable_io + branch: backport-tenable_io-3.10 + base_version: "3.10.1" + base_commit: "f34162d484" + maintained_until: null + archived: false + + - package: ti_abusech + branch: backport-ti_abusech-2.6 + base_version: "2.6.0" + base_commit: "196f9926bd" + maintained_until: null + archived: false + + - package: wiz + branch: backport-wiz-1.8 + base_version: "1.8.0" + base_commit: "e01695bb9b" # not in upstream/main; next commit is branch-exclusive + maintained_until: null + archived: false + diff --git a/.buildkite/pipeline.backport.yml b/.buildkite/pipeline.backport.yml index 5bafe5e62d5..0f37b461cac 100644 --- a/.buildkite/pipeline.backport.yml +++ b/.buildkite/pipeline.backport.yml @@ -13,12 +13,20 @@ steps: - label: "Check that it runs from UI" key: "check-ui" command: - - "buildkite-agent annotate \"The $BUILDKITE_PIPELINE_SLUG is used only for running from UI!\" --style 'warning'" + - "buildkite-agent annotate \"The $BUILDKITE_PIPELINE_SLUG is used only for running from UI or a trigger step!\" --style 'warning'" - "exit 1" - if: "build.source != 'ui'" + if: | + !( + build.source == 'ui' || + (build.source == 'trigger_job' && build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == 'integrations') + ) + + # Ensure that the check-ui step runs before any other step + - wait: ~ - input: "Input values for the variables" key: "input-variables" + if: "build.source != 'trigger_job'" fields: - select: "DRY_RUN" key: "DRY_RUN" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e6c95dc9dc5..01bcc6d2967 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -73,6 +73,32 @@ steps: key: "check-dev-scripts" command: ".buildkite/scripts/run_dev_scripts_tests.sh" + - label: ":ballot_box_with_check: Backports inventory validation" + key: "check-backports-inventory" + command: ".buildkite/scripts/check_backports_inventory.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + if_changed: + - ".backports.yml" + - ".buildkite/scripts/run_dev_scripts_tests.sh" + - "dev/scripts/**.sh" + - "dev/backports/**" + + - label: ":git: Trigger backport dry-runs" + key: "trigger-backport-dryrun" + command: ".buildkite/scripts/trigger_backport_dryrun.sh" + agents: + image: "${LINUX_AGENT_IMAGE}" + depends_on: + - step: "check-backports-inventory" + allow_failure: false + if_changed: + - ".backports.yml" + if: | + build.env('BUILDKITE_PULL_REQUEST') != "false" && + build.env('BUILDKITE_PIPELINE_SLUG') == "integrations" && + build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') == "main" + - label: ":junit: Sources Junit annotate" agents: # requires at least "bash", "curl" and "git" diff --git a/.buildkite/scripts/backport_branch.sh b/.buildkite/scripts/backport_branch.sh index 5181b1cf4fd..6391bf63be1 100755 --- a/.buildkite/scripts/backport_branch.sh +++ b/.buildkite/scripts/backport_branch.sh @@ -169,6 +169,7 @@ updateBackportBranchContents() { local COVERAGE_SCRIPTS_FOLDER="dev/coverage" local CODEOWNERS_SCRIPTS_FOLDER="dev/codeowners" local PACKAGENAMES_SCRIPTS_FOLDER="dev/packagenames" + local BACKPORTS_SCRIPTS_FOLDER="dev/backports" local DEV_SCRIPTS_FOLDER="dev/scripts" if git ls-tree -d --name-only main:${MAGEFILE_SCRIPTS_FOLDER} > /dev/null 2>&1 ; then @@ -193,13 +194,13 @@ updateBackportBranchContents() { git checkout "$SOURCE_BRANCH" -- "${PACKAGENAMES_SCRIPTS_FOLDER}" git add "${PACKAGENAMES_SCRIPTS_FOLDER}" - if git ls-tree -d --name-only "${SOURCE_BRANCH}:${DEV_SCRIPTS_FOLDER}" > /dev/null 2>&1; then - echo "Copying $DEV_SCRIPTS_FOLDER from $SOURCE_BRANCH..." - git checkout "$SOURCE_BRANCH" -- "${DEV_SCRIPTS_FOLDER}" - git add "${DEV_SCRIPTS_FOLDER}" - else - echo "Skipping $DEV_SCRIPTS_FOLDER (not found on $SOURCE_BRANCH)" - fi + echo "Copying $BACKPORTS_SCRIPTS_FOLDER from $SOURCE_BRANCH..." + git checkout "$SOURCE_BRANCH" -- "${BACKPORTS_SCRIPTS_FOLDER}" + git add "${BACKPORTS_SCRIPTS_FOLDER}" + + echo "Copying $DEV_SCRIPTS_FOLDER from $SOURCE_BRANCH..." + git checkout "$SOURCE_BRANCH" -- "${DEV_SCRIPTS_FOLDER}" + git add "${DEV_SCRIPTS_FOLDER}" echo "Copying magefile.go from $SOURCE_BRANCH..." git checkout "$SOURCE_BRANCH" -- "magefile.go" diff --git a/.buildkite/scripts/check_backports_inventory.sh b/.buildkite/scripts/check_backports_inventory.sh new file mode 100755 index 00000000000..35a886dc062 --- /dev/null +++ b/.buildkite/scripts/check_backports_inventory.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +add_bin_path +with_mage +with_yq + +echo "--- Validate .backports.yml inventory schema" +mage -d "${WORKSPACE}" -v validateBackportsInventory + +echo "--- Check if any files modified" +check_git_diff diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index a107ce2e5e9..eef563d66b5 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -779,6 +779,7 @@ is_pr_affected() { # Same for ".buildkite/scripts/packages/.+.sh": this pattern must not be added to "skip_ci_on_only_changed" to allow triggering the tests of the given package. local non_package_patterns=( 'packages/' + '\.backports\.yml' '\.buildkite/pipeline\.backport\.yml' '\.buildkite/pipeline\.publish\.yml' '\.buildkite/pipeline\.schedule-daily\.yml' @@ -786,6 +787,8 @@ is_pr_affected() { '\.buildkite/pipeline\.serverless\.yml' '\.buildkite/pull-requests\.json' '\.buildkite/scripts/backport_branch\.sh' + '\.buildkite/scripts/check_backports_inventory\.sh' + '\.buildkite/scripts/trigger_backport_dryrun\.sh' '\.buildkite/scripts/build_packages\.sh' '\.buildkite/scripts/check_changelog_entries\.sh' '\.buildkite/scripts/packages/.+\.sh' @@ -804,6 +807,7 @@ is_pr_affected() { '\.agents/skills/' 'catalog-info\.yaml' 'docs/' + 'dev/backports/' 'dev/scripts/' 'CODE_OF_CONDUCT\.md' 'CONTRIBUTING\.md' @@ -917,7 +921,7 @@ teardown_test_package() { # list all directories that are packages from the root of the repository list_all_directories() { - mage -d "${WORKSPACE}" listPackages + mage -d "${WORKSPACE}" listPackages |grep "^packages/elastic_package_registry$" } check_package() { diff --git a/.buildkite/scripts/run_dev_scripts_tests.sh b/.buildkite/scripts/run_dev_scripts_tests.sh index 8bc80087447..86121001104 100755 --- a/.buildkite/scripts/run_dev_scripts_tests.sh +++ b/.buildkite/scripts/run_dev_scripts_tests.sh @@ -6,14 +6,5 @@ set -euo pipefail REPO_ROOT="$(git rev-parse --show-toplevel)" -run_tests_if_exists() { - local script="$1" - if [[ ! -f "${script}" ]]; then - echo "Skipping ${script} (file not found)" - return 0 - fi - "${script}" -} - echo "=== Running get_release_commit.sh tests ===" -run_tests_if_exists "${REPO_ROOT}/dev/scripts/test_get_release_commit.sh" +bash "${REPO_ROOT}/dev/scripts/test_get_release_commit.sh" diff --git a/.buildkite/scripts/trigger_backport_dryrun.sh b/.buildkite/scripts/trigger_backport_dryrun.sh new file mode 100755 index 00000000000..206d6a2c752 --- /dev/null +++ b/.buildkite/scripts/trigger_backport_dryrun.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# Triggered from the main pipeline when .backports.yml changes in a PR. +# For each entry that is new (absent from the base branch), uploads a trigger +# step that runs the integrations-backport pipeline in DRY_RUN mode. + +source .buildkite/scripts/common.sh + +set -euo pipefail + +if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then + echo "Not a pull request, skipping backport dry-run trigger" + exit 0 +fi + +if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" != "main" ]]; then + echo "Pull request does not target main (base branch: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}), skipping backport dry-run trigger" + exit 0 +fi + +add_bin_path +with_yq +with_mage + +from="$(get_from_changeset)" +to="$(get_to_changeset)" +commit_merge="$(git merge-base "${from}" "${to}")" + +if ! git diff --name-only "${commit_merge}" "${to}" | grep -qE '^\.backports\.yml$'; then + echo ".backports.yml not changed, skipping backport dry-run trigger" + exit 0 +fi + +echo "--- .backports.yml changed — finding new entries" + +BASE_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" + +OLD_INVENTORY="" +PIPELINE_FILE="" + +cleanup() { + local exit_code=$? + [[ -n "${OLD_INVENTORY}" ]] && rm -f "${OLD_INVENTORY}" + [[ -n "${PIPELINE_FILE}" ]] && rm -f "${PIPELINE_FILE}" + exit "${exit_code}" +} +trap cleanup EXIT + +OLD_INVENTORY="$(mktemp)" +NEW_INVENTORY=".backports.yml" + +if ! git show "origin/${BASE_BRANCH}:.backports.yml" > "${OLD_INVENTORY}" 2>/dev/null; then + echo ".backports.yml is new on ${BASE_BRANCH} — skipping dry-runs for initial entries" + echo "To validate new entries, add them in a follow-up PR after this one merges." + exit 0 +fi + +if ! yq -e '.backports' "${OLD_INVENTORY}" > /dev/null; then + echo "ERROR: old inventory is not valid YAML or missing 'backports' key: ${OLD_INVENTORY}" + exit 1 +fi + +if ! yq -e '.backports' "${NEW_INVENTORY}" > /dev/null; then + echo "ERROR: new inventory is not valid YAML or missing 'backports' key: ${NEW_INVENTORY}" + exit 1 +fi + +PIPELINE_FILE="$(mktemp --suffix=.yml)" +entries_found=0 + +while IFS= read -r branch; do + entry=".backports[] | select(.branch == \"${branch}\")" + + active_exit=0 + mage CheckBackportBranchActive "${branch}" || active_exit=$? + if [[ "${active_exit}" -eq 2 ]]; then + echo "ERROR: failed to check active status for branch '${branch}'" + exit 1 + fi + if [[ "${active_exit}" -ne 0 ]]; then + echo " Skipping inactive entry: ${branch}" + continue + fi + + # Only trigger for entries that are new (absent from the old inventory). + # If the entry already existed, the git branch is already created; there is + # nothing to provision. This also covers re-activating a previously archived + # entry (archived:true → false): the branch exists, so no dry-run is needed. + old_branch="$(yq "${entry} | .branch" "${OLD_INVENTORY}")" + + if [[ -n "${old_branch}" ]]; then + echo " Skipping existing entry: ${branch} (already present in base branch)" + continue + fi + + pkg="$(yq "${entry} | .package" "${NEW_INVENTORY}")" + base_version="$(yq "${entry} | .base_version" "${NEW_INVENTORY}")" + base_commit="$(yq "${entry} | .base_commit" "${NEW_INVENTORY}")" + + echo " Queuing dry-run: ${branch} (package=${pkg} version=${base_version} base_commit=${base_commit})" + + if [[ "${entries_found}" -eq 0 ]]; then + printf 'steps:\n' > "${PIPELINE_FILE}" + fi + + cat >> "${PIPELINE_FILE}" <- +// +// where is one or more letters, digits, or underscores, and +// starts with a digit followed by digits, dots, and an optional +// trailing 'x' wildcard (e.g. "6.x" or "6.14.x"). +// Whitespace, quotes, colons, semicolons, and all other special characters +// are not permitted. +var branchRE = regexp.MustCompile(`^backport-[a-zA-Z0-9_]+-[0-9][0-9.]*x?$`) + +// ActiveResult is the result of a CheckActive call. +type ActiveResult struct { + Branch string `json:"branch"` + Active bool `json:"active"` + Archived bool `json:"archived"` + MaintainedUntil *string `json:"maintained_until"` +} + +// CheckActive looks up branch in the inventory at path and reports whether it +// is currently active. now is injected so callers can test with a fixed date. +// Returns an error if the inventory cannot be read, parsed, or the branch is not found. +func CheckActive(path, branch string, now time.Time) (ActiveResult, error) { + data, err := os.ReadFile(path) + if err != nil { + return ActiveResult{}, fmt.Errorf("reading inventory: %w", err) + } + var inv inventory + if err := yaml.Unmarshal(data, &inv); err != nil { + return ActiveResult{}, fmt.Errorf("parsing inventory: %w", err) + } + for _, e := range inv.Backports { + if e.Branch == branch { + return e.activeResult(now), nil + } + } + return ActiveResult{}, fmt.Errorf("branch %q not found in %s", branch, path) +} + +// activeResult applies the active-branch rules for a single entry. +// +// A branch is inactive when: +// - archived == true, OR +// - maintained_until is set and is strictly before today (UTC). +func (e entry) activeResult(now time.Time) ActiveResult { + archived := e.Archived != nil && *e.Archived + result := ActiveResult{ + Branch: e.Branch, + Active: true, + Archived: archived, + MaintainedUntil: e.MaintainedUntil, + } + if archived { + result.Active = false + return result + } + if e.MaintainedUntil != nil { + t, err := time.Parse(maintainedUntilLayout, *e.MaintainedUntil) + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) + if err == nil && t.Before(today) { + result.Active = false + } + } + return result +} + +// ValidateInventory reads the .backports.yml inventory at path and returns a +// combined error listing every schema violation found across all entries. +// +// packagesDir is the path to the packages/ directory used to verify that each +// entry's package field names a real package. Pass an empty string to skip +// this check (useful in unit tests that do not have a full checkout). +func ValidateInventory(path, packagesDir string) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading inventory: %w", err) + } + + var inv inventory + if err := yaml.Unmarshal(data, &inv); err != nil { + return fmt.Errorf("parsing inventory: %w", err) + } + + knownPackages, err := buildKnownPackages(packagesDir) + if err != nil { + return fmt.Errorf("loading packages from %s: %w", packagesDir, err) + } + + var errs []error + for i, e := range inv.Backports { + errs = append(errs, validateEntryFields(i, e, knownPackages, packagesDir)...) + } + errs = append(errs, validateDuplicates(inv.Backports)...) + + return errors.Join(errs...) +} + +// validateEntryFields checks that all required fields of a single entry are +// present and contain valid values. +func validateEntryFields(i int, e entry, knownPackages map[string]struct{}, packagesDir string) []error { + id := fmt.Sprintf("entry[%d]", i) + if e.Branch != "" { + id = fmt.Sprintf("branch %q", e.Branch) + } + + var errs []error + + if e.Package == "" { + errs = append(errs, fmt.Errorf("%s: missing required field 'package'", id)) + } else if knownPackages != nil { + if _, ok := knownPackages[e.Package]; !ok { + errs = append(errs, fmt.Errorf("%s: unknown package %q: not found under %s", id, e.Package, packagesDir)) + } + } + + if e.Branch == "" { + errs = append(errs, fmt.Errorf("%s: missing required field 'branch'", id)) + } else if !branchRE.MatchString(e.Branch) { + errs = append(errs, fmt.Errorf("%s: invalid branch %q: must match backport-- "+ + "(letters/digits/underscores in package name, version starts with a digit; "+ + "no whitespace, quotes, colons, semicolons or other special characters)", id, e.Branch)) + } + + if e.BaseVersion == "" { + errs = append(errs, fmt.Errorf("%s: missing required field 'base_version'", id)) + } else if _, parseErr := semver.StrictNewVersion(e.BaseVersion); parseErr != nil { + errs = append(errs, fmt.Errorf("%s: invalid base_version %q: must be a valid semantic version", id, e.BaseVersion)) + } + + if e.BaseCommit == "" { + errs = append(errs, fmt.Errorf("%s: missing required field 'base_commit'", id)) + } else if !shaRE.MatchString(e.BaseCommit) { + errs = append(errs, fmt.Errorf("%s: invalid base_commit %q: must be a lowercase hex SHA (7–40 chars)", id, e.BaseCommit)) + } + + if e.Archived == nil { + errs = append(errs, fmt.Errorf("%s: missing required field 'archived'", id)) + } + + if e.MaintainedUntil != nil { + if _, parseErr := time.Parse(maintainedUntilLayout, *e.MaintainedUntil); parseErr != nil { + errs = append(errs, fmt.Errorf("%s: invalid maintained_until %q: must be YYYY-MM-DD", id, *e.MaintainedUntil)) + } + } + + return errs +} + +// validateDuplicates checks for duplicate branch names and duplicate +// package/version pairs across all entries. +func validateDuplicates(entries []entry) []error { + seenBranches := make(map[string]struct{}) + seenPackageVersions := make(map[string]struct{}) + + var errs []error + for i, e := range entries { + id := fmt.Sprintf("entry[%d]", i) + if e.Branch != "" { + id = fmt.Sprintf("branch %q", e.Branch) + } + + if e.Branch != "" { + if _, seen := seenBranches[e.Branch]; seen { + errs = append(errs, fmt.Errorf("%s: duplicate branch %q", id, e.Branch)) + } else { + seenBranches[e.Branch] = struct{}{} + } + } + + if e.Package != "" && e.BaseVersion != "" { + key := e.Package + "@" + e.BaseVersion + if _, isException := duplicatePackageVersionExceptions[key]; !isException { + if _, seen := seenPackageVersions[key]; seen { + errs = append(errs, fmt.Errorf("%s: duplicate package/version %q/%q", id, e.Package, e.BaseVersion)) + } else { + seenPackageVersions[key] = struct{}{} + } + } + } + } + + return errs +} + +// buildKnownPackages scans packagesDir and returns a set of valid package names. +// Returns nil (no error) when packagesDir is empty, skipping package validation. +func buildKnownPackages(packagesDir string) (map[string]struct{}, error) { + if packagesDir == "" { + return nil, nil + } + paths, err := citools.ListPackages(packagesDir) + if err != nil { + return nil, err + } + known := make(map[string]struct{}, len(paths)) + for _, p := range paths { + manifest, err := citools.ReadPackageManifest(filepath.Join(p, citools.ManifestFileName)) + if err != nil { + return nil, fmt.Errorf("reading manifest at %s: %w", p, err) + } + known[manifest.Name] = struct{}{} + } + return known, nil +} diff --git a/dev/backports/inventory_test.go b/dev/backports/inventory_test.go new file mode 100644 index 00000000000..9ca5bfa11f8 --- /dev/null +++ b/dev/backports/inventory_test.go @@ -0,0 +1,632 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package backports + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeTemp(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "backports.yml") + require.NoError(t, os.WriteFile(path, []byte(content), 0600)) + return path +} + +// writePackagesDir creates a minimal packages/ directory under t.TempDir() +// containing one package per name, each with a valid manifest.yml. +// Returns the path to the packages/ directory. +func writePackagesDir(t *testing.T, packageNames ...string) string { + t.Helper() + base := t.TempDir() + for _, name := range packageNames { + dir := filepath.Join(base, "packages", name) + require.NoError(t, os.MkdirAll(dir, 0700)) + manifest := "format_version: \"1.0.0\"\nname: " + name + "\ntype: integration\nversion: \"1.0.0\"\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.yml"), []byte(manifest), 0600)) + } + return filepath.Join(base, "packages") +} + +const validEntry = `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +` + +func TestValidateInventory(t *testing.T) { + cases := []struct { + title string + contents string + wantErr bool + errContains []string + }{ + { + title: "valid entry with null maintained_until", + contents: validEntry, + }, + { + title: "valid entry with date maintained_until", + contents: `backports: + - package: security_detection_engine + branch: backport-security_detection_engine-8.19 + base_version: "8.19.0" + base_commit: "abcdef1234" + maintained_until: "2027-01-15" + archived: false +`, + }, + { + title: "valid entry with prerelease base_version", + contents: `backports: + - package: apm + branch: backport-apm-8.15 + base_version: "8.15.0-preview-1716438434" + base_commit: "86356203eb" + maintained_until: null + archived: false +`, + }, + { + title: "empty backports list", + contents: "backports: []\n", + }, + { + title: "missing package field", + contents: `backports: + - branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"missing required field 'package'"}, + }, + { + title: "missing branch field", + contents: `backports: + - package: aws + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"missing required field 'branch'"}, + }, + { + title: "missing base_version field", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"missing required field 'base_version'"}, + }, + { + title: "missing base_commit field", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"missing required field 'base_commit'"}, + }, + { + title: "missing archived field", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null +`, + wantErr: true, + errContains: []string{"missing required field 'archived'"}, + }, + { + title: "invalid base_version — not a semver", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "not-a-version" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid base_version", "semantic version"}, + }, + { + title: "invalid base_version — missing patch segment", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid base_version"}, + }, + { + title: "invalid base_commit — not hex", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "xyz_not_hex" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid base_commit", "lowercase hex SHA"}, + }, + { + title: "invalid base_commit — too short", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "abc12" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid base_commit", "lowercase hex SHA"}, + }, + { + title: "invalid base_commit — uppercase hex", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5B593F6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid base_commit", "lowercase hex SHA"}, + }, + { + title: "valid branch with x wildcard version", + contents: `backports: + - package: aws + branch: backport-aws-6.x + base_version: "6.0.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + }, + { + title: "valid branch with x wildcard and minor", + contents: `backports: + - package: aws + branch: backport-aws-6.14.x + base_version: "6.14.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + }, + { + title: "invalid branch — missing backport- prefix", + contents: `backports: + - package: aws + branch: aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid branch"}, + }, + { + title: "invalid branch — version does not start with a digit", + contents: `backports: + - package: aws + branch: backport-aws-v3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid branch"}, + }, + { + title: "invalid branch — contains whitespace", + contents: `backports: + - package: aws + branch: "backport-aws 3.17" + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid branch"}, + }, + { + title: "invalid branch — contains colon", + contents: `backports: + - package: aws + branch: "backport-aws:3.17" + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid branch"}, + }, + { + title: "invalid branch — contains single quote", + contents: "backports:\n - package: aws\n branch: \"backport-aws-3'17\"\n base_version: \"3.17.0\"\n base_commit: \"5b593f6681\"\n maintained_until: null\n archived: false\n", + wantErr: true, + errContains: []string{"invalid branch"}, + }, + { + title: "invalid branch — no version segment", + contents: `backports: + - package: aws + branch: backport-aws + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"invalid branch"}, + }, + { + title: "invalid maintained_until format", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: "01/15/2027" + archived: false +`, + wantErr: true, + errContains: []string{"invalid maintained_until", "must be YYYY-MM-DD"}, + }, + { + title: "multiple entries with errors are all reported", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false + - branch: backport-aws-1.51 + base_version: "1.51.2" + base_commit: "88ad4b8432" + maintained_until: "not-a-date" + archived: true +`, + wantErr: true, + errContains: []string{"missing required field 'package'", "invalid maintained_until"}, + }, + { + title: "duplicate branch name", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false + - package: aws + branch: backport-aws-3.17 + base_version: "3.18.0" + base_commit: "aabbccddee" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{`duplicate branch "backport-aws-3.17"`}, + }, + { + title: "duplicate package and base_version", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false + - package: aws + branch: backport-aws-3.17x + base_version: "3.17.0" + base_commit: "aabbccddee" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"duplicate package/version", "aws", "3.17.0"}, + }, + { + title: "same package different versions is valid", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false + - package: aws + branch: backport-aws-3.13 + base_version: "3.13.0" + base_commit: "aabbccddee" + maintained_until: null + archived: false +`, + }, + { + title: "same version different packages is valid", + contents: `backports: + - package: aws + branch: backport-aws-1.0 + base_version: "1.0.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false + - package: azure + branch: backport-azure-1.0 + base_version: "1.0.0" + base_commit: "aabbccddee" + maintained_until: null + archived: false +`, + }, + { + title: "security_detection_engine 8.17.7 duplicate is allowed (known exception)", + contents: `backports: + - package: security_detection_engine + branch: backport-security_detection_engine-8.17 + base_version: "8.17.7" + base_commit: "5b593f6681" + maintained_until: null + archived: false + - package: security_detection_engine + branch: backport-security_detection_engine-8.18 + base_version: "8.17.7" + base_commit: "aabbccddee" + maintained_until: null + archived: false +`, + }, + { + title: "both duplicate branch and duplicate package/version are reported", + contents: `backports: + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "5b593f6681" + maintained_until: null + archived: false + - package: aws + branch: backport-aws-3.17 + base_version: "3.17.0" + base_commit: "aabbccddee" + maintained_until: null + archived: false +`, + wantErr: true, + errContains: []string{"duplicate branch", "duplicate package/version"}, + }, + } + + for _, tc := range cases { + t.Run(tc.title, func(t *testing.T) { + path := writeTemp(t, tc.contents) + err := ValidateInventory(path, "") + if tc.wantErr { + require.Error(t, err) + for _, substr := range tc.errContains { + assert.True(t, strings.Contains(err.Error(), substr), + "expected error to contain %q, got: %s", substr, err.Error()) + } + } else { + require.NoError(t, err) + } + }) + } +} + +func TestValidateInventoryPackageValidation(t *testing.T) { + contents := func(pkg string) string { + return `backports: + - package: ` + pkg + ` + branch: backport-` + pkg + `-1.0 + base_version: "1.0.0" + base_commit: "abcdef1234" + maintained_until: null + archived: false +` + } + + t.Run("known package passes", func(t *testing.T) { + packagesDir := writePackagesDir(t, "aws", "kubernetes") + path := writeTemp(t, contents("aws")) + require.NoError(t, ValidateInventory(path, packagesDir)) + }) + + t.Run("unknown package fails", func(t *testing.T) { + packagesDir := writePackagesDir(t, "aws") + path := writeTemp(t, contents("no_such_package")) + err := ValidateInventory(path, packagesDir) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown package") + assert.Contains(t, err.Error(), "no_such_package") + }) + + t.Run("empty packagesDir skips package check", func(t *testing.T) { + path := writeTemp(t, contents("totally_made_up")) + require.NoError(t, ValidateInventory(path, "")) + }) +} + +func TestValidateInventoryFileNotFound(t *testing.T) { + err := ValidateInventory("/no/such/file.yml", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "reading inventory") +} + +const checkActiveInventory = `backports: + - package: mypkg + branch: backport-mypkg-1.0 + base_version: "1.0.0" + base_commit: "aabbccddee" + maintained_until: null + archived: false + + - package: mypkg + branch: backport-mypkg-2.0 + base_version: "2.0.0" + base_commit: "11223344ff" + maintained_until: null + archived: true + + - package: mypkg + branch: backport-mypkg-3.0 + base_version: "3.0.0" + base_commit: "aabbccddee" + maintained_until: "2020-01-01" + archived: false + + - package: mypkg + branch: backport-mypkg-4.0 + base_version: "4.0.0" + base_commit: "aabbccddee" + maintained_until: "2099-12-31" + archived: false + + - package: mypkg + branch: backport-mypkg-5.0 + base_version: "5.0.0" + base_commit: "aabbccddee" + maintained_until: "2099-12-31" + archived: true +` + +func TestCheckActive(t *testing.T) { + now := time.Date(2026, 6, 4, 12, 0, 0, 0, time.UTC) + path := writeTemp(t, checkActiveInventory) + + cases := []struct { + branch string + wantActive bool + wantArchived bool + wantMaintained *string + wantErr bool + }{ + { + branch: "backport-mypkg-1.0", + wantActive: true, + wantArchived: false, + }, + { + branch: "backport-mypkg-2.0", + wantActive: false, + wantArchived: true, + }, + { + branch: "backport-mypkg-3.0", + wantActive: false, + wantArchived: false, + wantMaintained: ptr("2020-01-01"), + }, + { + branch: "backport-mypkg-4.0", + wantActive: true, + wantArchived: false, + wantMaintained: ptr("2099-12-31"), + }, + { + branch: "backport-mypkg-5.0", + wantActive: false, + wantArchived: true, + wantMaintained: ptr("2099-12-31"), + }, + { + branch: "backport-mypkg-no-such", + wantErr: true, + }, + } + + for _, tc := range cases { + t.Run(tc.branch, func(t *testing.T) { + result, err := CheckActive(path, tc.branch, now) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.branch, result.Branch) + assert.Equal(t, tc.wantActive, result.Active) + assert.Equal(t, tc.wantArchived, result.Archived) + assert.Equal(t, tc.wantMaintained, result.MaintainedUntil) + }) + } +} + +func TestCheckActiveMaintainedUntilBoundary(t *testing.T) { + mu := "2026-06-04" + inv := `backports: + - package: mypkg + branch: backport-mypkg-1.0 + base_version: "1.0.0" + base_commit: "aabbccddee" + maintained_until: "` + mu + `" + archived: false +` + path := writeTemp(t, inv) + + t.Run("same day is still active", func(t *testing.T) { + now := time.Date(2026, 6, 4, 23, 59, 59, 0, time.UTC) + result, err := CheckActive(path, "backport-mypkg-1.0", now) + require.NoError(t, err) + assert.True(t, result.Active) + }) + + t.Run("day after is inactive", func(t *testing.T) { + now := time.Date(2026, 6, 5, 0, 0, 0, 0, time.UTC) + result, err := CheckActive(path, "backport-mypkg-1.0", now) + require.NoError(t, err) + assert.False(t, result.Active) + }) +} + +func TestCheckActiveFileNotFound(t *testing.T) { + _, err := CheckActive("/no/such/file.yml", "some-branch", time.Now()) + require.Error(t, err) + assert.Contains(t, err.Error(), "reading inventory") +} + +func ptr(s string) *string { return &s } diff --git a/dev/scripts/README.md b/dev/scripts/README.md new file mode 100644 index 00000000000..09292ed2e97 --- /dev/null +++ b/dev/scripts/README.md @@ -0,0 +1,12 @@ +# dev/scripts + +Developer and maintenance scripts that are **not** involved in package testing. + +Because none of these scripts affect package behaviour, the entire `dev/scripts/` +directory is listed as a `non_package_pattern` in +`.buildkite/scripts/common.sh`. This means changes here do not trigger the +package test matrix in CI. + +If a script is ever added here that does affect package testing, the +`non_package_patterns` entry for `dev/scripts/` must be narrowed or removed +accordingly so that the relevant packages are still tested. diff --git a/dev/scripts/backport_bootstrap_inventory.sh b/dev/scripts/backport_bootstrap_inventory.sh new file mode 100755 index 00000000000..4bde5000ff5 --- /dev/null +++ b/dev/scripts/backport_bootstrap_inventory.sh @@ -0,0 +1,337 @@ +#!/usr/bin/env bash +# backport_bootstrap_inventory.sh +# +# IMPORTANT: One-time bootstrap script kept for auditability. +# Run once to seed .backports.yml from existing backport-* branches. +# Do NOT use this script to maintain the inventory after the initial seed — +# update .backports.yml directly from that point on. +# +# See issue #19210 for context. +# +# After running, perform a manual review pass to: +# - Fill in known maintained_until dates (e.g. stack EOL dates). +# - Verify base_version entries marked with a WARN comment. +# +# Usage: ./dev/scripts/backport_bootstrap_inventory.sh [-r REMOTE] [-o OUTPUT] [-n] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +REMOTE="upstream" +OUTPUT="${REPO_ROOT}/.backports.yml" +FETCH="true" + +usage() { + cat >&2 <<'EOF' +Usage: backport_bootstrap_inventory.sh [-r REMOTE] [-o OUTPUT] [-n] + +Seed .backports.yml from existing backport-* branches on a remote. + +Options: + -r REMOTE Git remote name (default: upstream) + -o OUTPUT Output file path (default: /.backports.yml) + -n No-fetch: skip fetching remote refs (branches must already + be locally cached as refs/remotes//backport-*) + -h Show this help + +After running, manually review the output and fill in maintained_until dates. +EOF + exit 1 +} + +while getopts ":r:o:nh" opt; do + case "${opt}" in + r) REMOTE="${OPTARG}" ;; + o) OUTPUT="${OPTARG}" ;; + n) FETCH="false" ;; + h) usage; exit 0 ;; + \?) + echo "Invalid option: -${OPTARG}" >&2 + usage + ;; + :) + echo "Missing argument for -${OPTARG}" >&2 + usage + ;; + esac +done + +if [[ "${FETCH}" == "true" ]]; then + echo "Fetching backport-* branches from ${REMOTE}..." >&2 + git fetch "${REMOTE}" \ + "refs/heads/backport-*:refs/remotes/${REMOTE}/backport-*" \ + --no-tags 2>&1 >&2 +fi + +ONE_YEAR_SECS=$(( 365 * 24 * 60 * 60 )) +NOW="$(date +%s)" + +# Resolve the package name to a manifest.yml path at a given git object ref. +# Tries flat layout (packages//manifest.yml) first, then one level of nesting +# (packages///manifest.yml), then falls back to scanning all manifests. +# Prints the path on success, nothing on failure. +find_manifest_path() { + local ref="${1}" + local pkg="${2}" + + # 1. Flat layout: packages//manifest.yml + if git cat-file -e "${ref}:packages/${pkg}/manifest.yml" 2>/dev/null; then + echo "packages/${pkg}/manifest.yml" + return 0 + fi + + # 2. One-level nesting: packages///manifest.yml + local nested + nested="$(git ls-tree --name-only "${ref}" packages/ 2>/dev/null | while IFS= read -r folder; do + candidate="packages/${folder}/${pkg}/manifest.yml" + if git cat-file -e "${ref}:${candidate}" 2>/dev/null; then + echo "${candidate}" + break + fi + done)" + if [[ -n "${nested}" ]]; then + echo "${nested}" + return 0 + fi + + # 3. Full scan: match by name field (handles deeper or unusual layouts) + while IFS= read -r candidate; do + local name + name="$(git show "${ref}:${candidate}" 2>/dev/null \ + | grep -m1 "^name:" \ + | sed "s/^name:[[:space:]]*//" \ + | tr -d "\"'")" + if [[ "${name}" == "${pkg}" ]]; then + echo "${candidate}" + return 0 + fi + done < <(git ls-tree -r --name-only "${ref}" packages/ 2>/dev/null \ + | grep '/manifest.yml$' \ + | grep -v '/data_stream/') + + return 1 +} + +# Read the version field from a manifest at a given git object ref + path. +get_version_at() { + local ref="${1}" + local manifest_path="${2}" + git show "${ref}:${manifest_path}" 2>/dev/null \ + | grep -m1 "^version:" \ + | sed "s/^version:[[:space:]]*//" \ + | tr -d "\"'" +} + +echo "Generating ${OUTPUT} from refs/remotes/${REMOTE}/backport-* ..." >&2 + +{ + cat <<'HEADER' +# Backport branch inventory — single source of truth. +# See issue #19210 for schema documentation. +# +# Active branch logic: +# inactive if: archived == true +# OR (maintained_until != null AND maintained_until < today) +# +# Fill in maintained_until for branches with known EOL dates (YYYY-MM-DD). +# Example: security_detection_engine-8.19 -> "2027-01-15" (8.19 EOL) +# +# This file was seeded by dev/scripts/backport_bootstrap_inventory.sh. +# Do not re-run the bootstrap — update this file directly. +HEADER + + echo "backports:" + + while IFS= read -r shortref; do + branch="${shortref#"${REMOTE}/"}" + ref="refs/remotes/${shortref}" + + # --- Parse package name and version suffix from branch name --- + # Branch format: backport--.[.] + stripped="${branch#backport-}" + + pkg="" + ver_suffix="" + if echo "${stripped}" | grep -qE '^.+-[0-9]+\.[0-9]+(\.[0-9]+)?$'; then + pkg="$(echo "${stripped}" | sed -E 's/-([0-9]+\.[0-9]+(\.[0-9]+)?)$//')" + ver_suffix="$(echo "${stripped}" | sed -E 's/^.*-([0-9]+\.[0-9]+(\.[0-9]+)?)$/\1/')" + fi + + if [[ -z "${pkg}" || -z "${ver_suffix}" ]]; then + echo " # WARNING: skipped — could not parse branch name: ${branch}" >&2 + printf ' # SKIPPED (unparseable): %s\n\n' "${branch}" + continue + fi + + echo " Processing ${branch} ..." >&2 + + # Major.minor from branch name (used to verify version family). + ver_major_minor="$(echo "${ver_suffix}" | cut -d. -f1-2)" + # Regex-safe major.minor for grep/sed matching. + ver_family_regex="^${ver_major_minor//./\\.}(\.|\$)" + + # --- base_commit resolution --- + # + # The BASE_COMMIT is the commit on main from which the backport branch was + # created. Three fast strategies produce candidates; we then pick the + # OLDEST (by commit timestamp) whose package version matches the branch's + # major.minor family. This correctly handles: + # + # • Recent branches: merge-base is still in main (strategy A). + # • Old branches with CI-sync at creation time: parent of OLDEST CI-sync + # commit is the BASE_COMMIT (strategy C). + # • Old branches where CI-sync was RE-DONE after cherry-picks: the + # CI-sync parent is a cherry-pick, not the base. Walking from HEAD + # skipping automation/backport commits finds the first "normal" commit + # (strategy B), and "oldest wins" picks the right one. + # + # If no candidate matches the version family, fall back to branch-tip version. + + base_commit="" + base_version="" + version_warn="" + + # Helper: check version family match. + matches_family() { echo "${1}" | grep -qE "${ver_family_regex}"; } + + # Helper: get commit timestamp (unix seconds). + commit_ts() { git log -1 --format="%ct" "${1}" 2>/dev/null; } + + # Collect candidate SHAs. + cand_merge_base="" + cand_walk="" + cand_ci_parent="" + + # Strategy A: git merge-base + cand_merge_base="$(git merge-base "refs/remotes/${REMOTE}/main" "${ref}" 2>/dev/null)" || true + + # Strategy B: walk from HEAD, skip CI-automation and explicit backport commits, + # take the first "normal" commit (within the top 30). + walk_count=0 + while IFS= read -r sha; do + (( walk_count++ )) || true + [[ "${walk_count}" -gt 30 ]] && break + msg="$(git log -1 --format="%s" "${sha}" 2>/dev/null)" + # Skip CI automation commits. + echo "${msg}" | grep -qiE \ + "Update .buildkite (folder )?from main|Copy .buildkite from main|Add .buildkite.*to backport branch" \ + && continue + # Skip explicit backport cherry-picks and double-PR cherry-picks. + # Double-PR pattern "(#NNNNN) (#MMMMM)" at end of message is the standard + # format for backport cherry-picks in this repo (first PR = original on main, + # second PR = the backport PR targeting this branch). + echo "${msg}" | grep -qiE "\[backport\]|\(backport\)|backporting |\(#[0-9]+\) \(#[0-9]+\)$" && continue + cand_walk="${sha}" + break + done < <(git log "${ref}" --format="%H" 2>/dev/null) + + # Strategy C: parent of oldest (first) CI-sync commit. + first_ci="$(git log "${ref}" --format="%H %s" 2>/dev/null \ + | grep -iE "Update .buildkite (folder )?from main|Copy .buildkite from main|Add .buildkite.*to backport branch" \ + | tail -1 | awk '{print $1}')" || true + if [[ -n "${first_ci}" ]]; then + cand_ci_parent="$(git rev-parse "${first_ci}^" 2>/dev/null)" || true + fi + + # Among all candidates, pick the OLDEST whose version matches the version family. + best_commit="" + best_version="" + best_ts=9999999999 + + for candidate in "${cand_merge_base}" "${cand_walk}" "${cand_ci_parent}"; do + [[ -z "${candidate}" ]] && continue + m="$(find_manifest_path "${candidate}" "${pkg}" 2>/dev/null)" || true + [[ -z "${m}" ]] && continue + ver="$(get_version_at "${candidate}" "${m}")" + matches_family "${ver}" || continue + ts="$(commit_ts "${candidate}")" + [[ -z "${ts}" ]] && continue + if [[ -z "${best_commit}" || "${ts}" -lt "${best_ts}" ]]; then + best_ts="${ts}" + best_commit="${candidate}" + best_version="${ver}" + fi + done + + if [[ -n "${best_commit}" ]]; then + base_commit="${best_commit}" + base_version="${best_version}" + echo " base_commit: ${base_commit:0:10} (version ${base_version})" >&2 + else + # Fallback: use stale merge-base as commit, branch tip for version. + base_commit="${cand_merge_base}" + tip_manifest="$(find_manifest_path "${ref}" "${pkg}" 2>/dev/null)" || true + if [[ -n "${tip_manifest}" ]]; then + base_version="$(get_version_at "${ref}" "${tip_manifest}")" + echo " NOTE: using branch-tip version '${base_version}' — verify base_commit manually" >&2 + fi + fi + + if [[ -z "${base_version}" ]]; then + base_version="unknown" + version_warn=" # WARN: could not determine base_version — fill in manually" + echo " WARNING: could not determine base_version for ${branch}" >&2 + fi + + base_commit_short="${base_commit:0:10}" + + # --- Check if base_commit is reachable from upstream/main --- + # Old branches whose entire history diverged from main before a force-push + # (or was never on main) produce a base_commit that is a valid branch ancestor + # but not reachable from the current main lineage. + base_commit_in_main="false" + if [[ -n "${base_commit}" ]]; then + if git merge-base --is-ancestor "${base_commit}" \ + "refs/remotes/${REMOTE}/main" 2>/dev/null; then + base_commit_in_main="true" + else + echo " NOTE: base_commit ${base_commit_short} not reachable from ${REMOTE}/main" >&2 + fi + fi + + # --- archived: true if last commit is older than one year --- + archived="false" + last_ts="$(git log -1 --format="%ct" "${ref}" 2>/dev/null)" || last_ts=0 + if (( NOW - last_ts > ONE_YEAR_SECS )); then + archived="true" + fi + + # --- Emit YAML entry --- + if [[ -n "${version_warn}" ]]; then + printf '%s\n' "${version_warn}" + fi + printf ' - package: %s\n' "${pkg}" + printf ' branch: %s\n' "${branch}" + printf ' base_version: "%s"\n' "${base_version}" + if [[ -n "${base_commit_short}" ]]; then + if [[ "${base_commit_in_main}" == "false" ]]; then + printf ' base_commit: "%s" # not in upstream/main; next commit is branch-exclusive\n' \ + "${base_commit_short}" + else + printf ' base_commit: "%s"\n' "${base_commit_short}" + fi + else + printf ' base_commit: null # WARN: could not determine\n' + fi + printf ' maintained_until: null\n' + printf ' archived: %s\n' "${archived}" + printf '\n' + + done < <(git for-each-ref --format='%(refname:short)' \ + "refs/remotes/${REMOTE}/backport-*" \ + | sed -E 's|^[^/]+/backport-([a-z_]+)-([0-9]+\.[0-9]+(\.[0-9]+)?)$|\1 \2 &|' \ + | sort -k1,1 -k2,2Vr \ + | awk '{print $NF}') + +} > "${OUTPUT}" + +echo "" >&2 +echo "Done. Written to ${OUTPUT}" >&2 +echo "" >&2 +echo "NEXT STEPS:" >&2 +echo " 1. Review ${OUTPUT} for WARN entries and fix them." >&2 +echo " 2. Fill in maintained_until for branches with known EOL dates." >&2 +echo " Format: \"YYYY-MM-DD\" (e.g. Elastic 8.19 EOL = 2027-01-15)" >&2 +echo " 3. Commit .backports.yml and this bootstrap script together." >&2 diff --git a/magefile.go b/magefile.go index c0d2d011b1c..efe680c86ca 100644 --- a/magefile.go +++ b/magefile.go @@ -8,17 +8,20 @@ package main import ( "context" + "encoding/json" "fmt" "io" "os" "path/filepath" "strconv" + "time" "github.com/Masterminds/semver/v3" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" "github.com/pkg/errors" + "github.com/elastic/integrations/dev/backports" "github.com/elastic/integrations/dev/citools" "github.com/elastic/integrations/dev/codeowners" "github.com/elastic/integrations/dev/coverage" @@ -218,6 +221,11 @@ func ReportFailedTests(ctx context.Context, testResultsFolder string) error { return testsreporter.Check(ctx, testResultsFolder, options) } +// ValidateBackportsInventory validates the schema of .backports.yml at the repo root. +func ValidateBackportsInventory() error { + return backports.ValidateInventory(".backports.yml", "packages") +} + // ListPackages lists all packages found under the packages directory. func ListPackages() error { const packagesDir = "packages" @@ -314,6 +322,38 @@ func IsVersionLessThanLogsDBGA(version string) error { return nil } +// CheckBackportBranchActive reports whether a backport branch is active per .backports.yml. +// Prints ": active" or ": inactive ()". +// Pass -json for JSON output: mage CheckBackportBranchActive -json +// Exit codes: 0 = active, 1 = inactive, 2 = error (branch not found, parse error, etc.). +func CheckBackportBranchActive(branch string, asJSON *bool) error { + result, err := backports.CheckActive(".backports.yml", branch, time.Now().UTC()) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(2) + } + + if asJSON != nil && *asJSON { + data, _ := json.Marshal(result) + fmt.Println(string(data)) + } else { + if result.Active { + fmt.Printf("%s: active\n", branch) + } else { + reason := "archived" + if !result.Archived && result.MaintainedUntil != nil { + reason = fmt.Sprintf("maintained_until=%s is past", *result.MaintainedUntil) + } + fmt.Printf("%s: inactive (%s)\n", branch, reason) + } + } + + if !result.Active { + os.Exit(1) + } + return nil +} + // IsElasticPackageDependencyLessThan checks whether or not the elastic-package version set in go.mod is less than the given version func IsElasticPackageDependencyLessThan(version string) error { foundVersion, err := citools.PackageVersionGoMod("go.mod", elasticPackageModulePath)