[WIP] Jv perf scale baseline only has nightlies

ci-operator/config/stackrox/stackrox/stackrox-stackrox-jv-test-perf-scale__perf-scale.yaml

@@ -0,0 +1,70 @@
+base_images:
+  ocp-4:
+    name: automation-flavors
+    namespace: stackrox
+    tag: openshift-4-stable
+build_root:
+  image_stream_tag:
+    name: apollo-ci
+    namespace: stackrox
+    tag: stackrox-ui-test-0.5.2
+releases:
+  latest:
+    release:
+      channel: stable
+      version: 4.20.8
+resources:
+  '*':
+    requests:
+      cpu: 2000m
+      memory: 4000Mi
+test_binary_build_commands: .openshift-ci/dispatch.sh test-binary-build-commands
+tests:
+- always_run: false
+  as: 24nodes-scale-test
+  optional: true
+  reporter_config:
+    channel: '#acs-perfscale'
+    job_states_to_report:
+    - success
+    - failure
+    - error
+    report_template: '{{if eq .Status.State "success"}} :slack-green: Job *{{.Spec.Job}}*
+      ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> {{else}} :failed:
+      Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs>
+      {{end}}'
+  steps:
+    allow_skip_on_success: true
+    cluster_profile: aws-stackrox
+    env:
+      BASE_DOMAIN: perfscale.rox.systems
+      CD_V2_EXTRA_FLAGS: --churn-duration=20m --timeout=8h --metrics-profile metrics-aggregated.yml,https://raw.githubusercontent.com/stackrox/stackrox/refs/heads/master/tests/performance/scale/config/metrics-acs.yml
+      COMPUTE_NODE_REPLICAS: "24"
+      PODS_PER_NODE: "240"
+      ROX_SCANNER_V4_ENABLED: "true"
+      SCANNER_V4_MATCHER_READINESS: vulnerability
+      SCANNER_V4_MATCHER_READINESS_MAX_WAIT: "3600"
+      TEST_SUITE: ocp-perf-scale-tests
+      USER_TAGS: |
+        TicketId 382
+        RoxProject PerfScale
+      ZONES_COUNT: "3"
+    post:
+    - chain: gather-core-dump
+    - chain: ipi-aws-post
+    - ref: stackrox-stackrox-end
+    pre:
+    - ref: stackrox-stackrox-begin
+    - ref: stackrox-stackrox-e2e-test
+    - chain: ipi-conf-aws
+    - chain: ipi-install
+    - chain: create-infra-move-ingress-monitoring-registry
+    - ref: stackrox-install-helm
+    test:
+    - chain: stackrox-perfscale
+  timeout: 14h0m0s
+zz_generated_metadata:
+  branch: jv-test-perf-scale
+  org: stackrox
+  repo: stackrox
+  variant: perf-scale

ci-operator/jobs/stackrox/stackrox/stackrox-stackrox-jv-test-perf-scale-presubmits.yaml

@@ -0,0 +1,99 @@
+presubmits:
+  stackrox/stackrox:
+  - agent: kubernetes
+    always_run: false
+    branches:
+    - ^jv-test-perf-scale$
+    - ^jv-test-perf-scale-
+    cluster: build07
+    context: ci/prow/perf-scale-24nodes-scale-test
+    decorate: true
+    decoration_config:
+      skip_cloning: true
+      timeout: 14h0m0s
+    labels:
+      ci-operator.openshift.io/cloud: aws
+      ci-operator.openshift.io/cloud-cluster-profile: aws-stackrox
+      ci-operator.openshift.io/variant: perf-scale
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-stackrox-stackrox-jv-test-perf-scale-perf-scale-24nodes-scale-test
+    optional: true
+    reporter_config:
+      slack:
+        channel: '#acs-perfscale'
+        job_states_to_report:
+        - success
+        - failure
+        - error
+        report_template: '{{if eq .Status.State "success"}} :slack-green: Job *{{.Spec.Job}}*
+          ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> {{else}} :failed:
+          Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View
+          logs> {{end}}'
+    rerun_command: /test perf-scale-24nodes-scale-test
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --report-credentials-file=/etc/report/credentials
+        - --secret-dir=/secrets/ci-pull-credentials
+        - --target=24nodes-scale-test
+        - --variant=perf-scale
+        command:
+        - ci-operator
+        env:
+        - name: HTTP_SERVER_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+        imagePullPolicy: Always
+        name: ""
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/ci-pull-credentials
+          name: ci-pull-credentials
+          readOnly: true
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: ci-pull-credentials
+        secret:
+          secretName: ci-pull-credentials
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )perf-scale-24nodes-scale-test,?($|\s.*)

ci-operator/step-registry/openshift-qe/orion/cluster-density/openshift-qe-orion-cluster-density-ref.yaml

@@ -82,6 +82,13 @@ ref:
     default: "cluster-density"
     documentation:
       Workload type suffix for auto-selecting ORION_CONFIG based on worker node count.
+  - name: ORION_BASELINE_JOB_TYPE
+    default: ""
+    documentation: |-
+      When set, overrides the automatic job_type detection logic and uses this value
+      as the baseline job_type filter for Orion comparisons. This allows teams to control
+      which runs are included in the baseline. For example, set to "periodic" to always
+      compare against nightly periodic runs only, even when the job is triggered from a PR.
   commands: openshift-qe-orion-cluster-density-commands.sh
   timeout: 6h
   credentials:

ci-operator/step-registry/openshift-qe/orion/crd-scale/openshift-qe-orion-crd-scale-ref.yaml

@@ -78,6 +78,13 @@ ref:
     default: "kube-burner/kube-burner,kube-burner/kube-burner-ocp,cloud-bulldozer/e2e-benchmarking,openshift/release"
     documentation:
       Comma separated list of repositories to monitor before and after change points.
+  - name: ORION_BASELINE_JOB_TYPE
+    default: ""
+    documentation: |-
+      When set, overrides the automatic job_type detection logic and uses this value
+      as the baseline job_type filter for Orion comparisons. This allows teams to control
+      which runs are included in the baseline. For example, set to "periodic" to always
+      compare against nightly periodic runs only, even when the job is triggered from a PR.
   commands: openshift-qe-orion-crd-scale-commands.sh
   timeout: 6h
   credentials:

ci-operator/step-registry/openshift-qe/orion/node-density/openshift-qe-orion-node-density-ref.yaml

@@ -82,6 +82,13 @@ ref:
     default: "node-density"
     documentation:
       Workload type suffix for auto-selecting ORION_CONFIG based on worker node count.
+  - name: ORION_BASELINE_JOB_TYPE
+    default: ""
+    documentation: |-
+      When set, overrides the automatic job_type detection logic and uses this value
+      as the baseline job_type filter for Orion comparisons. This allows teams to control
+      which runs are included in the baseline. For example, set to "periodic" to always
+      compare against nightly periodic runs only, even when the job is triggered from a PR.
   commands: openshift-qe-orion-node-density-commands.sh
   timeout: 6h
   credentials:

ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-commands.sh

@@ -1,6 +1,46 @@
 #!/bin/bash
 set -x
 
+classify_baseline_job() {
+    if [[ -n "${ORION_BASELINE_JOB_TYPE:-}" ]]; then
+        echo "override"
+    elif [[ "${JOB_TYPE}" == "periodic" ]]; then
+        if [[ -n "${PULL_NUMBER:-}" ]] && [[ "${PULL_NUMBER}" -ne 0 ]]; then
+            echo "periodic_with_pull"
+        else
+            echo "periodic"
+        fi
+    elif [[ "${JOB_TYPE}" == "presubmit" && "${JOB_NAME}" =~ ^pull* ]] && [[ -n "${PULL_NUMBER:-}" ]]; then
+        echo "presubmit_pull"
+    elif [[ "${JOB_TYPE}" == "presubmit" && "${JOB_NAME}" == *rehearse* ]] && [[ -n "${PULL_NUMBER:-}" ]]; then
+        echo "rehearse_with_pull"
+    elif [[ "${JOB_TYPE}" == "presubmit" && "${JOB_NAME}" == *rehearse* ]]; then
+        echo "rehearse"
+    fi
+}
+
+resolve_baseline_job_type() {
+    local baseline_category=$1
+    case "$baseline_category" in
+        override)           echo "${ORION_BASELINE_JOB_TYPE}" ;;
+        periodic_with_pull) echo "(periodic OR pull)" ;;
+        periodic)           echo "periodic" ;;
+        presubmit_pull)     echo "(periodic OR pull)" ;;
+        rehearse_with_pull) echo "(rehearse OR pull OR periodic)" ;;
+        rehearse)           echo "(periodic OR rehearse)" ;;
+    esac
+}
+
+resolve_pull_number() {
+    local baseline_category=$1
+    case "$baseline_category" in
+        periodic_with_pull|presubmit_pull|rehearse_with_pull)
+            echo "(${PULL_NUMBER} OR 0)" ;;
+        *)
+            echo "0" ;;
+    esac
+}
+
 if [ ${RUN_ORION} == false ]; then
   exit 0
 fi
@@ -186,28 +226,9 @@ if [[ -n "${CHANGE_POINT_REPOS}" ]]; then
     EXTRA_FLAGS+=" --github-repos ${CHANGE_POINT_REPOS}"
 fi
 
-# pull_number input variable is required and
-# it must be set as $PULL_NUMBER OR 0 to get compared against periodic runs.
-pull_number='0'
-if [[ "${JOB_TYPE}" == "periodic" ]]; then
-    if [[ -n "${PULL_NUMBER:-}" ]] && [[ "${PULL_NUMBER}" -ne 0 ]]; then
-        pull_number="(${PULL_NUMBER} OR 0)"
-        job_type="(periodic OR pull)"
-    else
-        job_type="periodic"
-    fi
-elif [[ "${JOB_TYPE}" == "presubmit" && "${JOB_NAME}" =~ ^pull* ]] && [[ -n "${PULL_NUMBER:-}" ]]; then
-    # Indicates a ci test triggered in PR against a pull request
-    pull_number="(${PULL_NUMBER} OR 0)"
-    job_type="(periodic OR pull)"
-elif [[ "${JOB_TYPE}" == "presubmit" && "${JOB_NAME}" == *rehearse* ]] && [[ -n "${PULL_NUMBER:-}" ]]; then
-    # Indicates a rehearse job triggered from a PR
-    pull_number="(${PULL_NUMBER} OR 0)"
-    job_type="(rehearse OR pull OR periodic)"
-elif [[ "${JOB_TYPE}" == "presubmit" && "${JOB_NAME}" == *rehearse* ]]; then
-    # Indicates a rehearsal in PR against openshift/release repo
-    job_type="(periodic OR rehearse)"
-fi
+baseline_category=$(classify_baseline_job)
+job_type=$(resolve_baseline_job_type "$baseline_category")
+pull_number=$(resolve_pull_number "$baseline_category")
 
 set +e
 set -o pipefail

ci-operator/step-registry/openshift-qe/orion/openshift-qe-orion-ref.yaml

@@ -86,6 +86,13 @@ ref:
       When set, auto-selects ORION_CONFIG based on worker node count and this workload suffix.
       Matches orion example configs named {scale-prefix}-{ORION_WORKLOAD_TYPE}.yaml.
       Ignored if ORION_CONFIG is explicitly set.
+  - name: ORION_BASELINE_JOB_TYPE
+    default: ""
+    documentation: |-
+      When set, overrides the automatic job_type detection logic and uses this value
+      as the baseline job_type filter for Orion comparisons. This allows teams to control
+      which runs are included in the baseline. For example, set to "periodic" to always
+      compare against nightly periodic runs only, even when the job is triggered from a PR.
 
   commands: openshift-qe-orion-commands.sh
   timeout: 6h

ci-operator/step-registry/stackrox/perfscale/stackrox-perfscale-chain.yaml

@@ -22,6 +22,8 @@ chain:
     documentation: Output format for Orion (JUNIT, JSON, or TEXT). JUNIT format enables better error display in Prow.
   - name: RUN_ORION
     default: "true"
+  - name: ORION_BASELINE_JOB_TYPE
+    default: "periodic"
   - name: VERSION
     default: "4.20"
   documentation: |-