[ci] Remove hardcoded branches from DRA schedule (#15668)

This commit is the first making use of #15627 to remove hard coded
branches for the DRA Snapshot build schedule.

With this pattern, we will only need to keep `ci/branches.json` up to date,
as versions evolve, and not need to update/maintain hard coded branches
in `catalog-info.yaml` anymore.

Once this is verified working, we'll add a corresponding schedule
pipeline (in `catalog.info`) for the JDK matrix job.

Relates: elastic/ingest-dev#2664

.buildkite/scripts/common/trigger-pipeline-generate-steps.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# *******************************************************
+# this script is used by schedule-type pipelines
+# to automate triggering other pipelines (e.g. DRA) based
+# on ci/branches.json
+#
+# See:
+# https://elasticco.atlassian.net/browse/ENGPRD-318 /
+# https://github.com/elastic/ingest-dev/issues/2664
+# *******************************************************
+
+ACTIVE_BRANCHES_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/branches.json"
+
+function install_yq() {
+if ! [[ -x $(which yq) && $(yq --version) == *mikefarah* ]]; then
+  echo "--- Downloading prerequisites"
+  curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+  chmod a+x /usr/local/bin/yq
+fi
+}
+
+if [[ -z $PIPELINE_TO_TRIGGER ]]; then
+    echo "^^^ +++"
+    echo "Required environment variable [PIPELINE_TO_TRIGGER] is missing."
+    echo "Exiting now."
+    exit 1
+fi
+
+set -u
+set +e
+BRANCHES=$(curl --retry-all-errors --retry 5 --retry-delay 5 -fsSL $ACTIVE_BRANCHES_URL | jq -r '.branches[].branch')
+if [[ $? -ne 0 ]]; then
+  echo "There was an error downloading or parsing the json output from [$ACTIVE_BRANCHES_URL]. Exiting."
+  exit 1
+fi
+
+set -e
+
+install_yq
+
+echo 'steps:' >pipeline_steps.yaml
+
+for BRANCH in $BRANCHES; do
+    cat >>pipeline_steps.yaml <<EOF
+  - trigger: $PIPELINE_TO_TRIGGER
+    label: ":testexecute: Triggering ${PIPELINE_TO_TRIGGER} / ${BRANCH}"
+    branches: "main"
+    async: true
+    build:
+      branch: "$BRANCH"
+      message: ":testexecute: Triggering ${PIPELINE_TO_TRIGGER} / ${BRANCH}"
+EOF
+done
+
+echo "--- Printing generated steps"
+yq . pipeline_steps.yaml
+
+echo "--- Uploading steps to buildkite"
+cat pipeline_steps.yml | buildkite-agent pipeline upload

.buildkite/trigger_pipeline.yml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+steps:
+  - label: ":pipeline: Generate trigger steps for $PIPELINE_TO_TRIGGER"
+    command: ".buildkite/scripts/trigger-pipeline-generate-steps.sh"

catalog-info.yaml

@@ -140,33 +140,63 @@ spec:
     kind: Pipeline
     metadata:
       name: logstash-dra-snapshot-pipeline
-      description: ':logstash: The DRA SNAPSHOT (Daily, Auto) pipeline'
+      description: ':logstash: The DRA SNAPSHOT pipeline'
     spec:
       repository: elastic/logstash
       pipeline_file: ".buildkite/dra_pipeline.yml"
       maximum_timeout_in_minutes: 120
+      skip_intermediate_builds: true
+      provider_settings:
+        trigger_mode: none
+      env:
+        WORKFLOW_TYPE: 'snapshot'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        SLACK_NOTIFICATIONS_CHANNEL: '#logstash'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        logstash:
+          access_level: MANAGE_BUILD_AND_READ
+        ingest-eng-prod:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: logstash-scheduled-dra-snapshot-pipeline
+  description: 'Scheduled runs (Daily, Auto) of Logstash DRA SNAPSHOT pipeline per branch'
+  links:
+    - title: 'Scheduled runs (Daily, Auto) of Logstash DRA SNAPSHOT pipeline per branch'
+      url: https://buildkite.com/elastic/logstash-scheduled-dra-snapshot-pipeline
+spec:
+  type: buildkite-pipeline
+  owner: group:logstash
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: Logstash Scheduled DRA Snapshot Pipeline
+      description: ':alarm_clock: Scheduled runs of Logstash DRA Snapshot Pipeline per eligible branch'
+    spec:
+      repository: elastic/logstash
+      pipeline_file: ".buildkite/trigger_pipeline.yml"
+      maximum_timeout_in_minutes: 120
       schedules:
-        Daily 7_17:
-          branch: '7.17'
-          cronline: 30 02 * * *
-          message: Daily SNAPSHOT build for 7.17
-        Daily 8_11:
-          branch: '8.11'
-          cronline: 30 02 * * *
-          message: Daily SNAPSHOT build for 8.11
-        Daily 8_12:
-          branch: '8.12'
-          cronline: 30 02 * * *
-          message: Daily SNAPSHOT build for 8.12
         Daily main:
           branch: main
           cronline: 30 02 * * *
-          message: Daily SNAPSHOT build for main
+          message: Daily trigger of DRA Pipeline per branch
       skip_intermediate_builds: true
       provider_settings:
         trigger_mode: none
       env:
-        WORKFLOW_TYPE: 'snapshot'
+        PIPELINE_TO_TRIGGER: 'logstash-dra-snapshot-pipeline'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         SLACK_NOTIFICATIONS_CHANNEL: '#logstash'
         SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'