Allow use of version qualifier for staging artifacts

This commits adds support for an optional $VERSION_QUALIFIER env var
in the packaging pipeline to allow building prerelease artifacts.

Relates elastic/ingest-dev#4855

.buildkite/packaging.pipeline.yml

@@ -24,7 +24,9 @@ steps:
     key: start-gate-snapshot
 
   - name: Start of concurrency group for DRA Staging
-    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+    # exceptionally allow building staging from main when VERSION_QUALIFIER is set, to allow prerelease testing
+    # TODO remove OR clause below and above comment, and only allow matching /^[0-9]+\.[0-9x]+\$/ for build.branch
+    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
     command: echo "--> Start of concurrency gate dra-staging"
     concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
     concurrency: 1
@@ -58,7 +60,8 @@ steps:
           - build/distributions/**/*
 
       - label: Staging dashboards
-        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+        # TODO remove OR clause below (see earlier comment)
+        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
         depends_on: start-gate-staging
         key: dashboards-staging
         # TODO: container with go and make
@@ -170,8 +173,8 @@ steps:
   - group: Packaging Staging
     key: packaging-staging
     depends_on: start-gate-staging
-    ## Only for release
-    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+    # TODO remove OR clause below (see earlier comment)
+    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
     steps:
       - label: "STAGING: {{matrix}}"
         env:
@@ -276,8 +279,8 @@ steps:
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
 
       - label: DRA Staging
-        ## Only for release branches
-        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+        # TODO remove OR clause below (see earlier comment)
+        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
         key: dra-staging
         env:
           DRA_WORKFLOW: staging

.buildkite/scripts/dra.sh

@@ -15,7 +15,7 @@ set -euo pipefail
 BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
 
 BEAT_VERSION=$(make get-version)
-
+VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 
 function release_manager_login {
@@ -49,7 +49,9 @@ docker run --rm \
         --commit "${BUILDKITE_COMMIT}" \
         --workflow "${DRA_WORKFLOW}" \
         --version "${BEAT_VERSION}" \
-        --artifact-set "main"
+        --artifact-set "main" \
+        --qualifier "${VERSION_QUALIFIER}"
+
 
 echo "+++ :hammer_and_pick: Publishing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH], workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]"
 
@@ -68,6 +70,7 @@ docker run --rm \
         --workflow "${DRA_WORKFLOW}" \
         --version "${BEAT_VERSION}" \
         --artifact-set "main" \
+        --qualifier "${VERSION_QUALIFIER}" \
         ${DRY_RUN} | tee rm-output.txt