diff --git a/.buildkite/aws-tests-pipeline.yml b/.buildkite/aws-tests-pipeline.yml index 44f6d5a848a2..c25b420179eb 100644 --- a/.buildkite/aws-tests-pipeline.yml +++ b/.buildkite/aws-tests-pipeline.yml @@ -19,7 +19,7 @@ env: steps: - label: ":ubuntu: x-pack/filebeat: AWS Tests" key: "x-pack-filebeat-extended-cloud-test" - skip: "skipping as it was on Jenkins: elastic/ingest-dev#3467" + skip: "skipping: elastic/ingest-dev#3467" # Related issue: https://github.com/elastic/ingest-dev/issues/3467 env: MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform" diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index e675a4b8a175..29732955f7cf 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -292,7 +292,7 @@ steps: - label: ":ubuntu: x-pack/filebeat: AWS Tests" key: "x-pack-filebeat-extended-cloud-test" - skip: "skipping as it was on Jenkins: elastic/ingest-dev#3467" + skip: "skipping: elastic/ingest-dev#3467" # Related issue: https://github.com/elastic/ingest-dev/issues/3467 if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ env: diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index a9a4b82dd0a3..6695957179d7 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -130,7 +130,6 @@ steps: - github_commit_status: context: "x-pack/heartbeat: Win 2016 Unit Tests" - # Doesn't exist in Jenkins - label: ":windows: x-pack/heartbeat: Win 2022 Unit Tests" key: "mandatory-win-2022-unit-tests" command: | @@ -178,7 +177,6 @@ steps: - github_commit_status: context: "x-pack/heartbeat: Win 10 Unit Tests" - # Doesn't exist in Jenkins - label: ":windows: x-pack/heartbeat: Win 11 Unit Tests" command: | Set-Location -Path x-pack/heartbeat diff --git a/.ci/beats-tester-bc.groovy b/.ci/beats-tester-bc.groovy deleted file mode 100644 index 7916c4252357..000000000000 --- a/.ci/beats-tester-bc.groovy +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -pipeline { - agent none - environment { - BASE_DIR = 'src/github.com/elastic/beats' - PIPELINE_LOG_LEVEL = "INFO" - BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/main' - BASE_URL = "https://staging.elastic.co/${params.version}/downloads" - APM_BASE_URL = "${env.BASE_URL}/apm-server" - BEATS_BASE_URL = "${env.BASE_URL}/beats" - VERSION = "${params.version?.split('-')[0]}" - } - options { - timeout(time: 2, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - disableConcurrentBuilds() - } - parameters { - string(name: 'version', defaultValue: '', description: 'Id of the Build Candidate (7.10.0-b55684ff).') - string(name: 'BRANCH_REFERENCE', defaultValue: 'main', description: 'Branch to grab the Groovy script(for test changes).') - } - stages { - stage('Run Beat Tester') { - options { skipDefaultCheckout() } - when { - expression { - return '' != "${VERSION}" - } - } - steps { - build(job: env.BEATS_TESTER_JOB, propagate: true, wait: true, - parameters: [ - string(name: 'APM_URL_BASE', value: "${APM_BASE_URL}"), - string(name: 'BEATS_URL_BASE', value: "${BEATS_BASE_URL}"), - string(name: 'VERSION', value: "${VERSION}") - ]) - } - } - } -} diff --git a/.ci/beats-tester.groovy b/.ci/beats-tester.groovy deleted file mode 100644 index a560b455c6e7..000000000000 --- a/.ci/beats-tester.groovy +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -pipeline { - agent none - environment { - BASE_DIR = 'src/github.com/elastic/beats' - PIPELINE_LOG_LEVEL = "INFO" - BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/main' - } - options { - timeout(time: 1, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - disableConcurrentBuilds() - } - triggers { - issueCommentTrigger('(?i)^\\/beats-tester$') - upstream("Beats/packaging/${env.JOB_BASE_NAME}") - } - stages { - stage('Filter build') { - agent { label 'ubuntu-22' } - when { - beforeAgent true - anyOf { - triggeredBy cause: "IssueCommentCause" - expression { - def ret = isUserTrigger() || isUpstreamTrigger() - if(!ret){ - currentBuild.result = 'NOT_BUILT' - currentBuild.description = "The build has been skipped" - currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)" - echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job") - } - return ret - } - } - } - stages { - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - deleteDir() - gitCheckout(basedir: "${BASE_DIR}") - setEnvVar('VERSION', sh(script: "grep ':stack-version:' ${BASE_DIR}/libbeat/docs/version.asciidoc | cut -d' ' -f2", returnStdout: true).trim()) - } - } - stage('Build main') { - options { skipDefaultCheckout() } - when { branch 'main' } - steps { - runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") - } - } - stage('Build *.x branch') { - options { skipDefaultCheckout() } - when { branch '*.x' } - steps { - runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") - } - } - stage('Build PullRequest') { - options { skipDefaultCheckout() } - when { changeRequest() } - steps { - runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT", - beats: "https://storage.googleapis.com/beats-ci-artifacts/beats/pull-requests/pr-${env.CHANGE_ID}") - } - } - stage('Build release branch') { - options { skipDefaultCheckout() } - when { - not { - anyOf { - branch comparator: 'REGEXP', pattern: '(main|.*x)' - changeRequest() - } - } - } - steps { - // TODO: to use the git commit that triggered the upstream build - runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT") - } - } - } - } - } -} - -def runBeatsTesterJob(Map args = [:]) { - def apm = args.get('apm', '') - def beats = args.get('beats', '') - def version = args.version - - if (isUpstreamTrigger()) { - copyArtifacts(filter: 'beats-tester.properties', - flatten: true, - projectName: "Beats/packaging/${env.JOB_BASE_NAME}", - selector: upstream(fallbackToLastSuccessful: true)) - def props = readProperties(file: 'beats-tester.properties') - apm = props.get('APM_URL_BASE', '') - beats = props.get('BEATS_URL_BASE', '') - version = props.get('VERSION', '8.7.0-SNAPSHOT') - } - if (apm?.trim() || beats?.trim()) { - build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, - parameters: [ - string(name: 'APM_URL_BASE', value: apm), - string(name: 'BEATS_URL_BASE', value: beats), - string(name: 'VERSION', value: version) - ]) - } else { - build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: version) ]) - } -} diff --git a/.ci/build-docker-images.groovy b/.ci/build-docker-images.groovy deleted file mode 100644 index 5cf4212a9214..000000000000 --- a/.ci/build-docker-images.groovy +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -pipeline { - agent { label 'ubuntu-22' } - environment { - REPO = 'beats' - BASE_DIR = "src/github.com/elastic/${env.REPO}" - DOCKER_REGISTRY = 'docker.elastic.co' - DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod' - HOME = "${env.WORKSPACE}" - NOTIFY_TO = credentials('notify-to') - PIPELINE_LOG_LEVEL = 'INFO' - JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" - } - options { - timeout(time: 1, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) - quietPeriod(10) - } - triggers { - cron 'H H(0-5) * * 1-5' - } - parameters { - booleanParam(name: "RELEASE_TEST_IMAGES", defaultValue: "true", description: "If it's needed to build & push Beats' test images") - string(name: 'BRANCH_REFERENCE', defaultValue: "main", description: "Git branch/tag to use") - } - stages { - stage('Checkout') { - steps { - deleteDir() - gitCheckout(basedir: "${BASE_DIR}", - branch: "${params.BRANCH_REFERENCE}", - repo: "https://github.com/elastic/${REPO}.git", - credentialsId: "${JOB_GIT_CREDENTIALS}" - ) - dir("${BASE_DIR}"){ - setEnvVar("GO_VERSION", readFile(file: ".go-version")?.trim()) - } - } - } - stage('Metricbeat Test Docker images'){ - options { - warnError('Metricbeat Test Docker images failed') - } - when { - expression { return params.RELEASE_TEST_IMAGES } - } - steps { - dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}") - withMageEnv(){ - dir("${BASE_DIR}/metricbeat"){ - retryWithSleep(retries: 3, seconds: 5, backoff: true){ - sh(label: 'Build', script: "mage compose:buildSupportedVersions"); - sh(label: 'Push', script: "mage compose:pushSupportedVersions"); - } - } - } - } - } - stage('Metricbeat x-pack Test Docker images'){ - options { - warnError('Metricbeat x-pack Docker images failed') - } - when { - expression { return params.RELEASE_TEST_IMAGES } - } - steps { - dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}") - withMageEnv(){ - dir("${BASE_DIR}/x-pack/metricbeat"){ - retryWithSleep(retries: 3, seconds: 5, backoff: true){ - sh(label: 'Build', script: "mage compose:buildSupportedVersions"); - sh(label: 'Push', script: "mage compose:pushSupportedVersions"); - } - } - } - } - } - stage('Filebeat x-pack Test Docker images'){ - options { - warnError('Filebeat x-pack Test Docker images failed') - } - when { - expression { return params.RELEASE_TEST_IMAGES } - } - steps { - dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}") - withMageEnv(){ - dir("${BASE_DIR}/x-pack/filebeat"){ - retryWithSleep(retries: 3, seconds: 5, backoff: true){ - sh(label: 'Build', script: "mage compose:buildSupportedVersions"); - sh(label: 'Push', script: "mage compose:pushSupportedVersions"); - } - } - } - } - } - } - post { - cleanup { - notifyBuildResult() - } - } -} diff --git a/.ci/ironbank-validation.groovy b/.ci/ironbank-validation.groovy deleted file mode 100644 index 88ecc751d8c1..000000000000 --- a/.ci/ironbank-validation.groovy +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -pipeline { - agent { label 'ubuntu-22 && immutable' } - environment { - REPO = 'beats' - BASE_DIR = "src/github.com/elastic/${env.REPO}" - PIPELINE_LOG_LEVEL = "INFO" - BEATS_FOLDER = "x-pack/heartbeat" - SLACK_CHANNEL = '#ingest-notifications' - NOTIFY_TO = 'observability-robots-internal+ironbank-beats-validation@elastic.co' - } - options { - timeout(time: 1, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - disableConcurrentBuilds() - } - stages { - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - deleteDir() - gitCheckout(basedir: "${BASE_DIR}") - setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) - dir("${BASE_DIR}"){ - setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) - } - } - } - stage('Package'){ - options { skipDefaultCheckout() } - steps { - withMageEnv(){ - dir("${env.BASE_DIR}/${env.BEATS_FOLDER}") { - sh(label: 'make ironbank-package', script: "make -C ironbank package") - } - } - } - post { - failure { - notifyStatus(slackStatus: 'danger', subject: "[${env.REPO}@${BRANCH_NAME}] package for ${env.BEATS_FOLDER}", body: "Contact the heartbeats team. (<${env.RUN_DISPLAY_URL}|Open>)") - } - } - } - stage('Ironbank'){ - options { skipDefaultCheckout() } - steps { - withMageEnv(){ - dir("${env.BASE_DIR}/${env.BEATS_FOLDER}") { - sh(label: 'mage ironbank', script: 'mage ironbank') - } - } - } - post { - failure { - notifyStatus(slackStatus: 'danger', subject: "[${env.REPO}@${BRANCH_NAME}] Ironbank docker context for ${env.BEATS_FOLDER}", body: "Contact the @observablt-robots-team team. (<${env.RUN_DISPLAY_URL}|Open>)") - } - } - } - stage('Pre-flight'){ - options { skipDefaultCheckout() } - environment { - DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' - DOCKER_REGISTRY = 'docker.elastic.co' - } - steps { - // verify if the docker registry is available by using the dockerLogin step - // if so then run the validation which interacts with third party systems - retryWithSleep(retries: 3, seconds: 10, backoff: true) { - dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - } - } - } - stage('Validate'){ - options { skipDefaultCheckout() } - steps { - withMageEnv(){ - dir("${env.BASE_DIR}/${env.BEATS_FOLDER}") { - sh(label: 'make validate-ironbank', script: "make -C ironbank validate-ironbank") - } - } - } - post { - failure { - notifyStatus(slackStatus: 'danger', subject: "[${env.REPO}@${BRANCH_NAME}] Ironbank validation failed", body: "Contact the @observablt-robots-team team. (<${env.RUN_DISPLAY_URL}|Open>)") - } - } - } - } - post { - cleanup { - notifyBuildResult(prComment: true) - } - } -} - -def notifyStatus(def args = [:]) { - releaseNotification(slackChannel: "${env.SLACK_CHANNEL}", - slackColor: args.slackStatus, - slackCredentialsId: 'jenkins-slack-integration-token', - to: "${env.NOTIFY_TO}", - subject: args.subject, - body: args.body) -} diff --git a/.ci/jobs/beats-release-changelog.yml b/.ci/jobs/beats-release-changelog.yml deleted file mode 100644 index 6a331022e8f7..000000000000 --- a/.ci/jobs/beats-release-changelog.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- job: - name: Beats/Release/beats-release-changelog - display-name: 'Prepare the Changelog for a Release' - description: 'Automate the steps to prepare the Changelog for a Release' - view: Beats - project-type: pipeline - pipeline-scm: - script-path: release_scripts/pipeline-release-changelog.groovy - scm: - - git: - url: git@github.com:elastic/ingest-dev.git - refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* - wipe-workspace: 'True' - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git - branches: - - main diff --git a/.ci/jobs/beats-release-minor-major-project.yml b/.ci/jobs/beats-release-minor-major-project.yml deleted file mode 100644 index 53e4acb3f8c0..000000000000 --- a/.ci/jobs/beats-release-minor-major-project.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- job: - name: Beats/Release/beats-release-minor-major-project - display-name: 'Prepare Major/minor Release for the given project' - description: 'Automate the steps to prepare a new Release branch for the given project' - view: Beats - project-type: pipeline - pipeline-scm: - script-path: release_scripts/pipeline-release-minor-major-project.groovy - scm: - - git: - url: git@github.com:elastic/ingest-dev.git - refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* - wipe-workspace: 'True' - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git - branches: - - main diff --git a/.ci/jobs/beats-release-minor-major.yml b/.ci/jobs/beats-release-minor-major.yml deleted file mode 100644 index 04a2e07046d2..000000000000 --- a/.ci/jobs/beats-release-minor-major.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- job: - name: Beats/Release/beats-release-minor-major - display-name: 'Prepare Major/minor Release' - description: 'Automate the steps to prepare a new Release branch' - view: Beats - project-type: pipeline - pipeline-scm: - script-path: release_scripts/pipeline-release-minor-major.groovy - scm: - - git: - url: git@github.com:elastic/ingest-dev.git - refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* - wipe-workspace: 'True' - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git - branches: - - main diff --git a/.ci/jobs/beats-release-patch-project.yml b/.ci/jobs/beats-release-patch-project.yml deleted file mode 100644 index 7ee26de66b40..000000000000 --- a/.ci/jobs/beats-release-patch-project.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- job: - name: Beats/Release/beats-release-patch-project - display-name: 'Prepare Patch Release for the given project' - description: 'Automate the steps to prepare a new Patch for the given project' - view: Beats - project-type: pipeline - pipeline-scm: - script-path: release_scripts/pipeline-release-patch-project.groovy - scm: - - git: - url: git@github.com:elastic/ingest-dev.git - refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* - wipe-workspace: 'True' - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git - branches: - - main diff --git a/.ci/jobs/beats-release-patch.yml b/.ci/jobs/beats-release-patch.yml deleted file mode 100644 index 80c401e6c204..000000000000 --- a/.ci/jobs/beats-release-patch.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- job: - name: Beats/Release/beats-release-patch - display-name: 'Prepare Patch Release' - description: 'Automate the steps to prepare a new Patch' - view: Beats - project-type: pipeline - pipeline-scm: - script-path: release_scripts/pipeline-release-patch.groovy - scm: - - git: - url: git@github.com:elastic/ingest-dev.git - refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* - wipe-workspace: 'True' - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git - branches: - - main diff --git a/.ci/jobs/beats-schedule-daily.yml b/.ci/jobs/beats-schedule-daily.yml deleted file mode 100644 index 8bf6467f3c01..000000000000 --- a/.ci/jobs/beats-schedule-daily.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- job: - name: beats-schedule-daily - display-name: Jobs scheduled daily (weekdays) - description: Jobs scheduled daily (weekdays) - view: Beats - project-type: pipeline - parameters: - - string: - name: branch_specifier - default: main - description: the Git branch specifier to build - pipeline-scm: - script-path: .ci/schedule-daily.groovy - scm: - - git: - url: git@github.com:elastic/beats.git - refspec: +refs/heads/*:refs/remotes/origin/* - wipe-workspace: 'True' - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/beats.git - branches: - - $branch_specifier - triggers: - - timed: 'H H(2-3) * * 1-5' diff --git a/.ci/jobs/beats-schedule-weekly.yml b/.ci/jobs/beats-schedule-weekly.yml deleted file mode 100644 index 9fa7619f66d2..000000000000 --- a/.ci/jobs/beats-schedule-weekly.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- job: - name: beats-schedule-weekly - display-name: Jobs scheduled weekly (Sunday) - description: Jobs scheduled weekly (Sunday) - view: Beats - project-type: pipeline - parameters: - - string: - name: branch_specifier - default: main - description: the Git branch specifier to build - pipeline-scm: - script-path: .ci/schedule-weekly.groovy - scm: - - git: - url: git@github.com:elastic/beats.git - refspec: +refs/heads/*:refs/remotes/origin/* - wipe-workspace: 'True' - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/beats.git - branches: - - $branch_specifier - triggers: - - timed: 'H H(1-2) * * 0' diff --git a/.ci/jobs/beats-test-infra.yml b/.ci/jobs/beats-test-infra.yml deleted file mode 100644 index daf2b54ed331..000000000000 --- a/.ci/jobs/beats-test-infra.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- job: - name: Beats/beats-test-infra - display-name: 'Beats test infra' - description: 'Pipeline to run the test infra' - view: Beats - project-type: pipeline - pipeline-scm: - script-path: .ci/validateWorkersBeatsCi.groovy - scm: - - git: - url: git@github.com:elastic/apm-pipeline-library.git - refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* - wipe-workspace: 'True' - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/apm-test-pipeline.git - branches: - - main - triggers: - - timed: 'H H(3-4) * * 1-5' diff --git a/.ci/jobs/beats-tester-bc.yml b/.ci/jobs/beats-tester-bc.yml deleted file mode 100644 index ecd8fef1f5de..000000000000 --- a/.ci/jobs/beats-tester-bc.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- job: - name: Beats/beats-tester-bc - display-name: Beats Tester orchestrator for Build Candidates - description: Launch the beat-tester suit on Build Candidates easily - view: Beats - disabled: false - project-type: pipeline - parameters: - - string: - name: BRANCH_REFERENCE - default: main - description: the Git branch specifier - pipeline-scm: - script-path: .ci/beats-tester-bc.groovy - scm: - - git: - url: git@github.com:elastic/beats.git - refspec: +refs/heads/*:refs/remotes/origin/* - wipe-workspace: true - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/beats.git - branches: - - $BRANCH_REFERENCE diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml deleted file mode 100644 index 5f9560ed2f0e..000000000000 --- a/.ci/jobs/beats-tester.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -- job: - name: Beats/beats-tester - display-name: Beats Tester orchestrator - description: Orchestrate the beats-tester when packaging finished successfully - view: Beats - disabled: false - project-type: multibranch - script-path: .ci/beats-tester.groovy - scm: - - github: - branch-discovery: 'no-pr' - discover-pr-forks-strategy: 'merge-current' - discover-pr-forks-trust: 'permission' - discover-pr-origin: 'merge-current' - discover-tags: true - head-filter-regex: '(main|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' - disable-pr-notifications: true - notification-context: 'beats-tester' - repo: 'beats' - repo-owner: 'elastic' - credentials-id: github-app-beats-ci - ssh-checkout: - credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba - build-strategies: - - skip-initial-build: true - - tags: - ignore-tags-older-than: -1 - ignore-tags-newer-than: 30 - - named-branches: - - exact-name: - name: 'main' - case-sensitive: true - - regex-name: - regex: '7\.1[6789]' - case-sensitive: true - - regex-name: - regex: '8\.\d+' - case-sensitive: true - - change-request: - ignore-target-only-changes: true - clean: - after: true - before: true - prune: true - shallow-clone: true - depth: 10 - do-not-fetch-tags: true - submodule: - disable: false - recursive: true - parent-credentials: true - timeout: 100 - timeout: '15' - use-author: true - wipe-workspace: true diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml deleted file mode 100644 index fb63f85f5982..000000000000 --- a/.ci/jobs/beats.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- -- job: - name: Beats/beats - display-name: 'Beats' - description: 'Beats Main Pipeline' - view: Beats - concurrent: true - project-type: multibranch - prune-dead-branches: true - days-to-keep: 30 - script-path: 'Jenkinsfile' - triggers: [] - wrappers: [] - scm: - - github: - branch-discovery: 'no-pr' - discover-pr-forks-strategy: 'merge-current' - discover-pr-forks-trust: 'permission' - discover-pr-origin: 'merge-current' - head-filter-regex: '(main|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' - discover-tags: true - notification-context: "beats-ci" - repo: 'beats' - repo-owner: 'elastic' - credentials-id: github-app-beats-ci - ssh-checkout: - credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba - build-strategies: - - tags: - ignore-tags-older-than: -1 - ignore-tags-newer-than: 365 - - change-request: - ignore-target-only-changes: true - - named-branches: - - exact-name: - name: 'main' - case-sensitive: true - - regex-name: - regex: '6\.[89]' - case-sensitive: true - - regex-name: - regex: '7\.1[6789]' - case-sensitive: true - - regex-name: - regex: '8\.\d+' - case-sensitive: true - clean: - after: true - before: true - prune: true - shallow-clone: true - depth: 10 - do-not-fetch-tags: true - submodule: - disable: false - recursive: true - parent-credentials: true - timeout: 100 - reference-repo: /var/lib/jenkins/.git-references/beats.git - timeout: '15' - use-author: true - wipe-workspace: true diff --git a/.ci/jobs/build-it-docker-images.yml b/.ci/jobs/build-it-docker-images.yml deleted file mode 100644 index d5dfc956426d..000000000000 --- a/.ci/jobs/build-it-docker-images.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- job: - name: Beats/build-it-docker-images - display-name: ITs Docker images - description: Job to pre-build docker images used in integration tests. - view: Beats - project-type: pipeline - parameters: - - string: - name: BRANCH_REFERENCE - default: main - description: the Git branch specifier - pipeline-scm: - script-path: .ci/build-docker-images.groovy - scm: - - git: - url: git@github.com:elastic/beats.git - refspec: +refs/heads/*:refs/remotes/origin/* - wipe-workspace: true - name: origin - shallow-clone: true - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/beats.git - branches: - - $BRANCH_REFERENCE - triggers: - - timed: 'H H(0-5) * * 1-5' diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml deleted file mode 100644 index 40d416f28fcf..000000000000 --- a/.ci/jobs/defaults.yml +++ /dev/null @@ -1,19 +0,0 @@ - ---- - -##### GLOBAL METADATA - -- meta: - cluster: beats-ci - -##### JOB DEFAULTS - -- job: - logrotate: - numToKeep: 20 - node: linux - concurrent: true - publishers: - - email: - recipients: infra-root+build@elastic.co - prune-dead-branches: true diff --git a/.ci/jobs/folders.yml b/.ci/jobs/folders.yml deleted file mode 100644 index 4b6cc5c49440..000000000000 --- a/.ci/jobs/folders.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -#https://docs.openstack.org/infra/jenkins-job-builder/project_folder.html -- job: - name: Beats - description: Beats - project-type: folder - -- job: - name: Beats/Release - description: Jobs for release preparation - project-type: folder diff --git a/.ci/jobs/ironbank-validation.yml b/.ci/jobs/ironbank-validation.yml deleted file mode 100644 index bc8392591940..000000000000 --- a/.ci/jobs/ironbank-validation.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -- job: - name: Beats/ironbank-validation-mbp - display-name: Ironbank validation - description: Validates the Ironbank docker context - view: Beats - disabled: false - project-type: multibranch - script-path: .ci/ironbank-validation.groovy - scm: - - github: - branch-discovery: 'no-pr' - discover-pr-forks-strategy: 'merge-current' - discover-pr-forks-trust: 'permission' - discover-pr-origin: 'merge-current' - discover-tags: false - head-filter-regex: '(main|7\.17|8\.\d+|PR-.*)' - disable-pr-notifications: true - notification-context: 'ironbank' - repo: 'beats' - repo-owner: 'elastic' - credentials-id: github-app-beats-ci - ssh-checkout: - credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba - property-strategies: - all-branches: - - suppress-scm-triggering: true - build-strategies: - - skip-initial-build: true - - regular-branches: true - - change-request: - ignore-target-only-changes: true - clean: - after: true - before: true - prune: true - shallow-clone: true - depth: 10 - do-not-fetch-tags: true - submodule: - disable: false - recursive: true - parent-credentials: true - timeout: 100 - timeout: '15' - use-author: true - wipe-workspace: true diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml deleted file mode 100644 index 8dd8fb6bd85f..000000000000 --- a/.ci/jobs/packaging.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -- job: - name: Beats/packaging - display-name: Beats Packaging - description: Make the packages for beats and publish them on a GCS bucket. - view: Beats - disabled: false - project-type: multibranch - script-path: .ci/packaging.groovy - scm: - - github: - branch-discovery: 'no-pr' - discover-pr-forks-strategy: 'merge-current' - discover-pr-forks-trust: 'permission' - discover-pr-origin: 'merge-current' - discover-tags: false - head-filter-regex: '(PR-.*)' - disable-pr-notifications: true - notification-context: 'beats-packaging' - repo: 'beats' - repo-owner: 'elastic' - credentials-id: github-app-beats-ci - ssh-checkout: - credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba - build-strategies: - - skip-initial-build: true - - tags: - ignore-tags-older-than: -1 - ignore-tags-newer-than: 30 - - change-request: - ignore-target-only-changes: true - clean: - after: true - before: true - prune: true - shallow-clone: true - depth: 10 - do-not-fetch-tags: true - submodule: - disable: false - recursive: true - parent-credentials: true - timeout: 100 - timeout: '15' - use-author: true - wipe-workspace: true diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy deleted file mode 100644 index ea8f8f7daefe..000000000000 --- a/.ci/packaging.groovy +++ /dev/null @@ -1,460 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -import groovy.transform.Field - -pipeline { - agent none - environment { - REPO = 'beats' - BASE_DIR = "src/github.com/elastic/${env.REPO}" - JOB_GCS_BUCKET = 'beats-ci-artifacts' - JOB_GCS_BUCKET_STASH = 'beats-ci-temp' - JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials' - DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' - DOCKER_REGISTRY = 'docker.elastic.co' - PIPELINE_LOG_LEVEL = "INFO" - SLACK_CHANNEL = '#ingest-notifications' - NOTIFY_TO = 'beats-contrib+package-beats@elastic.co' - DRA_OUTPUT = 'release-manager.out' - } - options { - timeout(time: 4, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - disableConcurrentBuilds() - } - triggers { - issueCommentTrigger('(?i)^\\/packag[ing|e]$') - // disable upstream trigger on a PR basis - upstream("Beats/beats/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }") - } - stages { - stage('Filter build') { - options { skipDefaultCheckout() } - agent { label 'ubuntu-22 && immutable' } - when { - beforeAgent true - anyOf { - triggeredBy cause: "IssueCommentCause" - expression { - def ret = isUserTrigger() || isUpstreamTrigger() - if(!ret){ - currentBuild.result = 'NOT_BUILT' - currentBuild.description = "The build has been skipped" - currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)" - echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job") - } - return ret - } - } - } - environment { - HOME = "${env.WORKSPACE}" - } - stages { - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - deleteDir() - script { - if(isUpstreamTrigger()) { - try { - copyArtifacts(filter: 'packaging.properties', - flatten: true, - projectName: "Beats/beats/${env.JOB_BASE_NAME}", - selector: upstream(fallbackToLastSuccessful: true)) - def props = readProperties(file: 'packaging.properties') - gitCheckout(basedir: "${BASE_DIR}", branch: props.COMMIT) - } catch(err) { - // Fallback to the head of the branch as used to be. - gitCheckout(basedir: "${BASE_DIR}") - } - } else { - gitCheckout(basedir: "${BASE_DIR}") - } - } - setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) - // Stash without any build/dependencies context to support different architectures. - stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) - } - setEnvVar('IS_BRANCH_AVAILABLE', isBranchUnifiedReleaseAvailable(env.BRANCH_NAME)) - } - } - stage('Build Packages'){ - options { skipDefaultCheckout() } - steps { - generateSteps() - } - } - stage('DRA Snapshot') { - options { skipDefaultCheckout() } - // The Unified Release process keeps moving branches as soon as a new - // minor version is created, therefore old release branches won't be able - // to use the release manager as their definition is removed. - when { - expression { return env.IS_BRANCH_AVAILABLE == "true" } - } - steps { - // retryWithSleep and withNode can be remove once https://github.com/elastic/release-eng/issues/456 - // (internal only) is fixed. - retryWithSleep(retries: 3, seconds: 5, backoff: true) { - withNode(labels: 'ubuntu-22 && immutable', forceWorkspace: true) { - runReleaseManager(type: 'snapshot', outputFile: env.DRA_OUTPUT) - } - } - } - post { - failure { - notifyStatus(analyse: true, - file: "${BASE_DIR}/${env.DRA_OUTPUT}", - subject: "[${env.REPO}@${env.BRANCH_NAME}] The Daily releasable artifact failed.", - body: 'Contact the Release Platform team [#platform-release]') - } - } - } - stage('DRA Release Staging') { - options { skipDefaultCheckout() } - when { - allOf { - // The Unified Release process keeps moving branches as soon as a new - // minor version is created, therefore old release branches won't be able - // to use the release manager as their definition is removed. - expression { return env.IS_BRANCH_AVAILABLE == "true" } - not { branch 'main' } - } - } - steps { - // retryWithSleep and withNode can be remove once https://github.com/elastic/release-eng/issues/456 - // (internal only) is fixed. - retryWithSleep(retries: 3, seconds: 5, backoff: true) { - withNode(labels: 'ubuntu-22 && immutable', forceWorkspace: true) { - runReleaseManager(type: 'staging', outputFile: env.DRA_OUTPUT) - } - } - } - post { - failure { - notifyStatus(analyse: true, - file: "${BASE_DIR}/${env.DRA_OUTPUT}", - subject: "[${env.REPO}@${env.BRANCH_NAME}] The Daily releasable artifact failed.", - body: 'Contact the Release Platform team [#platform-release]') - } - } - } - } - post { - success { - writeFile(file: 'beats-tester.properties', - text: """\ - ## To be consumed by the beats-tester pipeline - COMMIT=${env.GIT_BASE_COMMIT} - BEATS_URL_BASE=https://storage.googleapis.com/${env.JOB_GCS_BUCKET}/${env.REPO}/commits/${env.GIT_BASE_COMMIT} - VERSION=${env.BEAT_VERSION}-SNAPSHOT""".stripIndent()) // stripIdent() requires '''/ - archiveArtifacts artifacts: 'beats-tester.properties' - } - } - } - } -} - -def getBucketUri(type) { - // It uses the folder structure done in uploadPackagesToGoogleBucket - // commit for the normal workflow, snapshots (aka SNAPSHOT=true) - // staging for the staging workflow, SNAPSHOT=false - def folder = type.equals('staging') ? 'staging' : 'commits' - return "gs://${env.JOB_GCS_BUCKET}/${env.REPO}/${folder}/${env.GIT_BASE_COMMIT}" -} - -def runReleaseManager(def args = [:]) { - def type = args.get('type', 'snapshot') - def bucketUri = getBucketUri(type) - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}") { - unstash "dependencies-${type}" - // TODO: as long as googleStorageDownload does not support recursive copy with **/* - dir("build/distributions") { - gsutil(command: "-m -q cp -r ${bucketUri} .", credentialsId: env.JOB_GCS_EXT_CREDENTIALS) - sh(label: 'move one level up', script: "mv ${env.GIT_BASE_COMMIT}/** .") - } - sh(label: "prepare-release-manager-artifacts ${type}", script: ".ci/scripts/prepare-release-manager.sh ${type}") - dockerLogin(secret: env.DOCKERELASTIC_SECRET, registry: env.DOCKER_REGISTRY) - releaseManager(project: 'beats', - version: env.BEAT_VERSION, - type: type, - artifactsFolder: 'build/distributions', - outputFile: args.outputFile) - } -} - -def generateSteps() { - def parallelTasks = [:] - def beats = [ - 'auditbeat', - 'filebeat', - 'heartbeat', - 'metricbeat', - 'packetbeat', - 'winlogbeat', - 'x-pack/agentbeat', - 'x-pack/auditbeat', - 'x-pack/dockerlogbeat', - 'x-pack/filebeat', - 'x-pack/functionbeat', - 'x-pack/heartbeat', - 'x-pack/metricbeat', - 'x-pack/osquerybeat', - 'x-pack/packetbeat', - 'x-pack/winlogbeat' - ] - - def armBeats = [ - 'auditbeat', - 'filebeat', - 'heartbeat', - 'metricbeat', - 'packetbeat', - 'x-pack/auditbeat', - 'x-pack/dockerlogbeat', - 'x-pack/filebeat', - 'x-pack/heartbeat', - 'x-pack/metricbeat', - 'x-pack/packetbeat' - ] - beats.each { beat -> - parallelTasks["linux-${beat}"] = generateLinuxStep(beat) - if (armBeats.contains(beat)) { - parallelTasks["arm-${beat}"] = generateArmStep(beat) - } - } - - // enable beats-dashboards within the existing worker - parallelTasks["beats-dashboards"] = { - withGithubNotify(context: "beats-dashboards") { - withEnv(["HOME=${env.WORKSPACE}"]) { - ['snapshot', 'staging'].each { type -> - deleteDir() - withBeatsEnv(type) { - sh(label: 'make dependencies.csv', script: 'make build/distributions/dependencies.csv') - sh(label: 'make beats-dashboards', script: 'make beats-dashboards') - stash(includes: 'build/distributions/**', name: "dependencies-${type}", useDefaultExcludes: false) - } - } - } - } - } - parallel(parallelTasks) -} - -def generateArmStep(beat) { - return { - withNode(labels: 'ubuntu-2204-aarch64') { - withEnv(["HOME=${env.WORKSPACE}", 'PLATFORMS=linux/arm64','PACKAGES=docker', "BEATS_FOLDER=${beat}"]) { - withGithubNotify(context: "Packaging Arm ${beat}") { - deleteDir() - release('snapshot') - dir("${BASE_DIR}"){ - pushCIDockerImages(arch: 'arm64') - } - } - // Staging is only needed from branches (main or release branches) - if (isBranch()) { - deleteDir() - release('staging') - } - } - } - } -} - -def generateLinuxStep(beat) { - return { - withNode(labels: 'ubuntu-22.04 && immutable') { - withEnv(["HOME=${env.WORKSPACE}", "PLATFORMS=${linuxPlatforms()}", "BEATS_FOLDER=${beat}"]) { - withGithubNotify(context: "Packaging Linux ${beat}") { - deleteDir() - release('snapshot') - dir("${BASE_DIR}"){ - pushCIDockerImages(arch: 'amd64') - } - } - - // Staging is only needed from branches (main or release branches) - if (isBranch()) { - // As long as we reuse the same worker to package more than - // once, the workspace gets corrupted with some permissions - // therefore let's reset the workspace to a new location - // in order to reuse the worker and successfully run the package - def work = "workspace/${env.JOB_BASE_NAME}-${env.BUILD_NUMBER}-staging" - ws(work) { - withEnv(["HOME=${env.WORKSPACE}"]) { - deleteDir() - release('staging') - } - } - } - } - } - } -} - -def linuxPlatforms() { - return [ - '+all', - 'linux/amd64', - 'linux/arm64', - // armv7 packaging isn't working, and we don't currently - // need it for release. Do not re-enable it without - // confirming it is fixed, you will break the packaging - // pipeline! - //'linux/armv7', - // The platforms above are disabled temporarly as crossbuild images are - // not available. See: https://github.com/elastic/golang-crossbuild/issues/71 - //'linux/ppc64le', - //'linux/mips64', - //'linux/s390x', - 'windows/amd64', - 'darwin/amd64', - 'darwin/arm64' - ].join(' ') -} - -/** -* @param arch what architecture -*/ -def pushCIDockerImages(Map args = [:]) { - def arch = args.get('arch', 'amd64') - catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') { - def defaultVariants = [ '' : 'beats', '-oss' : 'beats', '-ubi' : 'beats' ] - if (env?.BEATS_FOLDER?.endsWith('auditbeat')) { - tagAndPush(beatName: 'auditbeat', arch: arch, variants: defaultVariants) - } else if (env?.BEATS_FOLDER?.endsWith('filebeat')) { - tagAndPush(beatName: 'filebeat', arch: arch, variants: defaultVariants) - } else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) { - tagAndPush(beatName: 'heartbeat', arch: arch, variants: defaultVariants) - } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) { - tagAndPush(beatName: 'metricbeat', arch: arch, variants: defaultVariants) - } else if (env?.BEATS_FOLDER?.endsWith('osquerybeat')) { - tagAndPush(beatName: 'osquerybeat', arch: arch, variants: defaultVariants) - } else if ("${env.BEATS_FOLDER}" == "packetbeat"){ - tagAndPush(beatName: 'packetbeat', arch: arch) - } - } -} - -/** -* @param beatName name of the Beat -* @param arch what architecture -* @param variants list of docker variants -*/ -def tagAndPush(Map args = [:]) { - def images = [ ] - args.variants.each { variant, sourceNamespace -> - images += [ source: "${sourceNamespace}/${args.beatName}${variant}", - target: "observability-ci/${args.beatName}", - arch: args.arch ] - } - pushDockerImages( - registry: env.DOCKER_REGISTRY, - secret: env.DOCKERELASTIC_SECRET, - snapshot: true, - version: env.BEAT_VERSION, - images: images - ) -} - -def release(type){ - withBeatsEnv(type){ - // As agreed DEV=false for staging otherwise DEV=true - // this should avoid releasing binaries with the debug symbols and disabled most build optimizations. - withEnv([ - "DEV=${!type.equals('staging')}" - ]) { - dir("${BASE_DIR}"){ - if (env.BEATS_FOLDER.equals('x-pack/agentbeat') || env.BEATS_FOLDER.equals('x-pack/osquerybeat')) { - // sh(label: 'install msitools', script: '.buildkite/scripts/install-msitools.sh') - sh '''#!/usr/bin/env bash - set -euo pipefail - sudo apt-get update -y - DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends --yes msitools - ''' - } - } - dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - dir("${env.BEATS_FOLDER}") { - sh(label: "mage package ${type} ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package') - sh(label: "mage ironbank ${type} ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage ironbank') - def folder = getBeatsName(env.BEATS_FOLDER) - uploadPackagesToGoogleBucket( - credentialsId: env.JOB_GCS_EXT_CREDENTIALS, - repo: env.REPO, - bucket: env.JOB_GCS_BUCKET, - folder: folder, - pattern: "build/distributions/*" - ) - if (type.equals('staging')) { - dir("build/distributions") { - def bucketUri = getBucketUri(type) - log(level: 'INFO', text: "${env.BEATS_FOLDER} for ${type} requires to upload the artifacts to ${bucketUri}.") - googleStorageUploadExt(bucket: "${bucketUri}/${folder}", - credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", - pattern: "*", - sharedPublicly: true) - } - } else { - log(level: 'INFO', text: "${env.BEATS_FOLDER} for ${type} does not require to upload the staging artifacts.") - } - } - } - } -} - -/** -* There is a specific folder structure in https://staging.elastic.co/ and https://artifacts.elastic.co/downloads/ -* therefore the storage bucket in GCP should follow the same folder structure. -* This is required by https://github.com/elastic/beats-tester -* e.g. -* baseDir=name -> return name -* baseDir=name1/name2/name3-> return name2 -*/ -def getBeatsName(baseDir) { - return baseDir.replace('x-pack/', '') -} - -def withBeatsEnv(type, Closure body) { - def envVars = [ "PYTHON_ENV=${WORKSPACE}/python-env" ] - if (type.equals('snapshot')) { - envVars << "SNAPSHOT=true" - } - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") - withMageEnv(){ - withEnv(envVars) { - dir("${env.BASE_DIR}"){ - body() - } - } - } -} - -def notifyStatus(def args = [:]) { - def releaseManagerFile = args.get('file', '') - def analyse = args.get('analyse', false) - def subject = args.get('subject', '') - def body = args.get('body', '') - releaseManagerNotification(file: releaseManagerFile, - analyse: analyse, - slackChannel: "${env.SLACK_CHANNEL}", - slackColor: 'danger', - slackCredentialsId: 'jenkins-slack-integration-token', - to: "${env.NOTIFY_TO}", - subject: subject, - body: "Build: (<${env.RUN_DISPLAY_URL}|here>).\n ${body}") -} \ No newline at end of file diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh deleted file mode 100755 index 84f08fbfab08..000000000000 --- a/.ci/packer_cache.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash -# -# this file is run daily to generate worker packer images -# - -# shellcheck disable=SC1091 -source /usr/local/bin/bash_standard_lib.sh - -# shellcheck disable=SC1091 -source ./dev-tools/common.bash - -###################### -############ FUNCTIONS -###################### -function getBeatsVersion() { - grep 'defaultBeatVersion' libbeat/version/version.go | cut -d= -f2 | sed 's#"##g' | tr -d " " -} - -function dockerPullCommonImages() { - DOCKER_IMAGES="docker.elastic.co/observability-ci/database-instantclient:12.2.0.1 - docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 - docker.elastic.co/beats-dev/fpm:1.11.0 - golang:1.14.12-stretch - ubuntu:20.04 - " - for image in ${DOCKER_IMAGES} ; do - (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing." - done - docker tag \ - docker.elastic.co/observability-ci/database-instantclient:12.2.0.1 \ - store/oracle/database-instantclient:12.2.0.1 \ - || echo "Error setting the Oracle Instant Client tag" - docker tag \ - docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 \ - store/oracle/database-enterprise:12.2.0.1 \ - || echo "Error setting the Oracle Database tag" -} - -function dockerPullImages() { - SNAPSHOT=$1 - get_go_version - - DOCKER_IMAGES=" - docker.elastic.co/elasticsearch/elasticsearch:${SNAPSHOT} - docker.elastic.co/kibana/kibana:${SNAPSHOT} - docker.elastic.co/logstash/logstash:${SNAPSHOT} - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armhf - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armel - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9 - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian7 - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian8 - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main-debian9 - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-mips - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-ppc - docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-s390x - golang:${GO_VERSION} - docker.elastic.co/infra/release-manager:latest - " - for image in ${DOCKER_IMAGES} - do - (retry 2 docker pull "${image}") || echo "Error pulling ${image} Docker image. Continuing." - done -} - -################# -############ MAIN -################# -if [ -x "$(command -v docker)" ]; then - set -x - echo "Docker pull common docker images" - dockerPullCommonImages - - ## GitHub api returns up to 100 entries. - ## Probably we need a different approach to search the latest minor. - latest7Minor=$(curl -s https://api.github.com/repos/elastic/beats/branches\?per_page=100 | jq -r '.[].name' | grep "^7." | tail -1) - latest8Minor=$(curl -s https://api.github.com/repos/elastic/beats/branches\?per_page=100 | jq -r '.[].name' | grep "^8." | tail -1) - - for branch in main $latest7Minor $latest8Minor; do - if [ "$branch" != "main" ] ; then - echo "Checkout the branch $branch" - git checkout "$branch" - fi - - VERSION=$(getBeatsVersion) - dockerPullImages "${VERSION}-SNAPSHOT" - done -fi diff --git a/.ci/schedule-daily.groovy b/.ci/schedule-daily.groovy deleted file mode 100644 index c4fc3638af99..000000000000 --- a/.ci/schedule-daily.groovy +++ /dev/null @@ -1,56 +0,0 @@ -@Library('apm@current') _ - -pipeline { - agent none - environment { - NOTIFY_TO = credentials('notify-to') - PIPELINE_LOG_LEVEL = 'INFO' - } - options { - timeout(time: 1, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - } - triggers { - cron('H H(2-3) * * 1-5') - } - stages { - stage('Nighly beats builds') { - steps { - runMacosBuilds(quietPeriodFactor: 2000, branches: ['main', '8.', '8.', '8.', '7.']) - runIronbankBuilds(quietPeriodFactor: 100, branches: ['main', '8.', '8.', '8.', '7.']) - } - } - } - post { - cleanup { - notifyBuildResult(prComment: false) - } - } -} - -def runMacosBuilds(Map args = [:]) { - def branches = getBranchesFromAliases(aliases: args.branches) - - def quietPeriod = 0 - branches.each { branch -> - // IMPORTANT: ephemeral Orka VMs are not provisioned so, we cannot run builds on daily basis at all. - // build(quietPeriod: quietPeriod, job: "Beats/beats/${branch}", parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false) - // Increate the quiet period for the next iteration - quietPeriod += args.quietPeriodFactor - } -} - -def runIronbankBuilds(Map args = [:]) { - def branches = getBranchesFromAliases(aliases: args.branches) - - def quietPeriod = 0 - branches.each { branch -> - build(quietPeriod: quietPeriod, job: "Beats/ironbank-validation-mbp/${branch}", wait: false, propagate: false) - // Increate the quiet period for the next iteration - quietPeriod += args.quietPeriodFactor - } -} diff --git a/.ci/schedule-weekly.groovy b/.ci/schedule-weekly.groovy deleted file mode 100644 index ba870ce40d72..000000000000 --- a/.ci/schedule-weekly.groovy +++ /dev/null @@ -1,51 +0,0 @@ -@Library('apm@current') _ - -pipeline { - agent none - environment { - NOTIFY_TO = credentials('notify-to') - PIPELINE_LOG_LEVEL = 'INFO' - } - options { - timeout(time: 1, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - } - triggers { - cron('H H(1-2) * * 0') - } - stages { - stage('Weekly beats builds for AWS') { - steps { - runBuilds(quietPeriodFactor: 1000, branches: ['main', '8.', '8.', '8.', '7.'], parameters: [booleanParam(name: 'awsCloudTests', value: true)]) - } - } - stage('Weekly beats builds for Orka M1') { - steps { - // There are some limitations with the number of concurrent macos m1 that can run in parallel - // let's only run for the `main` branch for the timebeing and wait to start a bit longer, - // so the previous stage for AWS validation can run further - runBuilds(quietPeriodFactor: 10000, branches: ['main'], parameters: [booleanParam(name: 'macosM1Test', value: true)]) - } - } - } - post { - cleanup { - notifyBuildResult(prComment: false) - } - } -} - -def runBuilds(Map args = [:]) { - def branches = getBranchesFromAliases(aliases: args.branches) - - def quietPeriod = 0 - branches.each { branch -> - build(quietPeriod: quietPeriod, job: "Beats/beats/${branch}", parameters: args.parameters, wait: false, propagate: false) - // Increate the quiet period for the next iteration - quietPeriod += args.quietPeriodFactor - } -} diff --git a/.ci/scripts/generate_build_table.py b/.ci/scripts/generate_build_table.py deleted file mode 100755 index 95dbc9afa5d5..000000000000 --- a/.ci/scripts/generate_build_table.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 - -import os -import yaml - -if __name__ == "__main__": - - print("| Beat | Stage | Category | Command | MODULE | Platforms | When |") - print("|-------|--------|----------|----------|---------|------------|------|") - for root, dirs, files in os.walk("."): - dirs.sort() - for file in files: - if file.endswith("Jenkinsfile.yml") and root != ".": - with open(os.path.join(root, file), 'r') as f: - doc = yaml.load(f, Loader=yaml.FullLoader) - module = root.replace(".{}".format(os.sep), '') - for stage in doc["stages"]: - withModule = False - platforms = [doc["platform"]] - when = "mandatory" - category = 'default' - if "stage" in doc["stages"][stage]: - category = doc["stages"][stage]["stage"] - if "make" in doc["stages"][stage]: - command = doc["stages"][stage]["make"].replace("\n", " ") - if "mage" in doc["stages"][stage]: - command = doc["stages"][stage]["mage"].replace("\n", " ") - if "k8sTest" in doc["stages"][stage]: - command = doc["stages"][stage]["k8sTest"] - if "cloud" in doc["stages"][stage]: - command = doc["stages"][stage]["cloud"] - if "platforms" in doc["stages"][stage]: - platforms = doc["stages"][stage]["platforms"] - if "withModule" in doc["stages"][stage]: - withModule = doc["stages"][stage]["withModule"] - if "when" in doc["stages"][stage]: - if "not_changeset_full_match" not in doc["stages"][stage]["when"]: - when = "optional" - print("| {} | {} | `{}` | `{}` | {} | `{}` | {} |".format( - module, stage, category, command, withModule, platforms, when)) diff --git a/.editorconfig b/.editorconfig index 16a560fec2b2..ff8d698c9a8a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -29,10 +29,6 @@ indent_style = tab [*.mk] indent_style = tab -[Jenkinsfile] -indent_size = 2 -indent_style = space - [Vagrantfile] indent_size = 2 indent_style = space diff --git a/.github/paths-labeller.yml b/.github/paths-labeller.yml deleted file mode 100644 index 3da97541426a..000000000000 --- a/.github/paths-labeller.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - - "Team:Automation": - - ".ci/**/*.*" - - "Jenkinsfile" diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 04df94f4b256..000000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,1197 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -pipeline { - agent { label 'ubuntu-22 && immutable' } - environment { - AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' - AWS_REGION = "${params.awsRegion}" - REPO = 'beats' - BASE_DIR = "src/github.com/elastic/${env.REPO}" - DOCKERHUB_SECRET = 'secret/observability-team/ci/elastic-observability-dockerhub' - DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' - DOCKER_COMPOSE_VERSION = "1.21.0" - DOCKER_REGISTRY = 'docker.elastic.co' - JOB_GCS_BUCKET = 'beats-ci-temp' - JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials' - OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' - PIPELINE_LOG_LEVEL = 'INFO' - PYTEST_ADDOPTS = "${params.PYTEST_ADDOPTS}" - RUNBLD_DISABLE_NOTIFICATIONS = 'true' - SLACK_CHANNEL = "#ingest-notifications" - SNAPSHOT = 'true' - TERRAFORM_VERSION = "1.0.2" - XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' - KIND_VERSION = 'v0.20.0' - K8S_VERSION = 'v1.29.0' - } - options { - timeout(time: 6, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '60', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - quietPeriod(10) - rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) - } - triggers { - issueCommentTrigger("${obltGitHubComments()}") - } - parameters { - booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') - booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') - string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') - booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') - booleanParam(name: 'armTest', defaultValue: false, description: 'Allow ARM stages.') - booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') - booleanParam(name: 'macosM1Test', defaultValue: false, description: 'Allow macOS M1 stages.') - string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`') - } - stages { - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - deleteDir() - // Here we do a checkout into a temporary directory in order to have the - // side-effect of setting up the git environment correctly. - gitCheckout(basedir: "${pwd(tmp: true)}", githubNotifyFirstTimeContributor: true) - pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) - dir("${BASE_DIR}") { - // We use a raw checkout to avoid the many extra objects which are brought in - // with a `git fetch` as would happen if we used the `gitCheckout` step. - checkout scm - } - stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - // Skip all the stages except docs for PR's with asciidoc, md or deploy k8s templates changes only - setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '(.*\\.(asciidoc|md)|deploy/kubernetes/.*-kubernetes\\.yaml)' ], shouldMatchAll: true).toString()) - setEnvVar('GO_MOD_CHANGES', isGitRegionMatch(patterns: [ '^go.mod' ], shouldMatchAll: false).toString()) - setEnvVar('PACKAGING_CHANGES', isGitRegionMatch(patterns: [ '(^dev-tools/packaging/.*|.go-version)' ], shouldMatchAll: false).toString()) - setEnvVar('GO_VERSION', readFile(".go-version").trim()) - withEnv(["HOME=${env.WORKSPACE}"]) { - retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } - } - } - dir("${BASE_DIR}"){ - setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim()) - } - } - } - stage('Checks'){ - options { skipDefaultCheckout() } - environment { - GOFLAGS = '-mod=readonly' - } - steps { - withGithubNotify(context: "Checks") { - stageStatusCache(id: 'Checks'){ - // test the ./dev-tools/run_with_go_ver used by the Unified Release process - dir("${BASE_DIR}") { - sh "HOME=${WORKSPACE} GO_VERSION=${GO_VERSION} ./dev-tools/run_with_go_ver make test-mage" - } - withBeatsEnv(archive: false, id: "checks") { - dumpVariables() - whenTrue(env.ONLY_DOCS == 'false') { - runChecks() - } - } - } - } - } - } - stage('Build&Test') { - options { skipDefaultCheckout() } - when { - // Always when running builds on branches/tags - // On a PR basis, skip if changes are only related to docs. - // Always when forcing the input parameter - anyOf { - not { changeRequest() } // If no PR - allOf { // If PR and no docs changes - expression { return env.ONLY_DOCS == "false" } - changeRequest() - } - expression { return params.runAllStages } // If UI forced - } - } - steps { - runBuildAndTest(filterStage: 'mandatory') - } - } - stage('Extended') { - options { skipDefaultCheckout() } - when { - // Always when running builds on branches/tags - // On a PR basis, skip if changes are only related to docs. - // Always when forcing the input parameter - anyOf { - not { changeRequest() } // If no PR - allOf { // If PR and no docs changes - expression { return env.ONLY_DOCS == "false" } - changeRequest() - } - expression { return params.runAllStages } // If UI forced - } - } - steps { - runBuildAndTest(filterStage: 'extended') - } - } - stage('ExtendedWin') { - options { skipDefaultCheckout() } - when { - // On a branches/tags, skip if changes are only related to docs. - // Always when forcing the input parameter - anyOf { - allOf { // If no PR and no docs changes - expression { return env.ONLY_DOCS == "false" } - not { changeRequest() } - } - expression { return params.runAllStages } // If UI forced - } - } - steps { - runBuildAndTest(filterStage: 'extended_win') - } - } - stage('Packaging') { - options { skipDefaultCheckout() } - when { - // On a PR basis, skip if changes are only related to docs. - // Always when forcing the input parameter - anyOf { - allOf { // If PR and no docs changes - expression { return env.ONLY_DOCS == "false" } - changeRequest() - } - expression { return params.runAllStages } // If UI forced - } - } - steps { - runBuildAndTest(filterStage: 'packaging') - } - } - stage('Packaging-Pipeline') { - agent none - options { skipDefaultCheckout() } - when { - allOf { - anyOf { - expression { return env.GO_MOD_CHANGES == "true" } - expression { return env.PACKAGING_CHANGES == "true" } - } - changeRequest() - } - } - steps { - withGithubNotify(context: 'Packaging') { - build(job: "Beats/packaging/${env.BRANCH_NAME}", propagate: true, wait: true) - } - } - } - } - post { - success { - writeFile(file: 'packaging.properties', text: """## To be consumed by the packaging pipeline -COMMIT=${env.GIT_BASE_COMMIT} -VERSION=${env.VERSION}-SNAPSHOT""") - archiveArtifacts artifacts: 'packaging.properties' - } - } -} - -// When to create a GiHub issue -def isGitHubIssueEnabled() { - return isBranch() && currentBuild.currentResult != "SUCCESS" && currentBuild.currentResult != "ABORTED" -} - -def runChecks() { - def mapParallelTasks = [:] - def content = readYaml(file: 'Jenkinsfile.yml') - content['projects'].each { projectName -> - generateStages(project: projectName, changeset: content['changeset'], filterStage: 'checks').each { k,v -> - mapParallelTasks["${k}"] = v - } - } - // Run pre-commit within the current node and in Jenkins - // hence there is no need to use docker login in the GitHub actions - // some docker images are hosted in an internal docker registry. - mapParallelTasks['pre-commit'] = runPreCommit() - parallel(mapParallelTasks) -} - -def runPreCommit() { - return { - withGithubNotify(context: 'Check pre-commit', tab: 'tests') { - preCommit(commit: "${GIT_BASE_COMMIT}", junit: true) - } - } -} - -def runBuildAndTest(Map args = [:]) { - def filterStage = args.get('filterStage', 'mandatory') - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - def mapParallelTasks = [:] - def content = readYaml(file: 'Jenkinsfile.yml') - if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) { - error 'Pull Request has been configured to be disabled when there is a skip-ci label match' - } else { - content['projects'].each { projectName -> - generateStages(project: projectName, changeset: content['changeset'], filterStage: filterStage).each { k,v -> - mapParallelTasks["${k}"] = v - } - } - notifyBuildReason() - parallel(mapParallelTasks) - } - } -} - - -/** -* Only supported the main branch -*/ -def getFlakyBranch() { - if(isPR()) { - return getBranchIndice(env.CHANGE_TARGET) - } else { - return getBranchIndice(env.BRANCH_NAME) - } -} - -/** -* Only supported the main branch -*/ -def getBranchIndice(String compare) { - return 'main' -} - -/** -* This method is the one used for running the parallel stages, therefore -* its arguments are passed by the beatsStages step. -*/ -def generateStages(Map args = [:]) { - def projectName = args.project - def filterStage = args.get('filterStage', 'all') - def changeset = args.changeset - def mapParallelStages = [:] - def fileName = "${projectName}/Jenkinsfile.yml" - if (fileExists(fileName)) { - def content = readYaml(file: fileName) - // changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation. - if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: new GetProjectDependencies(steps: this))) { - mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this), filterStage: filterStage) - } - } else { - log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") - } - return mapParallelStages -} - -def cloud(Map args = [:]) { - withGithubNotify(context: args.context) { - withNode(labels: args.label, forceWorkspace: true){ - withCloudTestEnv(args) { - startCloudTestEnv(name: args.directory, dirs: args.dirs, withAWS: args.withAWS) - try { - targetWithoutNode(dirs: args.dirs, context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) - } finally { - terraformCleanup(name: args.directory, dir: args.directory, withAWS: args.withAWS) - } - } - } - } -} - -def k8sTest(Map args = [:]) { - def versions = args.versions - versions.each{ v -> - withNode(labels: args.label, forceWorkspace: true){ - stage("${args.context} ${v}"){ - withEnv(["K8S_VERSION=${v}"]){ - withGithubNotify(context: "${args.context} ${v}") { - withBeatsEnv(archive: false, withModule: false) { - withTools(k8s: true) { - sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests") - sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") - } - } - } - } - } - } - } -} - -/** -* It relies on: -* - KIND_VERSION which it's defined in the top-level environment section. -* - K8S_VERSION which it's defined by default in the top-level environment section or set in the withEnv. -*/ -def withTools(Map args = [:], Closure body) { - if (args.get('k8s', false)) { - withK8s() { - body() - } - } else if (args.get('gcp', false)) { - withGCP() { - body() - } - } else if (args.get('nodejs', false)) { - withNodeJSEnv(version: '18.17.1') { - withEnv(["ELASTIC_SYNTHETICS_CAPABLE=true"]) { - cmd(label: "Install @elastic/synthetics", script: "npm i -g @elastic/synthetics") - body() - } - } - } else { - body() - } -} - -def withK8s(Closure body) { - withEnv(["KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ - retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") } - retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") } - try { - // Add some environmental resilience when setup does not work the very first time. - def i = 0 - retryWithSleep(retries: 3, seconds: 5, backoff: true){ - try { - sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") - } catch(err) { - i++ - sh(label: 'Delete cluster', script: 'kind delete cluster') - if (i > 2) { - error("Setup kind failed with error '${err.toString()}'") - } - } - } - body() - } finally { - sh(label: 'Delete cluster', script: 'kind delete cluster') - } - } -} - -def withGCP(Closure body) { - withGCPEnv(secret: 'secret/observability-team/ci/elastic-observability-account-auth'){ - body() - } -} - -/** -* This method runs the packaging for ARM -*/ -def packagingArm(Map args = [:]) { - def PLATFORMS = [ 'linux/arm64' ].join(' ') - withEnv([ - "PLATFORMS=${PLATFORMS}", - "PACKAGES=docker" - ]) { - target(args) - } -} - -/** -* This method runs the packaging for Linux -*/ -def packagingLinux(Map args = [:]) { - def PLATFORMS = [ '+all', - 'linux/amd64', - 'linux/arm64', - // armv7 packaging isn't working, and we don't currently - // need it for release. Do not re-enable it without - // confirming it is fixed, you will break the packaging - // pipeline! - //'linux/armv7', - // The platforms above are disabled temporarly as crossbuild images are - // not available. See: https://github.com/elastic/golang-crossbuild/issues/71 - //'linux/ppc64le', - //'linux/mips64', - //'linux/s390x', - 'windows/amd64', - 'darwin/amd64', - 'darwin/arm64' - ].join(' ') - withEnv([ - "PLATFORMS=${PLATFORMS}" - ]) { - target(args) - } -} - -/** -* Upload the packages to their snapshot or pull request buckets -* @param beatsFolder beats folder -*/ -def publishPackages(beatsFolder){ - dir(beatsFolder) { - uploadPackagesToGoogleBucket( - credentialsId: env.JOB_GCS_EXT_CREDENTIALS, - repo: env.REPO, - bucket: env.JOB_GCS_BUCKET, - folder: getBeatsName(beatsFolder), - pattern: "build/distributions/**/*" - ) - } -} - -/** -* Push the docker images for the given beat. -* @param beatsFolder beats folder -* @param arch what architecture -*/ -def pushCIDockerImages(Map args = [:]) { - def arch = args.get('arch', 'amd64') - def beatsFolder = args.beatsFolder - catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') { - def defaultVariants = [ '' : 'beats', '-oss' : 'beats', '-ubi' : 'beats' ] - if (beatsFolder.endsWith('auditbeat')) { - tagAndPush(beatName: 'auditbeat', arch: arch, variants: defaultVariants) - } else if (beatsFolder.endsWith('filebeat')) { - tagAndPush(beatName: 'filebeat', arch: arch, variants: defaultVariants) - } else if (beatsFolder.endsWith('heartbeat')) { - tagAndPush(beatName: 'heartbeat', arch: arch, variants: defaultVariants) - } else if (beatsFolder.endsWith('metricbeat')) { - tagAndPush(beatName: 'metricbeat', arch: arch, variants: defaultVariants) - } else if (beatsFolder.endsWith('osquerybeat')) { - tagAndPush(beatName: 'osquerybeat', arch: arch, variants: defaultVariants) - } else if ("${beatsFolder}" == "packetbeat"){ - tagAndPush(beatName: 'packetbeat', arch: arch, variants: defaultVariants) - } - } -} - -/** -* @param beatName name of the Beat -* @param arch what architecture -* @param variants list of docker variants -*/ -def tagAndPush(Map args = [:]) { - def images = [ ] - args.variants.each { variant, sourceNamespace -> - images += [ source: "${sourceNamespace}/${args.beatName}${variant}", - target: "observability-ci/${args.beatName}", - arch: args.arch ] - } - pushDockerImages( - registry: env.DOCKER_REGISTRY, - secret: env.DOCKER_ELASTIC_SECRET, - snapshot: env.SNAPSHOT, - version: env.VERSION, - images: images - ) -} - -/** -* There is a specific folder structure in https://staging.elastic.co/ and https://artifacts.elastic.co/downloads/ -* therefore the storage bucket in GCP should follow the same folder structure. -* This is required by https://github.com/elastic/beats-tester -* e.g. -* baseDir=name -> return name -* baseDir=name1/name2/name3-> return name2 -*/ -def getBeatsName(baseDir) { - return baseDir.replace('x-pack/', '') -} - -/** -* This method runs the end 2 end testing -*/ -def e2e(Map args = [:]) { - if (!args.e2e?.get('enabled', false)) { return } - // Skip running the tests on branches or tags if configured. - if (!isPR() && args.e2e?.get('when', false)) { - if (isBranch() && !args.e2e.when.get('branches', true)) { return } - if (isTag() && !args.e2e.when.get('tags', true)) { return } - } - if (args.e2e.get('entrypoint', '')?.trim()) { - e2e_with_entrypoint(args) - } else { - runE2E(testMatrixFile: '.ci/.e2e-tests-beats.yaml', - beatVersion: "${env.VERSION}-SNAPSHOT", - gitHubCheckName: "e2e-${args.context}", - gitHubCheckRepo: env.REPO, - gitHubCheckSha1: env.GIT_BASE_COMMIT) - } -} - -/** -* This method runs the end 2 end testing in the same worker where the packages have been -* generated, this should help to speed up the things -*/ -def e2e_with_entrypoint(Map args = [:]) { - def entrypoint = args.e2e?.get('entrypoint') - def dockerLogFile = "docker_logs_${entrypoint}.log" - dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") { - // TBC with the target branch if running on a PR basis. - git(branch: 'main', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git') - if(isDockerInstalled()) { - dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - } - def goVersionForE2E = readFile('.go-version').trim() - withEnv(["GO_VERSION=${goVersionForE2E}", - "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}", - "BEAT_VERSION=${env.VERSION}-SNAPSHOT", - "LOG_LEVEL=TRACE"]) { - def status = 0 - filebeat(output: dockerLogFile){ - try { - sh(script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}") - } finally { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml") - archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml" - } - } - } - } -} - -/** -* This method runs in a node -*/ -def target(Map args = [:]) { - withNode(labels: args.label, forceWorkspace: true){ - targetWithoutNode(args) - } -} - -/** -* This method runs the given command supporting two kind of scenarios: -* - make -C then the dir(location) is not required, aka by disaling isMage: false -* - mage then the dir(location) is required, aka by enabling isMage: true. -*/ -def targetWithoutNode(Map args = [:]) { - def dirs = args.get('dirs',[]) - def command = args.command - def context = args.context - def directory = args.get('directory', '') - def withModule = args.get('withModule', false) - def isMage = args.get('isMage', false) - def isE2E = args.e2e?.get('enabled', false) - def isPackaging = args.get('package', false) - def installK8s = args.get('installK8s', false) - def dockerArch = args.get('dockerArch', 'amd64') - def enableRetry = args.get('enableRetry', false) - def withGCP = args.get('withGCP', false) - def withNodejs = args.get('withNodejs', false) - String name = normalise(args.directory) - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { - dumpVariables() - // unstash terraform outputs in the same directory where the files were stashed - dirs?.each { folder -> - dir("${folder}") { - try { - unstash("terraform-${name}") - //unstash does not print verbose output , hence printing contents of the directory for logging purposes - sh "ls -la ${pwd()}" - } catch (error) { - echo "error unstashing: ${error}" - } - } - } - withTools(k8s: installK8s, gcp: withGCP, nodejs: withNodejs) { - if (isPackaging && (directory.equals('x-pack/agentbeat') || directory.equals('x-pack/osquerybeat'))) { - sh(label: 'install msitools', script: '.buildkite/scripts/install-msitools.sh') - } - // make commands use -C while mage commands require the dir(folder) - // let's support this scenario with the location variable. - dir(isMage ? directory : '') { - if (enableRetry) { - // Retry the same command to bypass any kind of flakiness. - // Downside: genuine failures will be repeated. - retry(3) { - cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}") - } - } else { - cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}") - } - } - } - // Publish packages should happen always to easily consume those artifacts if the - // e2e were triggered and failed. - if (isPackaging) { - publishPackages("${directory}") - pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch) - } - if(isE2E) { - e2e(args) - } - } - } -} - -/** -* This method wraps all the environment setup and pre-requirements to run any commands. -*/ -def withBeatsEnv(Map args = [:], Closure body) { - def archive = args.get('archive', true) - def withModule = args.get('withModule', false) - def directory = args.get('directory', '') - - def path, magefile, pythonEnv, testResults, artifacts, gox_flags, userProfile - - if(isUnix()) { - gox_flags = (isArm() && is64arm()) ? '-arch arm' : '-arch amd64' - path = "${env.WORKSPACE}/bin:${env.PATH}:/usr/local/bin" - magefile = "${WORKSPACE}/.magefile" - pythonEnv = "${WORKSPACE}/python-env" - testResults = '**/build/TEST*.xml' - artifacts = '**/build/TEST*.out' - } else { - // NOTE: to support Windows 7 32 bits the arch in the mingw and go context paths is required. - def mingwArch = is32() ? '32' : '64' - def chocoPath = 'C:\\ProgramData\\chocolatey\\bin' - def chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' - userProfile="${env.WORKSPACE}" - path = "${env.WORKSPACE}\\bin;${chocoPath};${chocoPython3Path};C:\\tools\\mingw${mingwArch}\\bin;${env.PATH}" - magefile = "${env.WORKSPACE}\\.magefile" - testResults = "**\\build\\TEST*.xml" - artifacts = "**\\build\\TEST*.out" - gox_flags = '-arch 386' - } - - // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace. - // Windows workers are ephemerals, so this should not really affect us. - if(isUnix()) { - deleteDir() - } - - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' - withEnv([ - "DOCKER_PULL=0", - "GOPATH=${env.WORKSPACE}", - "GOX_FLAGS=${gox_flags}", - "HOME=${env.WORKSPACE}", - "MAGEFILE_CACHE=${magefile}", - "MODULE=${module}", - "PATH=${path}", - "PYTHON_ENV=${pythonEnv}", - "RACE_DETECTOR=true", - "TEST_COVERAGE=true", - "TEST_TAGS=${env.TEST_TAGS},oracle", - "OLD_USERPROFILE=${env.USERPROFILE}", - "USERPROFILE=${userProfile}" - ]) { - if(isDockerInstalled()) { - dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - dockerLogin(secret: "${DOCKERHUB_SECRET}", registry: 'docker.io') - } - withMageEnv(version: "${env.GO_VERSION}"){ - dir("${env.BASE_DIR}") { - // Go/Mage installation is not anymore configured with env variables and installed - // with installTools but delegated to the parent closure withMageEnv. - installTools(args) - // Skip to upload the generated files by default. - def upload = false - try { - // Add more stability when dependencies are not accessible temporarily - // See https://github.com/elastic/beats/issues/21609 - // retry/try/catch approach reports errors, let's avoid it to keep the - // notifications cleaner. - if (cmd(label: 'Download modules to local cache', script: 'go mod download', returnStatus: true) > 0) { - cmd(label: 'Download modules to local cache - retry', script: 'go mod download', returnStatus: true) - } - withOtelEnv() { - withTerraformEnv(version: env.TERRAFORM_VERSION) { - body() - } - } - } catch(err) { - // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage - upload = true - error("Error '${err.toString()}'") - } finally { - if (archive) { - archiveArtifacts(allowEmptyArchive: true, artifacts: "${directory}/build/system-tests/docker-logs/TEST-docker-compose-*.log, ${directory}/build/integration-tests/**/**") - archiveTestOutput(directory: directory, testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) - } - tearDown() - } - } - } - } -} - -/** -* Tear down the setup for the permanent workers. -*/ -def tearDown() { - catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { - cmd(label: 'Remove the entire module cache', script: 'go clean -modcache', returnStatus: true) - fixPermissions("${WORKSPACE}") - // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace. - // Windows workers are ephemerals, so this should not really affect us. - if (isUnix()) { - dir("${WORKSPACE}") { - deleteDir() - } - } - } -} - -/** -* This method fixes the filesystem permissions after the build has happenend. The reason is to -* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental -* issues. -*/ -def fixPermissions(location) { - if(isUnix()) { - try { - timeout(5) { - sh(label: 'Fix permissions', script: """#!/usr/bin/env bash - set +x - echo "Cleaning up ${location}" - source ./dev-tools/common.bash - docker_setup - script/fix_permissions.sh ${location}""", returnStatus: true) - } - } catch (Throwable e) { - echo "There were some failures when fixing the permissions. ${e.toString()}" - } - } -} - -/** -* This method installs the required dependencies that are for some reason not available in the -* CI Workers. -*/ -def installTools(args) { - def stepHeader = "${args.id?.trim() ? args.id : env.STAGE_NAME}" - if(isUnix()) { - retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "${stepHeader} - Install Python/Docker", script: '.ci/scripts/install-tools.sh') } - // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. - // See https://github.com/elastic/beats/issues/17787. - sh(label: 'check git config', script: ''' - if [ -z "$(git config --get user.email)" ]; then - git config --global user.email "beatsmachine@users.noreply.github.com" - git config --global user.name "beatsmachine" - fi''') - } else { - retryWithSleep(retries: 3, seconds: 5, backoff: true){ bat(label: "${stepHeader} - Install Python", script: ".ci/scripts/install-tools.bat") } - } -} - -/** -* This method gathers the module name, if required, in order to run the ITs only if -* the changeset affects a specific module. -* -* For such, it's required to look for changes under the module folder and exclude anything else -* such as asciidoc and png files. -*/ -def getCommonModuleInTheChangeSet(String directory) { - // Use contains to support the target(target: 'make -C ') while target(directory: '', target: '...') - def pattern = (directory.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) - def module = '' - - // Transform folder structure in regex format since path separator is required to be escaped - def transformedDirectory = directory.replaceAll('/', '\\/') - def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" - def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" - dir("${env.BASE_DIR}") { - module = getGitMatchingGroup(pattern: pattern, exclude: exclude) - if(!fileExists("${directory}/module/${module}")) { - module = '' - } - } - return module -} - -/** -* This method archives and report the tests output, for such, it searches in certain folders -* to bypass some issues when working with big repositories. -*/ -def archiveTestOutput(Map args = [:]) { - def directory = args.get('directory', '') - - catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - if (isUnix()) { - fixPermissions("${WORKSPACE}") - } - // Remove pycache directory and go vendors cache folders - if (isUnix()) { - dir('build') { - sh(label: 'Delete folders that are causing exceptions (See JENKINS-58421)', returnStatus: true, - script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;') - } - } else { - bat(label: 'Delete ve folder', returnStatus: true, - script: 'FOR /d /r . %%d IN ("ve") DO @IF EXIST "%%d" rmdir /s /q "%%d"') - } - cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py', returnStatus: true) - dir('build') { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) - if (args.upload) { - tarAndUploadArtifacts(file: "test-build-artifacts-${args.id}.tgz", location: '.') - } - } - if (args.upload) { - catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - withMageEnv(version: "${env.GO_VERSION}"){ - dir(directory){ - cmd(label: "Archive system tests files", script: 'mage packageSystemTests', returnStatus: true) - } - } - - def fileName = 'build/system-tests-*.tar.gz' // see dev-tools/mage/target/common/package.go#PackageSystemTests method - def files = findFiles(glob: "${fileName}") - - if (files?.length > 0) { - googleStorageUploadExt( - bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}", - credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", - pattern: "${fileName}", - sharedPublicly: true - ) - } else { - log(level: 'WARN', text: "There are no system-tests files to upload Google Storage}") - } - } - } - } -} - -/** -* Wrapper to tar and upload artifacts to Google Storage to avoid killing the -* disk space of the jenkins instance -*/ -def tarAndUploadArtifacts(Map args = [:]) { - def fileName = args.file.replaceAll('[^A-Za-z-0-9]','-') - tar(file: fileName, dir: args.location, archive: false, allowMissing: true) - googleStorageUploadExt(bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}", - credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", - pattern: "${fileName}", - sharedPublicly: true) -} - -/** -* This method executes a closure with credentials for cloud test -* environments. -*/ -def withCloudTestEnv(Map args = [:], Closure body) { - def maskedVars = [] - def testTags = "${env.TEST_TAGS}" - - // Allow AWS credentials when the build was configured to do so with: - // - the cloudtests build parameters - // - the aws github label - // - forced with the cloud argument aws github label - if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws') || args.get('withAWS', false)) { - testTags = "${testTags},aws" - def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data - if (!aws.containsKey('access_key')) { - error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'") - } - if (!aws.containsKey('secret_key')) { - error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'") - } - maskedVars.addAll([ - [var: "AWS_REGION", password: "${env.AWS_REGION}"], - [var: "AWS_ACCESS_KEY_ID", password: aws.access_key], - [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key], - ]) - log(level: 'INFO', text: 'withCloudTestEnv: it has been configured to run in AWS.') - } - - withEnv([ - "TEST_TAGS=${testTags}", - ]) { - withEnvMask(vars: maskedVars) { - body() - } - } -} - -/** -* Start testing environment on cloud using terraform. Terraform files are -* stashed so they can be used by other stages. They are also archived in -* case manual cleanup is needed. -* -* Example: -* startCloudTestEnv(name: 'x-pack-metricbeat', dirs: ['x-pack/metricbeat/module/aws']) -* ... -* terraformCleanup(name: 'x-pack-metricbeat', dir: 'x-pack/metricbeat') -*/ -def startCloudTestEnv(Map args = [:]) { - String name = normalise(args.name) - def dirs = args.get('dirs',[]) - stage("${name}-prepare-cloud-env"){ - withBeatsEnv(archive: false, withModule: false) { - try { - // Run the docker services to setup the emulated cloud environment - sh(label: 'Run docker-compose services for emulated cloud env', script: ".ci/scripts/install-docker-services.sh ", returnStatus: true) - dirs?.each { folder -> - retryWithSleep(retries: 2, seconds: 5, backoff: true){ - terraformApply(folder) - } - } - } catch(err) { - dirs?.each { folder -> - // If it failed then cleanup without failing the build - sh(label: 'Terraform Cleanup', script: ".ci/scripts/terraform-cleanup.sh ${folder}", returnStatus: true) - } - // Cleanup the docker services - sh(label: 'Docker Compose Cleanup', script: ".ci/scripts/docker-services-cleanup.sh", returnStatus: true) - - error('startCloudTestEnv: terraform apply failed.') - } finally { - dirs?.each { folder -> - // Archive terraform states in case manual cleanup is needed. - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') - dir("${folder}") { - stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**,outputs*.yml') - } - } - } - } - } -} - -/** -* Run terraform in the given directory -*/ -def terraformApply(String directory) { - terraformInit(directory) - dir(directory) { - withEnv(["TF_VAR_BRANCH=${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')}", - "TF_VAR_BUILD_ID=${BUILD_ID}", - "TF_VAR_CREATED_DATE=${new Date().getTime()}", - "TF_VAR_ENVIRONMENT=ci", - "TF_VAR_REPO=${env.REPO}"]) { - sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") - } - } -} - -/** -* Tear down the terraform environments, by looking for all terraform states in directory -* then it runs terraform destroy for each one. -* It uses terraform states previously stashed by startCloudTestEnv. -* This also tears down any associated docker services -*/ -def terraformCleanup(Map args = [:]) { - String name = normalise(args.name) - String directory = args.dir - stage("${name}-tear-down-cloud-env"){ - withBeatsEnv(archive: false, withModule: false) { - unstash("terraform-${name}") - retryWithSleep(retries: 2, seconds: 5, backoff: true) { - sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") - } - // Cleanup associated docker services - sh(label: 'Docker Compose Cleanup', script: ".ci/scripts/docker-services-cleanup.sh") - } - } -} - -/** -* Prepare the terraform context in the given directory -*/ -def terraformInit(String directory) { - dir(directory) { - sh(label: "Terraform Init on ${directory}", script: "terraform init") - } -} - -/** -* Replace the slashes in the directory in case there are nested folders. -*/ -def normalise(String directory) { - return directory.replaceAll("[\\W]|_",'-') -} - -/** -* For debugging purposes. -*/ -def dumpVariables(){ - echo "### MAGE DUMP ###" - cmd(label: 'Dump mage variables', script: 'mage dumpVariables') - echo "### END MAGE DUMP ###" - echo """ - ### ENV DUMP ### - BEAT_VERSION: ${env.BEAT_VERSION} - BEATS: ${env.BEATS} - BUILD_DIR: ${env.BUILD_DIR} - COMMIT_ID: ${env.COMMIT_ID} - COVERAGE_DIR: ${env.COVERAGE_DIR} - COVERAGE_TOOL: ${env.COVERAGE_TOOL} - COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO} - DOCKER_CACHE: ${env.DOCKER_CACHE} - DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME} - DOCKER_COMPOSE: ${env.DOCKER_COMPOSE} - FIND: ${env.FIND} - GOBUILD_FLAGS: ${env.GOBUILD_FLAGS} - GOIMPORTS: ${env.GOIMPORTS} - GOIMPORTS_REPO: ${env.GOIMPORTS_REPO} - GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX} - GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP} - GOX_FLAGS: ${env.GOX_FLAGS} - GOX_OS: ${env.GOX_OS} - GOX_OSARCH: ${env.GOX_OSARCH} - HOME: ${env.HOME} - PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS} - PYTEST_OPTIONS: ${env.PYTEST_OPTIONS} - NOW: ${env.NOW} - PATH: ${env.PATH} - PKG_BUILD_DIR: ${env.PKG_BUILD_DIR} - PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR} - PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS} - PROJECTS: ${env.PROJECTS} - PROJECTS_ENV: ${env.PROJECTS_ENV} - PYTHON_ENV: ${env.PYTHON_ENV} - PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE} - PYTHON_EXE: ${env.PYTHON_EXE} - PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES} - PROCESSES: ${env.PROCESSES} - STRESS_TESTS: ${env.STRESS_TESTS} - STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS} - SYSTEM_TESTS: ${env.SYSTEM_TESTS} - TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO} - TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT} - TEST_TAGS: ${env.TEST_TAGS} - TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT} - TIMEOUT: ${env.TIMEOUT} - USERPROFILE: ${env.USERPROFILE} - VENV_PARAMS: ${env.VENV_PARAMS} - XPACK_SUFFIX: ${env.XPACK_SUFFIX} - ### END ENV DUMP ### - """ -} - -def isDockerInstalled(){ - if (env?.NODE_LABELS?.toLowerCase().contains('macosx')) { - log(level: 'WARN', text: "Macosx workers require some docker-machine context. They are not used for anything related to docker stuff yet.") - return false - } - if (isUnix()) { - return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) == 0 - } - return false -} - -/** -* Notify the build reason. -*/ -def notifyBuildReason() { - // Archive the build reason here, since the workspace can be deleted when running the parallel stages. - archiveArtifacts(allowEmptyArchive: true, artifacts: 'build-reasons/*.*') - if (isPR()) { - echo 'TODO: Add a comment with the build reason (this is required to be implemented in the shared library)' - } -} - -/** -* This class is the one used for running the parallel stages, therefore -* its arguments are passed by the beatsStages step. -* -* What parameters/arguments are supported: -* - label -> the worker labels -* - project -> the name of the project that should match with the folder name. -* - content -> the specific stage data in the /Jenkinsfile.yml -* - context -> the name of the stage, normally -(-)? -*/ -class RunCommand extends co.elastic.beats.BeatsFunction { - public RunCommand(Map args = [:]){ - super(args) - } - public run(Map args = [:]){ - steps.stageStatusCache(args){ - def withModule = args.content.get('withModule', false) - def installK8s = args.content.get('installK8s', false) - def withAWS = args.content.get('withAWS', false) - def withGCP = args.content.get('withGCP', false) - def withNodejs = args.content.get('withNodejs', false) - // - // What's the retry policy for fighting the flakiness: - // 1) Lint/Packaging/Cloud/k8sTest stages don't retry, since their failures are normally legitim - // 2) All the remaining stages will retry the command within the same worker/workspace if any failure - // - // NOTE: stage: checks uses target function while cloud and k8sTest use a different function - // - def enableRetry = (args.content.get('stage', 'enabled').toLowerCase().equals('checks') || - args?.content?.containsKey('packaging-arm') || - args?.content?.containsKey('packaging-linux')) ? false : true - if(args?.content?.containsKey('make')) { - steps.target(context: args.context, - command: args.content.make, - directory: args.project, - label: args.label, - withModule: withModule, - installK8s: installK8s, - isMage: false, - id: args.id, - enableRetry: enableRetry) - } - if(args?.content?.containsKey('mage')) { - steps.target(context: args.context, - command: args.content.mage, - directory: args.project, - label: args.label, - installK8s: installK8s, - withModule: withModule, - isMage: true, - withGCP: withGCP, - withNodejs: withNodejs, - id: args.id, - enableRetry: enableRetry) - } - if(args?.content?.containsKey('packaging-arm')) { - steps.packagingArm(context: args.context, - command: args.content.get('packaging-arm'), - directory: args.project, - label: args.label, - isMage: true, - id: args.id, - e2e: args.content.get('e2e'), - package: true, - dockerArch: 'arm64', - enableRetry: enableRetry) - } - if(args?.content?.containsKey('packaging-linux')) { - steps.packagingLinux(context: args.context, - command: args.content.get('packaging-linux'), - directory: args.project, - label: args.label, - isMage: true, - id: args.id, - e2e: args.content.get('e2e'), - package: true, - dockerArch: 'amd64', - enableRetry: enableRetry) - } - if(args?.content?.containsKey('k8sTest')) { - steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) - } - if(args?.content?.containsKey('cloud')) { - steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id, withAWS: withAWS) - } - } - } -} - -/** -* This class retrieves the dependencies of a Go module for such it transforms them in a -* regex pattern. -*/ -class GetProjectDependencies extends co.elastic.beats.BeatsFunction { - public GetProjectDependencies(Map args = [:]){ - super(args) - } - public run(Map args = [:]){ - def output = "" - steps.withEnv(["HOME=${steps.env.WORKSPACE}"]) { - output = steps.sh(label: 'Get vendor dependency patterns', returnStdout: true, - script: ".ci/scripts/get-vendor-dependencies.sh ${args.project}") - } - return output?.split('\n').collect{ item -> item as String } - } -} diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml deleted file mode 100644 index 399e0be48d11..000000000000 --- a/Jenkinsfile.yml +++ /dev/null @@ -1,46 +0,0 @@ -projects: - - "auditbeat" - - "deploy/kubernetes" - - "filebeat" - - "heartbeat" - - "libbeat" - - "metricbeat" - - "packetbeat" - - "winlogbeat" - - "x-pack/agentbeat" - - "x-pack/auditbeat" - - "x-pack/dockerlogbeat" - - "x-pack/filebeat" - - "x-pack/functionbeat" - - "x-pack/heartbeat" - - "x-pack/libbeat" - - "x-pack/metricbeat" - - "x-pack/osquerybeat" - - "x-pack/packetbeat" - - "x-pack/winlogbeat" - -## Changeset macros that are defined here and used in each specific 2.0 pipeline. -changeset: - ci: - - "^Jenkinsfile" - - "^\\.ci/scripts/.*" - oss: - - "^go.mod" - - "^pytest.ini" - - "^dev-tools/.*" - - "^libbeat/.*" - - "^testing/.*" - xpack: - - "^go.mod" - - "^pytest.ini" - - "^dev-tools/.*" - - "^libbeat/.*" - - "^testing/.*" - - "^x-pack/libbeat/.*" - -disabled: - when: - labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. - - skip-ci - ## TODO: This will allow to configure what to do based on the PR configuration - draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/README.md b/README.md index ff342930232f..264d1b87acb6 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ It is possible to trigger some jobs by putting a comment on a GitHub PR. (This service is only available for users affiliated with Elastic and not for open-source contributors.) * [beats][] - * `jenkins run the tests please` or `jenkins run tests` or `/test` will kick off a default build. + * `buildkite test it` or `/test` will kick off a default build. * `/test macos` will kick off a default build with also the `macos` stages. * `/test ` will kick off the default build for the given PR in addition to the `` build itself. * `/test for macos` will kick off a default build with also the `macos` stage for the ``. diff --git a/dev-tools/common.bash b/dev-tools/common.bash index be2bb0d71b31..9763dd6d104f 100644 --- a/dev-tools/common.bash +++ b/dev-tools/common.bash @@ -76,32 +76,6 @@ setup_go_path() { debug "GOPATH=${GOPATH}" } -jenkins_setup() { - : "${HOME:?Need to set HOME to a non-empty value.}" - : "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}" - - if [ -z ${GO_VERSION:-} ]; then - get_go_version - fi - - # Setup Go. - export GOPATH=${WORKSPACE} - export PATH=${GOPATH}/bin:${PATH} - eval "$(gvm ${GO_VERSION})" - - # Workaround for Python virtualenv path being too long. - export TEMP_PYTHON_ENV=$(mktemp -d) - - # Workaround for cryptography package (pip dependency) relying on rust - export CRYPTOGRAPHY_DONT_BUILD_RUST=1 - - export PYTHON_ENV="${TEMP_PYTHON_ENV}/python-env" - - # Write cached magefile binaries to workspace to ensure - # each run starts from a clean slate. - export MAGEFILE_CACHE="${WORKSPACE}/.magefile" -} - docker_setup() { OS="$(uname)" case $OS in diff --git a/dev-tools/find_pr_jenkins_jobs.sh b/dev-tools/find_pr_jenkins_jobs.sh deleted file mode 100755 index f42cbe4da19b..000000000000 --- a/dev-tools/find_pr_jenkins_jobs.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# Description: -# This script finds Jenkins jobs for a given Beats PR. -# -# Usage: -# ./find_pr_jenkins_job.sh PR_NUMBER -# -# Example: -# ./find_pr_jenkins_job.sh 15790 -# -# Dependencies: -# curl, jq - -set -e - -NUM_JOBS_TO_SEARCH=100 - -get_pr_from_input() { - pr=$1 - if [ -z $pr ]; then - echo "Usage: ./find_jenkins_job.sh PR_NUMBER" >&2 - exit 1 - fi - - echo $pr -} - -find_latest_beats_pr_job() { - curl -s 'https://beats-ci.elastic.co/job/elastic+beats+pull-request/api/json' | jq '.builds[0].number' -} - -find_job_for_pr() { - job=$1 - pr=$2 - - found=$(curl -s "https://beats-ci.elastic.co/job/elastic+beats+pull-request/$job/api/json" \ - | jq -c ".actions[] | select(._class == \"org.jenkinsci.plugins.ghprb.GhprbParametersAction\").parameters[] | select(.name == \"ghprbPullId\" and .value == \"$pr\")" \ - | wc -l) - - echo $found -} - -main() { - pr=$(get_pr_from_input $1) - echo "Searching last $NUM_JOBS_TO_SEARCH Jenkins jobs for PR number: $pr..." - - n=$(find_latest_beats_pr_job $pr) - let e=$n-$NUM_JOBS_TO_SEARCH - - while [ $n -gt $e ]; do - found=$(find_job_for_pr $n $pr) - if [ $found -gt 0 ]; then - echo "https://beats-ci.elastic.co/job/elastic+beats+pull-request/$n/" - fi - - let n=$n-1 - done -} - -main $1 diff --git a/dev-tools/jenkins_ci.sh b/dev-tools/jenkins_ci.sh deleted file mode 100755 index e425ef1752d9..000000000000 --- a/dev-tools/jenkins_ci.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -set -euox pipefail - -: "${HOME:?Need to set HOME to a non-empty value.}" -: "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}" -: "${beat:?Need to set beat to a non-empty value.}" - -if [ ! -d "$beat" ]; then - echo "$beat does not exist" - mkdir -p build - touch build/TEST-empty.out - exit -fi - -source ./dev-tools/common.bash - -jenkins_setup - -cleanup() { - echo "Running cleanup..." - rm -rf $TEMP_PYTHON_ENV - - if docker info > /dev/null ; then - make stop-environment || true - make fix-permissions || true - echo "Killing all running containers..." - ids=$(docker ps -q) - if [ -n "$ids" ]; then - docker kill $ids - fi - echo "Cleaning stopped docker containers and dangling images/networks/volumes..." - docker system prune -f || true - fi - - echo "Cleanup complete." -} -trap cleanup EXIT - -rm -rf ${GOPATH}/pkg -cd ${beat} -RACE_DETECTOR=1 make clean check testsuite diff --git a/dev-tools/jenkins_intake.sh b/dev-tools/jenkins_intake.sh deleted file mode 100755 index de8ec969ddbc..000000000000 --- a/dev-tools/jenkins_intake.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -euox pipefail - -: "${HOME:?Need to set HOME to a non-empty value.}" -: "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}" - -source ./dev-tools/common.bash - -jenkins_setup - -cleanup() { - echo "Running cleanup..." - rm -rf $TEMP_PYTHON_ENV - echo "Cleanup complete." -} -trap cleanup EXIT - -make check diff --git a/dev-tools/jenkins_release.sh b/dev-tools/jenkins_release.sh deleted file mode 100755 index 02950637610d..000000000000 --- a/dev-tools/jenkins_release.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -set -euox pipefail - -: "${HOME:?Need to set HOME to a non-empty value.}" -: "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}" - -source $(dirname "$0")/common.bash - -jenkins_setup -docker_setup - -cleanup() { - echo "Running cleanup..." - rm -rf $TEMP_PYTHON_ENV - - if docker info > /dev/null ; then - echo "Killing all running containers..." - ids=$(docker ps -q) - if [ -n "$ids" ]; then - docker kill $ids - fi - echo "Cleaning stopped docker containers and dangling images/networks/volumes..." - docker system prune -f || true - - os=$(uname -s) - if [ "$os" == "Darwin" ]; then - # Clean images accept for the ones we're currently using in order to - # gain some disk space. - echo "Disk space before image cleanup:" - df -h / - docker system df - echo "Cleaning images" - docker images --format "{{.ID}} {{.Repository}}:{{.Tag}}" \ - | grep -v "docker.elastic.co/beats-dev/golang-crossbuild:$(cat .go-version)-" \ - | awk '{print $1}' \ - | xargs docker rmi -f || true - echo "Disk space after image cleanup:" - df -h / - docker system df - fi - fi - - echo "Cleanup complete." -} -trap cleanup EXIT - -# This controls the defaults used the Jenkins package job. They can be -# overridden by setting them in the environment prior to running this script. -export SNAPSHOT="${SNAPSHOT:-true}" -export PLATFORMS="${PLATFORMS:-+linux/armv7 +linux/ppc64le +linux/s390x +linux/mips64}" - -make release diff --git a/docs/devguide/terraform.asciidoc b/docs/devguide/terraform.asciidoc index 2c21c8f43140..0cdd0198f214 100644 --- a/docs/devguide/terraform.asciidoc +++ b/docs/devguide/terraform.asciidoc @@ -77,25 +77,5 @@ resources. These tests are disabled by default in CI. Terraform states are archived as artifacrs of builds, this allows to manually destroy resources created by builds that were not able to do a proper cleanup. -Here is a checklist to add support for a cloud feature in Jenkins: -* In the feature code: - * Tests have a build tag so they are disabled by default. When run from mage, - its execution can be selected using the `TEST_TAGS` environment variable, e.g: - `TEST_TAGS=aws` for AWS tests. - * There is some Terraform configuration that defines a cloud scenario where - tests pass. This configuration should be in the directory of the feature. -* In the Jenkinsfile: - * Add a boolean parameter to run the tests on this environment, e.g. - `awsCloudTests`. This parameter should be set to false by default. - * Add a conditional block in `withCloudTestEnv` that: - * Will be executed if the previously added boolean parameter, or `allCloudTests` - are set to true. - * Adds the tag to `TEST_TAGS` (as comma separated values), so tests are - selected. - * Defines how to obtain the credentials and provide them to the tests. - * In the stage of the specific beat: - * Add a stage that calls to `startCloudTestEnv`, if there isn't anyone. - * Add a post cleanup step that calls to `terraformCleanup`, if there isn't anyone. - * Add a environment to the list of environments started by `startCloudEnv`, - with the condition to start the scenario, and the path to the directory - with its definition, e.g. `[cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws']` + + diff --git a/filebeat/scripts/jenkins/unit-test.sh b/filebeat/scripts/jenkins/unit-test.sh deleted file mode 100755 index de676b95ecac..000000000000 --- a/filebeat/scripts/jenkins/unit-test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -exuo pipefail - -source ./dev-tools/common.bash - -jenkins_setup - -mage GoUnitTest || echo -e "\033[31;49mTests FAILED\033[0m" diff --git a/libbeat/.gitignore b/libbeat/.gitignore index 49eb469dd6fc..857e0723f18a 100644 --- a/libbeat/.gitignore +++ b/libbeat/.gitignore @@ -23,7 +23,6 @@ _testmain.go # IDE / OS specific files .idea -.jenkins /libbeat.yml /libbeat.reference.yml diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml deleted file mode 100644 index 98d7281a0f86..000000000000 --- a/x-pack/filebeat/Jenkinsfile.yml +++ /dev/null @@ -1,151 +0,0 @@ -when: - branches: true ## for all the branches - changeset: ## when PR contains any of those entries in the changeset - - "^x-pack/filebeat/.*" - - "@ci" ## special token regarding the changeset for the ci - - "@xpack" ## special token regarding the changeset for the xpack - comments: ## when PR comment contains any of those entries - - "/test x-pack/filebeat" - labels: ## when PR labels matches any of those entries - - "x-pack-filebeat" - parameters: ## when parameter was selected in the UI. - - "x-pack-filebeat" - tags: true ## for all the tags -platform: "immutable && ubuntu-22" ## default label for all the stages -stages: - arm: - mage: "mage build unitTest" - platforms: ## override default label in this specific stage. - - "ubuntu-2204-aarch64" - when: ## Override the top-level when. - comments: - - "/test x-pack/filebeat for arm" - labels: - - "arm" - parameters: - - "armTest" - branches: true ## for all the branches - tags: true ## for all the tags - stage: extended - unitTest: - mage: "mage build unitTest" - stage: mandatory - goIntegTest: - mage: "mage goIntegTest" - stage: mandatory - pythonIntegTest: - mage: "mage pythonIntegTest" ## run the ITs only if the changeset affects a specific module. - stage: mandatory - macos: - mage: "mage build unitTest" - platforms: ## override default label in this specific stage. - - "macos12 && x86_64" - when: ## Override the top-level when. - comments: - - "/test x-pack/filebeat for macos" - labels: - - "macOS" - parameters: - - "macosTest" - tags: true ## for all the tags - stage: extended - macosM1: - mage: "mage build unitTest" - platforms: ## override default label in this specific stage. - - "orka && darwin && aarch64" - when: ## Override the top-level when. - comments: - - "/test filebeat for macos-m1" - labels: - - "macos-m1" - parameters: - - "macosM1Test" - tags: false ## for all the tags - stage: extended - windows-2022: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2022" - stage: mandatory - windows-2019: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2019" - stage: extended_win - windows-2016: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2016" - stage: mandatory - windows-2012: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2012-r2" - stage: extended_win - windows-11: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-11" - stage: extended_win - windows-10: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-10" - stage: extended_win - windows-8: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-8" - stage: extended_win - cloud: - cloud: "mage build test" - withModule: true ## run the ITs only if the changeset affects a specific module. - dirs: ## apply terraform for the given directory. - - "x-pack/filebeat/input/awss3/_meta/terraform" - when: ## Override the top-level when. - parameters: - - "awsCloudTests" - comments: - - "/test x-pack/filebeat for aws cloud" - labels: - - "aws" - stage: extended - # Skip test until fixed https://github.com/elastic/beats/issues/36425 - # cloudAWS: - # cloud: "mage build test goIntegTest" - # withAWS: true ## Enable the tests to run in AWS - # withModule: true ## run the ITs only if the changeset affects a specific module. - # dirs: ## run the cloud tests for the given modules. - # - "x-pack/filebeat/input/awss3/_meta/terraform" - # when: ## Override the top-level when. - # changeset: ## when PR contains any of those entries in the changeset - # - "^x-pack/filebeat/input/awss3/.*" - # - "^x-pack/filebeat/module/aws/.*" - # - "^x-pack/filebeat/input/awscloudwatch/.*" - # - "^x-pack/filebeat/Jenkinsfile.yml" - # - "^x-pack/libbeat/common/aws/.*" - # stage: extended - packaging-linux: - packaging-linux: "mage package" - e2e: - enabled: false - stage: packaging - when: - branches: false ## Only on a PR basis for the time being - tags: false ## packaging on branches/tags is already in place with the downstream build. - changeset: ## when PR contains any of those entries in the changeset - - "^x-pack/filebeat/.*" - - "@xpack" ## special token regarding the changeset for the xpack - packaging-arm: - packaging-arm: "mage package" - e2e: - enabled: false - platforms: ## override default label in this specific stage. - - "ubuntu-2204-aarch64" - stage: packaging - when: - branches: false ## Only on a PR basis for the time being - tags: false ## packaging on branches/tags is already in place with the downstream build. - changeset: ## when PR contains any of those entries in the changeset - - "^x-pack/filebeat/.*" - - "@xpack" ## special token regarding the changeset for the xpack diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml deleted file mode 100644 index 27574b62adad..000000000000 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ /dev/null @@ -1,139 +0,0 @@ -when: - branches: true ## for all the branches - changeset: ## when PR contains any of those entries in the changeset - - "^x-pack/metricbeat/.*" - - "@ci" ## special token regarding the changeset for the ci - - "@xpack" ## special token regarding the changeset for the xpack - comments: ## when PR comment contains any of those entries - - "/test x-pack/metricbeat" - labels: ## when PR labels matches any of those entries - - "x-pack-metricbeat" - parameters: ## when parameter was selected in the UI. - - "x-pack-metricbeat" - tags: true ## for all the tags -platform: "immutable && ubuntu-22" ## default label for all the stages -stages: - unitTest: - mage: "mage build unitTest" - stage: mandatory - goIntegTest: - mage: "mage goIntegTest" - withModule: true - stage: mandatory - pythonIntegTest: - mage: "mage pythonIntegTest" - withModule: true - stage: mandatory - cloud: - cloud: "mage build test" - withModule: true ## run the ITs only if the changeset affects a specific module. - dirs: ## run the cloud tests for the given modules. - - "x-pack/metricbeat/module/aws" - when: ## Override the top-level when. - parameters: - - "awsCloudTests" - comments: - - "/test x-pack/metricbeat for aws cloud" - labels: - - "aws" - # stage: extended - # Skip test until fixed https://github.com/elastic/beats/issues/36425 - #cloudAWS: - # cloud: "mage build test goIntegTest" - # withAWS: true ## Enable the tests to run in AWS - # withModule: true ## run the ITs only if the changeset affects a specific module. - # dirs: ## run the cloud tests for the given modules. - # - "x-pack/metricbeat/module/aws" - # when: ## Override the top-level when. - # changeset: ## when PR contains any of those entries in the changeset - # - "^x-pack/metricbeat/module/aws/.*" - # - "^x-pack/metricbeat/Jenkinsfile.yml" - # - "^x-pack/libbeat/common/aws/.*" - # stage: extended - macos: - mage: "mage build unitTest" - platforms: ## override default label in this specific stage. - - "macos12 && x86_64" - when: ## Override the top-level when. - comments: - - "/test x-pack/metricbeat for macos" - labels: - - "macOS" - parameters: - - "macosTest" - tags: true ## for all the tags - stage: extended - # Ignore as long as there are test failures, see https://github.com/elastic/beats/issues/33036 -# macosM1: -# mage: "mage build unitTest" -# platforms: ## override default label in this specific stage. -# - "orka && darwin && aarch64" -# when: ## Override the top-level when. -# comments: -# - "/test metricbeat for macos-m1" -# labels: -# - "macos-m1" -# parameters: -# - "macosM1Test" -# tags: false ## for all the tags -# stage: extended - windows-2022: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2022" - stage: mandatory - windows-2019: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2019" - stage: extended_win - windows-2016: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2016" - stage: mandatory - windows-2012: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2012-r2" - stage: extended_win - windows-11: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-11" - stage: extended_win - windows-10: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-10" - stage: extended_win - windows-8: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-8" - stage: extended_win - packaging-linux: - packaging-linux: "mage package" - e2e: - enabled: false - entrypoint: 'metricbeat-test.sh' - stage: packaging - when: - branches: false ## Only on a PR basis for the time being - tags: false ## packaging on branches/tags is already in place with the downstream build. - changeset: ## when PR contains any of those entries in the changeset - - "^x-pack/metricbeat/.*" - - "@xpack" ## special token regarding the changeset for the xpack - packaging-arm: - packaging-arm: "mage package" - e2e: - enabled: false - platforms: ## override default label in this specific stage. - - "ubuntu-2204-aarch64" - stage: packaging - when: - branches: false ## Only on a PR basis for the time being - tags: false ## packaging on branches/tags is already in place with the downstream build. - changeset: ## when PR contains any of those entries in the changeset - - "^x-pack/metricbeat/.*" - - "@xpack" ## special token regarding the changeset for the xpack