Working on buildkite integration runner.

elastic · Aug 30, 2024 · e9548d6 · e9548d6
1 parent 6bb6b1e
commit e9548d6
Show file tree

Hide file tree

Showing 5 changed files with 446 additions and 74 deletions.
diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml
@@ -14,85 +14,98 @@ steps:
       provider: "gcp"
       machineType: "n1-standard-8"
 
-  - label: "Serverless integration test"
-    key: "serverless-integration-tests"
-    depends_on:
-      - package-it
-    concurrency_group: elastic-agent-extended-testing/serverless-integration
-    concurrency: 8
-    env:
-      # we run each step in a different data center to spread the load
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
-    command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
-      .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
+  #- label: "Serverless integration test"
+  #  key: "serverless-integration-tests"
+  #  depends_on:
+  #    - package-it
+  #  concurrency_group: elastic-agent-extended-testing/serverless-integration
+  #  concurrency: 8
+  #  env:
+  #    # we run each step in a different data center to spread the load
+  #    TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
+  #  command: |
+  #    buildkite-agent artifact download build/distributions/** . --step 'package-it'
+  #    .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+  #  artifact_paths:
+  #    - "build/TEST-**"
+  #    - "build/diagnostics/*"
+  #  agents:
+  #    provider: "gcp"
+  #    machineType: "n1-standard-8"
+  #  notify:
+  #    - github_commit_status:
+  #        context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
 
-  - label: "Extended runtime leak tests"
-    key: "extended-integration-tests"
-    depends_on:
-      - package-it
-    concurrency_group: elastic-agent-extended-testing/leak-tests
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
-    command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
-      .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
+  #- label: "Extended runtime leak tests"
+  #  key: "extended-integration-tests"
+  #  depends_on:
+  #    - package-it
+  #  concurrency_group: elastic-agent-extended-testing/leak-tests
+  #  concurrency: 8
+  #  env:
+  #    TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
+  #  command: |
+  #    buildkite-agent artifact download build/distributions/** . --step 'package-it'
+  #    .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
+  #  artifact_paths:
+  #    - "build/TEST-**"
+  #    - "build/diagnostics/*"
+  #  agents:
+  #    provider: "gcp"
+  #    machineType: "n1-standard-8"
+  #  notify:
+  #    - github_commit_status:
+  #        context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
 
-  - label: "Integration tests"
-    key: "integration-tests"
-    depends_on:
-      - package-it
-    concurrency_group: elastic-agent-extended-testing/integration
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-f"
-    command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
-      .buildkite/scripts/steps/integration_tests.sh stateful
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Integration tests"
+  #- label: "Integration tests"
+  #  key: "integration-tests"
+  #  depends_on:
+  #    - package-it
+  #  concurrency_group: elastic-agent-extended-testing/integration
+  #  concurrency: 8
+  #  env:
+  #    TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-f"
+  #  command: |
+  #    buildkite-agent artifact download build/distributions/** . --step 'package-it'
+  #    .buildkite/scripts/steps/integration_tests.sh stateful
+  #  artifact_paths:
+  #    - "build/TEST-**"
+  #    - "build/diagnostics/*"
+  #  agents:
+  #    provider: "gcp"
+  #    machineType: "n1-standard-8"
+  #  notify:
+  #    - github_commit_status:
+  #        context: "buildkite/elastic-agent-extended-testing - Integration tests"
+
+  #- label: "Serverless Beats Tests"
+  #  depends_on:
+  #    - package-it
+  #  key: "serverless-beats-integration-tests"
+  #  concurrency_group: elastic-agent-extended-testing/beats-integration
+  #  concurrency: 8
+  #  env:
+  #    TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
+  #  command: |
+  #    buildkite-agent artifact download build/distributions/** . --step 'package-it'
+  #    .buildkite/scripts/steps/beats_tests.sh
+  #  # if: "build.env('CRON') == 'yes'"
+  #  agents:
+  #    provider: "gcp"
+  #    machineType: "n1-standard-8"
+  #  notify:
+  #    - github_commit_status:
+  #        context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
 
-  - label: "Serverless Beats Tests"
+  - label: "[BK] Integration tests"
+    key: "bk-integration-tests"
     depends_on:
       - package-it
-    key: "serverless-beats-integration-tests"
-    concurrency_group: elastic-agent-extended-testing/beats-integration
-    concurrency: 8
-    env:
-      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
     command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
-      .buildkite/scripts/steps/beats_tests.sh
-    # if: "build.env('CRON') == 'yes'"
+      mage integration:buildkite | buildkite-agent buildkite pipeline upload
     agents:
       provider: "gcp"
-      machineType: "n1-standard-8"
+      machineType: "n1-standard-4"
     notify:
       - github_commit_status:
-          context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
+          context: "buildkite/elastic-agent-extended-testing - [BK] Integration tests"
diff --git a/magefile.go b/magefile.go
@@ -98,6 +98,8 @@ var (
 	goIntegTestTimeout = 2 * time.Hour
 	// goProvisionAndTestTimeout is the timeout used for both provisioning and running tests.
 	goProvisionAndTestTimeout = goIntegTestTimeout + 30*time.Minute
+
+	logIntegrationSelections = true
 )
 
 func init() {
@@ -2406,6 +2408,31 @@ func (Integration) TestOnRemote(ctx context.Context) error {
 	return nil
 }
 
+func (Integration) Buildkite(ctx context.Context) error {
+	// don't pollute the output with log information
+	//
+	// this target outputs the YAML to stdout
+	logIntegrationSelections = false
+
+	goTestFlags := os.Getenv("GOTEST_FLAGS")
+	batches, err := define.DetermineBatches("testing/integration", goTestFlags, "integration")
+	if err != nil {
+		return fmt.Errorf("failed to determine batches: %w", err)
+	}
+	r, err := createTestRunner(false, "", goTestFlags, batches...)
+	if err != nil {
+		return fmt.Errorf("error creating test runner: %w", err)
+	}
+	steps, err := r.Buildkite()
+	if err != nil {
+		return fmt.Errorf("error generating buildkite steps: %w", err)
+	}
+
+	// output's YAML
+	fmt.Printf("%s\n", steps)
+	return nil
+}
+
 func integRunner(ctx context.Context, matrix bool, singleTest string) error {
 	if _, ok := ctx.Deadline(); !ok {
 		// If the context doesn't have a timeout (usually via the mage -t option), give it one.
@@ -2558,7 +2585,9 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 	default:
 		return nil, fmt.Errorf("INSTANCE_PROVISIONER environment variable must be one of 'ogc' or 'multipass', not %s", instanceProvisionerMode)
 	}
-	fmt.Printf(">>>> Using %s instance provisioner\n", instanceProvisionerMode)
+	if logIntegrationSelections {
+		fmt.Printf(">>>> Using %s instance provisioner\n", instanceProvisionerMode)
+	}
 
 	email, err := ogcCfg.ClientEmail()
 	if err != nil {
@@ -2593,7 +2622,9 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 			ess.ProvisionerServerless,
 			stackProvisionerMode)
 	}
-	fmt.Printf(">>>> Using %s stack provisioner\n", stackProvisionerMode)
+	if logIntegrationSelections {
+		fmt.Printf(">>>> Using %s stack provisioner\n", stackProvisionerMode)
+	}
 
 	timestamp := timestampEnabled()