Add ability to generate buildkite pipelines from integration testing framework

.buildkite/integration.pipeline.yml

@@ -82,7 +82,7 @@ steps:
           files: "build/TEST-*.xml"
           format: "junit"
           branches: "main"
-          debug: true    
+          debug: true
 
   - label: "Serverless Beats Tests"
     depends_on:
@@ -103,4 +103,3 @@ steps:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
 
-

magefile.go

@@ -43,6 +43,7 @@ import (
 	"github.com/elastic/elastic-agent/pkg/testing/ess"
 	"github.com/elastic/elastic-agent/pkg/testing/kubernetes/kind"
 	"github.com/elastic/elastic-agent/pkg/testing/multipass"
+	"github.com/elastic/elastic-agent/pkg/testing/null"
 	"github.com/elastic/elastic-agent/pkg/testing/ogc"
 	"github.com/elastic/elastic-agent/pkg/testing/runner"
 	"github.com/elastic/elastic-agent/pkg/testing/tools/git"
@@ -2452,6 +2453,64 @@ func (Integration) TestOnRemote(ctx context.Context) error {
 	return nil
 }
 
+func (Integration) Buildkite() error {
+	goTestFlags := os.Getenv("GOTEST_FLAGS")
+	batches, err := define.DetermineBatches("testing/integration", goTestFlags, "integration")
+	if err != nil {
+		return fmt.Errorf("failed to determine batches: %w", err)
+	}
+	agentVersion, agentStackVersion, err := getTestRunnerVersions()
+	if err != nil {
+		return fmt.Errorf("failed to get agent versions: %w", err)
+	}
+	goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion()
+	if err != nil {
+		return fmt.Errorf("failed to get go versions: %w", err)
+	}
+
+	cfg := runner.Config{
+		AgentVersion: agentVersion,
+		StackVersion: agentStackVersion,
+		GOVersion:    goVersion,
+		BuildDir:     filepath.Join("build", "distributions"),
+		RepoDir:      ".",
+		StateDir:     ".integration-cache",
+		Platforms:    testPlatforms(),
+		Packages:     testPackages(),
+		Groups:       testGroups(),
+		Matrix:       false,
+		VerboseMode:  mg.Verbose(),
+		TestFlags:    goTestFlags,
+	}
+	r, err := runner.NewRunner(cfg, null.NewInstanceProvisioner(), null.NewStackProvisioner(), batches...)
+	if err != nil {
+		return fmt.Errorf("failed to create runner: %w", err)
+	}
+
+	steps, err := r.Buildkite()
+	if err != nil {
+		return fmt.Errorf("error generating buildkite steps: %w", err)
+	}
+
+	// write output to steps.yaml
+	cwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("error getting current working directory: %w", err)
+	}
+	ymlFilePath := filepath.Join(cwd, "steps.yml")
+	file, err := os.Create(ymlFilePath)
+	if err != nil {
+		return fmt.Errorf("error creating file: %w", err)
+	}
+	defer file.Close()
+	if _, err := file.WriteString(steps); err != nil {
+		return fmt.Errorf("error writing to file: %w", err)
+	}
+
+	fmt.Printf(">>> Generated buildkite steps written to: %s\n", ymlFilePath)
+	return nil
+}
+
 func integRunner(ctx context.Context, matrix bool, singleTest string) error {
 	if _, ok := ctx.Deadline(); !ok {
 		// If the context doesn't have a timeout (usually via the mage -t option), give it one.
@@ -2522,18 +2581,14 @@ func integRunnerOnce(ctx context.Context, matrix bool, singleTest string) (int,
 	return results.Failures, nil
 }
 
-func createTestRunner(matrix bool, singleTest string, goTestFlags string, batches ...define.Batch) (*runner.Runner, error) {
-	goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion()
-	if err != nil {
-		return nil, err
-	}
-
+func getTestRunnerVersions() (string, string, error) {
+	var err error
 	agentStackVersion := os.Getenv("AGENT_STACK_VERSION")
 	agentVersion := os.Getenv("AGENT_VERSION")
 	if agentVersion == "" {
 		agentVersion, err = mage.DefaultBeatBuildVariableSources.GetBeatVersion()
 		if err != nil {
-			return nil, err
+			return "", "", err
 		}
 		if agentStackVersion == "" {
 			// always use snapshot for stack version
@@ -2549,6 +2604,21 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 	if agentStackVersion == "" {
 		agentStackVersion = agentVersion
 	}
+
+	return agentVersion, agentStackVersion, nil
+}
+
+func createTestRunner(matrix bool, singleTest string, goTestFlags string, batches ...define.Batch) (*runner.Runner, error) {
+	goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion()
+	if err != nil {
+		return nil, err
+	}
+
+	agentVersion, agentStackVersion, err := getTestRunnerVersions()
+	if err != nil {
+		return nil, err
+	}
+
 	agentBuildDir := os.Getenv("AGENT_BUILD_DIR")
 	if agentBuildDir == "" {
 		agentBuildDir = filepath.Join("build", "distributions")
@@ -2604,7 +2674,6 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 	default:
 		return nil, fmt.Errorf("INSTANCE_PROVISIONER environment variable must be one of 'ogc' or 'multipass', not %s", instanceProvisionerMode)
 	}
-	fmt.Printf(">>>> Using %s instance provisioner\n", instanceProvisionerMode)
 
 	email, err := ogcCfg.ClientEmail()
 	if err != nil {
@@ -2639,7 +2708,6 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 			ess.ProvisionerServerless,
 			stackProvisionerMode)
 	}
-	fmt.Printf(">>>> Using %s stack provisioner\n", stackProvisionerMode)
 
 	timestamp := timestampEnabled()
 

pkg/testing/buildkite/steps.go

@@ -0,0 +1,25 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package buildkite
+
+type StepAgent struct {
+	Provider     string `json:"provider,omitempty" yaml:"provider,omitempty"`
+	ImageProject string `json:"imageProject,omitempty" yaml:"imageProject,omitempty"`
+	MachineType  string `json:"machineType,omitempty" yaml:"machineType,omitempty"`
+	Image        string `json:"image,omitempty" yaml:"image,omitempty"`
+}
+
+type Step struct {
+	Key                    string            `json:"key,omitempty" yaml:"key,omitempty"`
+	Label                  string            `json:"label,omitempty" yaml:"label,omitempty"`
+	Command                string            `json:"command,omitempty" yaml:"command,omitempty"`
+	Env                    map[string]string `json:"env,omitempty" yaml:"env,omitempty"`
+	ArtifactPaths          []string          `json:"artifact_paths,omitempty" yaml:"artifact_paths,omitempty"`
+	Agents                 []StepAgent       `json:"agents,omitempty" yaml:"agents,omitempty"`
+	DependsOn              []string          `json:"depends_on,omitempty" yaml:"depends_on,omitempty"`
+	AllowDependencyFailure bool              `json:"allow_dependency_failure,omitempty" yaml:"allow_dependency_failure,omitempty"`
+	Steps                  []Step            `json:"steps,omitempty" yaml:"steps,omitempty"`
+	Skip                   bool              `json:"skip,omitempty" yaml:"skip,omitempty"`
+}

pkg/testing/null/instance.go

@@ -0,0 +1,51 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package null
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/elastic/elastic-agent/pkg/testing/define"
+	"github.com/elastic/elastic-agent/pkg/testing/runner"
+)
+
+const (
+	Name = "null"
+)
+
+type instanceProvisioner struct {
+	logger runner.Logger
+}
+
+// NewInstanceProvisioner creates the null provisioner
+func NewInstanceProvisioner() runner.InstanceProvisioner {
+	return &instanceProvisioner{}
+}
+
+func (p *instanceProvisioner) Name() string {
+	return Name
+}
+
+func (p *instanceProvisioner) SetLogger(l runner.Logger) {
+	p.logger = l
+}
+
+func (p *instanceProvisioner) Type() runner.ProvisionerType {
+	return runner.ProvisionerTypeVM
+}
+
+func (p *instanceProvisioner) Supported(os define.OS) bool {
+	return true
+}
+
+func (p *instanceProvisioner) Provision(ctx context.Context, cfg runner.Config, batches []runner.OSBatch) ([]runner.Instance, error) {
+	return nil, fmt.Errorf("null provisioner cannot provision")
+}
+
+func (p *instanceProvisioner) Clean(ctx context.Context, _ runner.Config, instances []runner.Instance) error {
+	// nothing to clean
+	return nil
+}

pkg/testing/null/stack.go

@@ -0,0 +1,43 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+package null
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/elastic/elastic-agent/pkg/testing/runner"
+)
+
+type stackProvisioner struct {
+	log runner.Logger
+}
+
+// NewStackProvisioner creates a new null stack provisioner
+func NewStackProvisioner() runner.StackProvisioner {
+	return &stackProvisioner{}
+}
+
+func (prov *stackProvisioner) Name() string {
+	return Name
+}
+
+func (prov *stackProvisioner) SetLogger(l runner.Logger) {
+	prov.log = l
+}
+
+func (prov *stackProvisioner) Create(ctx context.Context, request runner.StackRequest) (runner.Stack, error) {
+	return runner.Stack{}, fmt.Errorf("null provisioner cannot provision")
+}
+
+func (prov *stackProvisioner) WaitForReady(ctx context.Context, stack runner.Stack) (runner.Stack, error) {
+	// nothing to wait for
+	return runner.Stack{}, nil
+}
+
+func (prov *stackProvisioner) Delete(ctx context.Context, stack runner.Stack) error {
+	// nothing to delete
+	return nil
+}