Add ability to generate buildkite pipelines from integration testing framework

magefile.go

@@ -40,6 +40,8 @@ import (
 	"github.com/elastic/elastic-agent/dev-tools/mage/downloads"
 	"github.com/elastic/elastic-agent/dev-tools/mage/manifest"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download"
+	"github.com/elastic/elastic-agent/pkg/testing/buildkite"
+	tcommon "github.com/elastic/elastic-agent/pkg/testing/common"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
 	"github.com/elastic/elastic-agent/pkg/testing/ess"
 	"github.com/elastic/elastic-agent/pkg/testing/kubernetes/kind"
@@ -2134,12 +2136,12 @@ func askForVM() (runner.StateInstance, error) {
 	return instances[id], nil
 }
 
-func askForStack() (runner.Stack, error) {
+func askForStack() (tcommon.Stack, error) {
 	mg.Deps(Integration.Stacks)
 
 	state, err := readFrameworkState()
 	if err != nil {
-		return runner.Stack{}, fmt.Errorf("could not read state file: %w", err)
+		return tcommon.Stack{}, fmt.Errorf("could not read state file: %w", err)
 	}
 
 	if len(state.Stacks) == 1 {
@@ -2150,17 +2152,17 @@ func askForStack() (runner.Stack, error) {
 	id := 0
 	fmt.Print("Stack number: ")
 	if _, err := fmt.Scanf("%d", &id); err != nil {
-		return runner.Stack{}, fmt.Errorf("cannot read Stack number: %w", err)
+		return tcommon.Stack{}, fmt.Errorf("cannot read Stack number: %w", err)
 	}
 
 	if id >= len(state.Stacks) {
-		return runner.Stack{}, fmt.Errorf("Invalid Stack number, it must be between 0 and %d", len(state.Stacks)-1)
+		return tcommon.Stack{}, fmt.Errorf("Invalid Stack number, it must be between 0 and %d", len(state.Stacks)-1)
 	}
 
 	return state.Stacks[id], nil
 }
 
-func generateEnvFile(stack runner.Stack) error {
+func generateEnvFile(stack tcommon.Stack) error {
 	fileExists := true
 	stat, err := os.Stat("./env.sh")
 	if err != nil {
@@ -2471,6 +2473,57 @@ func (Integration) TestOnRemote(ctx context.Context) error {
 	return nil
 }
 
+func (Integration) Buildkite() error {
+	goTestFlags := os.Getenv("GOTEST_FLAGS")
+	batches, err := define.DetermineBatches("testing/integration", goTestFlags, "integration")
+	if err != nil {
+		return fmt.Errorf("failed to determine batches: %w", err)
+	}
+	agentVersion, agentStackVersion, err := getTestRunnerVersions()
+	if err != nil {
+		return fmt.Errorf("failed to get agent versions: %w", err)
+	}
+	goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion()
+	if err != nil {
+		return fmt.Errorf("failed to get go versions: %w", err)
+	}
+
+	cfg := tcommon.Config{
+		AgentVersion: agentVersion,
+		StackVersion: agentStackVersion,
+		GOVersion:    goVersion,
+		Platforms:    testPlatforms(),
+		Packages:     testPackages(),
+		Groups:       testGroups(),
+		Matrix:       false,
+		VerboseMode:  mg.Verbose(),
+		TestFlags:    goTestFlags,
+	}
+
+	steps, err := buildkite.GenerateSteps(cfg, batches...)
+	if err != nil {
+		return fmt.Errorf("error generating buildkite steps: %w", err)
+	}
+
+	// write output to steps.yaml
+	cwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("error getting current working directory: %w", err)
+	}
+	ymlFilePath := filepath.Join(cwd, "steps.yml")
+	file, err := os.Create(ymlFilePath)
+	if err != nil {
+		return fmt.Errorf("error creating file: %w", err)
+	}
+	defer file.Close()
+	if _, err := file.WriteString(steps); err != nil {
+		return fmt.Errorf("error writing to file: %w", err)
+	}
+
+	fmt.Printf(">>> Generated buildkite steps written to: %s\n", ymlFilePath)
+	return nil
+}
+
 func integRunner(ctx context.Context, matrix bool, singleTest string) error {
 	if _, ok := ctx.Deadline(); !ok {
 		// If the context doesn't have a timeout (usually via the mage -t option), give it one.
@@ -2541,18 +2594,14 @@ func integRunnerOnce(ctx context.Context, matrix bool, singleTest string) (int,
 	return results.Failures, nil
 }
 
-func createTestRunner(matrix bool, singleTest string, goTestFlags string, batches ...define.Batch) (*runner.Runner, error) {
-	goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion()
-	if err != nil {
-		return nil, err
-	}
-
+func getTestRunnerVersions() (string, string, error) {
+	var err error
 	agentStackVersion := os.Getenv("AGENT_STACK_VERSION")
 	agentVersion := os.Getenv("AGENT_VERSION")
 	if agentVersion == "" {
 		agentVersion, err = mage.DefaultBeatBuildVariableSources.GetBeatVersion()
 		if err != nil {
-			return nil, err
+			return "", "", err
 		}
 		if agentStackVersion == "" {
 			// always use snapshot for stack version
@@ -2568,6 +2617,21 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 	if agentStackVersion == "" {
 		agentStackVersion = agentVersion
 	}
+
+	return agentVersion, agentStackVersion, nil
+}
+
+func createTestRunner(matrix bool, singleTest string, goTestFlags string, batches ...define.Batch) (*runner.Runner, error) {
+	goVersion, err := mage.DefaultBeatBuildVariableSources.GetGoVersion()
+	if err != nil {
+		return nil, err
+	}
+
+	agentVersion, agentStackVersion, err := getTestRunnerVersions()
+	if err != nil {
+		return nil, err
+	}
+
 	agentBuildDir := os.Getenv("AGENT_BUILD_DIR")
 	if agentBuildDir == "" {
 		agentBuildDir = filepath.Join("build", "distributions")
@@ -2606,7 +2670,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 		Datacenter:       datacenter,
 	}
 
-	var instanceProvisioner runner.InstanceProvisioner
+	var instanceProvisioner tcommon.InstanceProvisioner
 	instanceProvisionerMode := os.Getenv("INSTANCE_PROVISIONER")
 	switch instanceProvisionerMode {
 	case "", ogc.Name:
@@ -2619,7 +2683,6 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 	default:
 		return nil, fmt.Errorf("INSTANCE_PROVISIONER environment variable must be one of 'ogc' or 'multipass', not %s", instanceProvisionerMode)
 	}
-	fmt.Printf(">>>> Using %s instance provisioner\n", instanceProvisionerMode)
 
 	email, err := ogcCfg.ClientEmail()
 	if err != nil {
@@ -2632,7 +2695,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 		Region:     essRegion,
 	}
 
-	var stackProvisioner runner.StackProvisioner
+	var stackProvisioner tcommon.StackProvisioner
 	stackProvisionerMode := os.Getenv("STACK_PROVISIONER")
 	switch stackProvisionerMode {
 	case "", ess.ProvisionerStateful:
@@ -2654,7 +2717,6 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 			ess.ProvisionerServerless,
 			stackProvisionerMode)
 	}
-	fmt.Printf(">>>> Using %s stack provisioner\n", stackProvisionerMode)
 
 	timestamp := timestampEnabled()
 
@@ -2684,7 +2746,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 	diagDir := filepath.Join("build", "diagnostics")
 	_ = os.MkdirAll(diagDir, 0755)
 
-	cfg := runner.Config{
+	cfg := tcommon.Config{
 		AgentVersion:   agentVersion,
 		StackVersion:   agentStackVersion,
 		BuildDir:       agentBuildDir,