[CI] Integration tests regrouping (#6349)

* TestRepeatedInstallUninstall optimisation * cleanup * cleanup * cleanup * cleanup * TestInstallDefendWithMTLSandEncCertKey to fleet-install-defend group * Extracted standalone-upgrade group * Commented iterations for TestRepeatedInstallUninstall * Reordered groups in pipeline matrix * Fix error message to only include agents on the same policy. --------- Co-authored-by: Denis <denis.rechkunov@elastic.co> Co-authored-by: Blake Rouse <blake.rouse@elastic.co>
elastic · Feb 13, 2025 · 37be67f · 37be67f
1 parent ed7c19f
commit 37be67f
Show file tree

Hide file tree

Showing 6 changed files with 60 additions and 35 deletions.
diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml
@@ -40,8 +40,11 @@ steps:
         matrix:
           - default
           - fleet
+          - fleet-endpoint-security
           - fleet-privileged
+          - standalone-upgrade
           - upgrade
+          - install-uninstall
 
       - label: "Win2022:non-sudo:{{matrix}}"
         command: |
@@ -91,15 +94,18 @@ steps:
           image: "family/platform-ingest-elastic-agent-ubuntu-2404"
         matrix:
           - default
-          - container
-          - fleet-upgrade-to-pr-build
           - upgrade
+          - standalone-upgrade
           - fleet
-          - fqdn
-          - deb
+          - fleet-endpoint-security
           - fleet-airgapped
-          - fleet-privileged
           - fleet-airgapped-privileged
+          - fleet-privileged
+          - fleet-upgrade-to-pr-build
+          - install-uninstall
+          - fqdn
+          - deb
+          - container
 
       - label: "arm:sudo: {{matrix}}"
         skip: true
@@ -115,15 +121,18 @@ steps:
           instanceType: "m6g.2xlarge"
         matrix:
           - default
-          - container
-          - fleet-upgrade-to-pr-build
           - upgrade
+          - standalone-upgrade
           - fleet
-          - fqdn
-          - deb
+          - fleet-endpoint-security
           - fleet-airgapped
-          - fleet-privileged
           - fleet-airgapped-privileged
+          - fleet-privileged
+          - fleet-upgrade-to-pr-build
+          - install-uninstall
+          - fqdn
+          - deb
+          - container
 
       - label: "arm:non-sudo: {{matrix}}"
         skip: true

diff --git a/pkg/testing/tools/fleettools/fleet.go b/pkg/testing/tools/fleettools/fleet.go
@@ -29,29 +29,29 @@ func GetAgentByPolicyIDAndHostnameFromList(ctx context.Context, client *kibana.C
 		return nil, err
 	}
 
-	var agentHostnames []string
-	hostnameAgents := make([]*kibana.AgentExisting, 0)
+	onPolicy := make([]string, 0, len(listAgentsResp.Items))
+	matching := make([]*kibana.AgentExisting, 0, 1)
 	for i, item := range listAgentsResp.Items {
 		agentHostname := item.LocalMetadata.Host.Hostname
 		agentPolicyID := item.PolicyID
-
-		agentHostnames = append(agentHostnames, agentHostname)
-
-		if strings.EqualFold(agentHostname, hostname) && agentPolicyID == policyID {
-			hostnameAgents = append(hostnameAgents, &listAgentsResp.Items[i])
+		if agentPolicyID == policyID {
+			onPolicy = append(onPolicy, agentHostname)
+			if strings.EqualFold(agentHostname, hostname) {
+				matching = append(matching, &listAgentsResp.Items[i])
+			}
 		}
 	}
 
-	if len(hostnameAgents) == 0 {
+	if len(matching) == 0 {
 		return nil, fmt.Errorf("unable to find agent with hostname [%s] for policy [%s]. Found: %v",
-			hostname, policyID, agentHostnames)
+			hostname, policyID, onPolicy)
 	}
 
-	if len(hostnameAgents) > 1 {
-		return nil, fmt.Errorf("found %d agents with hostname [%s]; expected to find only one", len(hostnameAgents), hostname)
+	if len(matching) > 1 {
+		return nil, fmt.Errorf("found %d agents with hostname [%s]; expected to find only one", len(matching), hostname)
 	}
 
-	return hostnameAgents[0], nil
+	return matching[0], nil
 }
 
 func GetAgentIDByHostname(ctx context.Context, client *kibana.Client, policyID, hostname string) (string, error) {

diff --git a/testing/integration/endpoint_security_test.go b/testing/integration/endpoint_security_test.go
@@ -69,7 +69,7 @@ var protectionTests = []struct {
 // test automatically.
 func TestInstallAndCLIUninstallWithEndpointSecurity(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Fleet,
+		Group: FleetEndpointSecurity,
 		Stack: &define.Stack{},
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
@@ -96,7 +96,7 @@ func TestInstallAndCLIUninstallWithEndpointSecurity(t *testing.T) {
 // but at this point endpoint is already uninstalled.
 func TestInstallAndUnenrollWithEndpointSecurity(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Fleet,
+		Group: FleetEndpointSecurity,
 		Stack: &define.Stack{},
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
@@ -124,7 +124,7 @@ func TestInstallAndUnenrollWithEndpointSecurity(t *testing.T) {
 // but at this point endpoint should be already uninstalled.
 func TestInstallWithEndpointSecurityAndRemoveEndpointIntegration(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Fleet,
+		Group: FleetEndpointSecurity,
 		Stack: &define.Stack{},
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
@@ -399,7 +399,7 @@ type agentPolicyUpdateRequest struct {
 // path other than default
 func TestEndpointSecurityNonDefaultBasePath(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Fleet,
+		Group: FleetEndpointSecurity,
 		Stack: &define.Stack{},
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
@@ -470,7 +470,7 @@ func TestEndpointSecurityNonDefaultBasePath(t *testing.T) {
 // Tests that install of Elastic Defend fails if Agent is installed unprivileged.
 func TestEndpointSecurityUnprivileged(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Fleet,
+		Group: FleetEndpointSecurity,
 		Stack: &define.Stack{},
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
@@ -550,7 +550,7 @@ func TestEndpointSecurityUnprivileged(t *testing.T) {
 // Tests that trying to switch from privileged to unprivileged with Elastic Defend fails.
 func TestEndpointSecurityCannotSwitchToUnprivileged(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Fleet,
+		Group: FleetEndpointSecurity,
 		Stack: &define.Stack{},
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
@@ -620,7 +620,7 @@ func TestEndpointSecurityCannotSwitchToUnprivileged(t *testing.T) {
 // TestEndpointLogsAreCollectedInDiagnostics tests that diagnostics archive contain endpoint logs
 func TestEndpointLogsAreCollectedInDiagnostics(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Fleet,
+		Group: FleetEndpointSecurity,
 		Stack: &define.Stack{},
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
@@ -838,7 +838,7 @@ func agentIsHealthyNoEndpoint(t *testing.T, ctx context.Context, agentClient cli
 // when an installed agent is running a policy with tamper protection enabled fails.
 func TestForceInstallOverProtectedPolicy(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Fleet,
+		Group: FleetEndpointSecurity,
 		Stack: &define.Stack{},
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
@@ -904,7 +904,7 @@ func TestForceInstallOverProtectedPolicy(t *testing.T) {
 
 func TestInstallDefendWithMTLSandEncCertKey(t *testing.T) {
 	stack := define.Require(t, define.Requirements{
-		Group: Fleet,
+		Group: FleetEndpointSecurity,
 		Stack: &define.Stack{},
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation

diff --git a/testing/integration/groups_test.go b/testing/integration/groups_test.go
@@ -35,9 +35,18 @@ const (
 	// Upgrade group of tests. Used for testing upgrades.
 	Upgrade = "upgrade"
 
+	// StandaloneUpgrade group of tests. Used for TestStandaloneUpgrade.
+	StandaloneUpgrade = "standalone-upgrade"
+
 	// Deb group of tests. Used for testing .deb packages install & upgrades
 	Deb = "deb"
 
 	// RPM group of tests. Used for testing .rpm packages install & upgrades
 	RPM = "rpm"
+
+	// InstallUninstall group of tests. Used for testing repeated install & uninstall scenarios
+	InstallUninstall = "install-uninstall"
+
+	// FleetInstall group of tests. Used for the long running fleet-related "TestInstall..." tests.
+	FleetEndpointSecurity = "fleet-endpoint-security"
 )
diff --git a/testing/integration/install_test.go b/testing/integration/install_test.go
@@ -642,7 +642,7 @@ func testUninstallAuditUnenroll(ctx context.Context, fixture *atesting.Fixture,
 // into Fleet. Current testing shows each iteration takes around 16 seconds.
 func TestRepeatedInstallUninstallFleet(t *testing.T) {
 	info := define.Require(t, define.Requirements{
-		Group: Fleet,
+		Group: InstallUninstall,
 		Stack: &define.Stack{}, // needs a fleet-server.
 		// We require sudo for this test to run
 		// `elastic-agent install` (even though it will
@@ -673,8 +673,7 @@ func TestRepeatedInstallUninstallFleet(t *testing.T) {
 	require.NoError(t, err)
 
 	maxRunTime := 2 * time.Minute
-	iterations := 100
-	for i := 0; i < iterations; i++ {
+	for i := 0; i < iterations(); i++ {
 		successful := t.Run(fmt.Sprintf("%s-%d", t.Name(), i), func(t *testing.T) {
 			ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(maxRunTime))
 			defer cancel()
@@ -714,6 +713,14 @@ func TestRepeatedInstallUninstallFleet(t *testing.T) {
 	}
 }
 
+func iterations() int {
+	// If running in CI, reduce the number of iterations to speed up the test.
+	if os.Getenv("BUILDKITE_PULL_REQUEST") != "" {
+		return 50
+	}
+	return 100
+}
+
 func randStr(length int) string {
 	letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
 

diff --git a/testing/integration/upgrade_standalone_test.go b/testing/integration/upgrade_standalone_test.go
@@ -24,7 +24,7 @@ import (
 
 func TestStandaloneUpgrade(t *testing.T) {
 	define.Require(t, define.Requirements{
-		Group: Upgrade,
+		Group: StandaloneUpgrade,
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
 	})