set SkipDaemonRestart in tryDelayEnroll (#4042) (#4052)

* set SkipDaemonRestart in tryDelayEnroll

this skips trying to restart the process which is currently broken
because the control daemon hasn't been started yet.  This restores the
previous behavior where the restart failing wasn't fatal.

* add integration test

(cherry picked from commit ba0ea0b)

Co-authored-by: Lee E Hinman <57081003+leehinman@users.noreply.github.com>

internal/pkg/agent/cmd/run.go

@@ -456,6 +456,10 @@ func tryDelayEnroll(ctx context.Context, logger *logger.Logger, cfg *configurati
 	}
 	options.DelayEnroll = false
 	options.FleetServer.SpawnAgent = false
+	// enrollCmd daemonReloadWithBackoff is broken
+	// see https://github.com/elastic/elastic-agent/issues/4043
+	// SkipDaemonRestart to true avoids running that code.
+	options.SkipDaemonRestart = true
 	c, err := newEnrollCmd(
 		ctx,
 		logger,

pkg/testing/fixture_install.go

@@ -58,6 +58,7 @@ type InstallOpts struct {
 	NonInteractive bool   // --non-interactive
 	ProxyURL       string // --proxy-url
 	Unprivileged   bool   // --unprivileged
+	DelayEnroll    bool   // --delay-enroll
 
 	EnrollOpts
 }
@@ -82,6 +83,9 @@ func (i InstallOpts) toCmdArgs() []string {
 	if i.Unprivileged {
 		args = append(args, "--unprivileged")
 	}
+	if i.DelayEnroll {
+		args = append(args, "--delay-enroll")
+	}
 
 	args = append(args, i.EnrollOpts.toCmdArgs()...)
 

pkg/testing/tools/tools.go

@@ -123,6 +123,11 @@ func InstallAgentForPolicy(ctx context.Context, t *testing.T,
 		timeout = time.Until(deadline)
 	}
 
+	// Don't check fleet status if --delay-enroll
+	if installOpts.DelayEnroll {
+		return nil
+	}
+
 	// Wait for Agent to be healthy
 	require.Eventually(
 		t,

testing/integration/delay_enroll_test.go

@@ -0,0 +1,86 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+//go:build integration
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-agent-libs/kibana"
+	atesting "github.com/elastic/elastic-agent/pkg/testing"
+	"github.com/elastic/elastic-agent/pkg/testing/define"
+	"github.com/elastic/elastic-agent/pkg/testing/tools"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/check"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/testcontext"
+)
+
+func TestDelayEnroll(t *testing.T) {
+	info := define.Require(t, define.Requirements{
+		Group: Fleet,
+		Stack: &define.Stack{},
+		Local: false,
+		Sudo:  true,
+		OS:    []define.OS{{Type: define.Linux}},
+	})
+
+	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
+	defer cancel()
+
+	agentFixture, err := define.NewFixture(t, define.Version())
+	require.NoError(t, err)
+
+	// 1. Create a policy in Fleet with monitoring enabled.
+	// To ensure there are no conflicts with previous test runs against
+	// the same ESS stack, we add the current time at the end of the policy
+	// name. This policy does not contain any integration.
+	t.Log("Enrolling agent in Fleet with a test policy")
+	createPolicyReq := kibana.AgentPolicy{
+		Name:        fmt.Sprintf("test-policy-enroll-%d", time.Now().Unix()),
+		Namespace:   info.Namespace,
+		Description: "test policy for agent enrollment",
+		MonitoringEnabled: []kibana.MonitoringEnabledOption{
+			kibana.MonitoringEnabledLogs,
+			kibana.MonitoringEnabledMetrics,
+		},
+		AgentFeatures: []map[string]interface{}{
+			{
+				"name":    "test_enroll",
+				"enabled": true,
+			},
+		},
+	}
+
+	installOpts := atesting.InstallOpts{
+		NonInteractive: true,
+		Force:          true,
+		DelayEnroll:    true,
+	}
+	// Install the Elastic-Agent with the policy that was just
+	// created.
+	_, err = tools.InstallAgentWithPolicy(
+		ctx,
+		t,
+		installOpts,
+		agentFixture,
+		info.KibanaClient,
+		createPolicyReq)
+	require.NoError(t, err)
+
+	// Start elastic-agent via service, this should do the enrollment
+	cmd := exec.Command("/usr/bin/systemctl", "start", "elastic-agent")
+	stdErrStdout, err := cmd.CombinedOutput()
+	require.NoErrorf(t, err, "systemctl start elastic-agent output was %s", stdErrStdout)
+
+	// check to make sure enroll worked
+	check.ConnectedToFleet(ctx, t, agentFixture, 5*time.Minute)
+
+}