From ddd832b4c3a16296ff6d0b2693629de42a375446 Mon Sep 17 00:00:00 2001 From: Lee E Hinman <57081003+leehinman@users.noreply.github.com> Date: Tue, 19 Mar 2024 18:58:16 -0500 Subject: [PATCH] integration deb rpm (#4424) * add a DEB integration test * fix package building of rpm and deb packages * fix rebase conflicts * add deb,rpm to serverless buildkite * goimports -local --- .buildkite/scripts/steps/integration_tests.sh | 2 +- dev-tools/mage/pkg.go | 9 +- pkg/testing/define/define.go | 2 +- pkg/testing/fetch_test.go | 39 ++++++ pkg/testing/fetcher.go | 28 ++-- pkg/testing/fetcher_artifact.go | 4 +- pkg/testing/fetcher_artifact_test.go | 8 +- pkg/testing/fetcher_http.go | 4 +- pkg/testing/fetcher_http_test.go | 5 +- pkg/testing/fetcher_local.go | 4 +- pkg/testing/fetcher_local_test.go | 12 +- pkg/testing/fixture.go | 29 ++++- pkg/testing/fixture_install.go | 122 +++++++++++++++++- pkg/testing/runner/debian.go | 102 ++++++++------- pkg/testing/runner/runner.go | 70 ++++++---- pkg/testing/runner/windows.go | 86 ++++++------ testing/integration/groups_test.go | 3 + testing/integration/logs_ingestion_test.go | 69 ++++++++++ testing/integration/upgrade_fleet_test.go | 6 +- 19 files changed, 454 insertions(+), 150 deletions(-) create mode 100644 pkg/testing/fetch_test.go diff --git a/.buildkite/scripts/steps/integration_tests.sh b/.buildkite/scripts/steps/integration_tests.sh index 2a129193267..4143e2413e5 100755 --- a/.buildkite/scripts/steps/integration_tests.sh +++ b/.buildkite/scripts/steps/integration_tests.sh @@ -19,7 +19,7 @@ else OVERRIDE_TEST_AGENT_VERSION="" fi # PACKAGE -AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip mage package +AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip,rpm,deb mage package # Run integration tests set +e diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go index e00bbbdb661..b13e3d7390f 100644 --- a/dev-tools/mage/pkg.go +++ b/dev-tools/mage/pkg.go @@ -34,7 +34,7 @@ func Package() error { // platforms := updateWithDarwinUniversal(Platforms) platforms := Platforms - var tasks []interface{} + tasks := make(map[string][]interface{}) for _, target := range platforms { for _, pkg := range Packages { if pkg.OS != target.GOOS() || pkg.Arch != "" && pkg.Arch != target.Arch() { @@ -94,12 +94,15 @@ func Package() error { spec = spec.Evaluate() - tasks = append(tasks, packageBuilder{target, spec, pkgType}.Build) + tasks[target.GOOS()+"-"+target.Arch()] = append(tasks[target.GOOS()+"-"+target.Arch()], packageBuilder{target, spec, pkgType}.Build) } } } - Parallel(tasks...) + for k, v := range tasks { + fmt.Printf(">> package: Building %s\n", k) + Parallel(v...) + } return nil } diff --git a/pkg/testing/define/define.go b/pkg/testing/define/define.go index 72105caa461..20eac6c8f85 100644 --- a/pkg/testing/define/define.go +++ b/pkg/testing/define/define.go @@ -151,7 +151,7 @@ func runOrSkip(t *testing.T, req Requirements, local bool) *Info { panic("failed to get OS information") } if !req.runtimeAllowed(runtime.GOOS, runtime.GOARCH, osInfo.Version, osInfo.Platform) { - t.Skip("platform, architecture, version, and distro not supported by test") + t.Skipf("platform: %s, architecture: %s, version: %s, and distro: %s combination is not supported by test. required: %v", runtime.GOOS, runtime.GOARCH, osInfo.Version, osInfo.Platform, req.OS) return nil } namespace, err := getNamespace(t, local) diff --git a/pkg/testing/fetch_test.go b/pkg/testing/fetch_test.go new file mode 100644 index 00000000000..bb20c718f21 --- /dev/null +++ b/pkg/testing/fetch_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package testing + +import ( + "errors" + gtesting "testing" +) + +func TestGetPackageSuffix(t *gtesting.T) { + tests := map[string]struct { + os string + arch string + format string + expected string + err error + }{ + "windows zip": {os: "windows", arch: "amd64", format: "zip", expected: "windows-x86_64.zip", err: nil}, + "windows msi": {os: "windows", arch: "amd64", format: "msi", expected: "", err: ErrUnsupportedPlatform}, + "linux deb": {os: "linux", arch: "amd64", format: "deb", expected: "amd64.deb", err: nil}, + "linux rpm": {os: "linux", arch: "amd64", format: "rpm", expected: "x86_64.rpm", err: nil}, + "linux tar.gz": {os: "linux", arch: "amd64", format: "targz", expected: "linux-x86_64.tar.gz", err: nil}, + "linux pkg.tar.zst": {os: "linux", arch: "amd64", format: "pkg.tar.zst", expected: "", err: ErrUnsupportedPlatform}, + "darwin arm64": {os: "darwin", arch: "arm64", format: "targz", expected: "darwin-aarch64.tar.gz", err: nil}, + } + for name, tc := range tests { + t.Run(name, func(t *gtesting.T) { + got, err := GetPackageSuffix(tc.os, tc.arch, tc.format) + if !errors.Is(err, tc.err) { + t.Fatalf("wrong error. expected: %v got: %v", tc.err, err) + } + if got != tc.expected { + t.Fatalf("wrong output. expected: %s got: %s", tc.expected, got) + } + }) + } +} diff --git a/pkg/testing/fetcher.go b/pkg/testing/fetcher.go index baa734495f9..efd2ce5c641 100644 --- a/pkg/testing/fetcher.go +++ b/pkg/testing/fetcher.go @@ -31,19 +31,23 @@ var ( // packageArchMap provides a mapping for the endings of the builds of Elastic Agent based on the // operating system and architecture. var packageArchMap = map[string]string{ - "linux-amd64": "linux-x86_64.tar.gz", - "linux-arm64": "linux-arm64.tar.gz", - "windows-amd64": "windows-x86_64.zip", - "darwin-amd64": "darwin-x86_64.tar.gz", - "darwin-arm64": "darwin-aarch64.tar.gz", + "linux-amd64-targz": "linux-x86_64.tar.gz", + "linux-amd64-deb": "amd64.deb", + "linux-amd64-rpm": "x86_64.rpm", + "linux-arm64-targz": "linux-arm64.tar.gz", + "linux-arm64-deb": "arm64.deb", + "linux-arm64-rpm": "aarch64.rpm", + "windows-amd64-zip": "windows-x86_64.zip", + "darwin-amd64-targz": "darwin-x86_64.tar.gz", + "darwin-arm64-targz": "darwin-aarch64.tar.gz", } // GetPackageSuffix returns the suffix ending for the builds of Elastic Agent based on the // operating system and architecture. -func GetPackageSuffix(operatingSystem string, architecture string) (string, error) { - suffix, ok := packageArchMap[fmt.Sprintf("%s-%s", operatingSystem, architecture)] +func GetPackageSuffix(operatingSystem string, architecture string, packageFormat string) (string, error) { + suffix, ok := packageArchMap[fmt.Sprintf("%s-%s-%s", operatingSystem, architecture, packageFormat)] if !ok { - return "", fmt.Errorf("%w: %s/%s", ErrUnsupportedPlatform, operatingSystem, architecture) + return "", fmt.Errorf("%w: %s/%s/%s", ErrUnsupportedPlatform, operatingSystem, architecture, packageFormat) } return suffix, nil } @@ -68,7 +72,7 @@ type Fetcher interface { // // The extraction is handled by the caller. This should only download the file // and place it into the directory. - Fetch(ctx context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) + Fetch(ctx context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) } // fetchCache is global to all tests, reducing the time required to fetch the needed artifacts @@ -105,6 +109,12 @@ func splitFileType(name string) (string, string, error) { if strings.HasSuffix(name, ".zip") { return strings.TrimSuffix(name, ".zip"), ".zip", nil } + if strings.HasSuffix(name, ".deb") { + return strings.TrimSuffix(name, ".deb"), ".deb", nil + } + if strings.HasSuffix(name, ".rpm") { + return strings.TrimSuffix(name, ".rpm"), ".rpm", nil + } return "", "", fmt.Errorf("unknown file extension type: %s", filepath.Ext(name)) } diff --git a/pkg/testing/fetcher_artifact.go b/pkg/testing/fetcher_artifact.go index ec99477182c..3f3c1ec8ce8 100644 --- a/pkg/testing/fetcher_artifact.go +++ b/pkg/testing/fetcher_artifact.go @@ -59,8 +59,8 @@ func (f *artifactFetcher) Name() string { } // Fetch fetches the Elastic Agent and places the resulting binary at the path. -func (f *artifactFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) { - suffix, err := GetPackageSuffix(operatingSystem, architecture) +func (f *artifactFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) { + suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat) if err != nil { return nil, err } diff --git a/pkg/testing/fetcher_artifact_test.go b/pkg/testing/fetcher_artifact_test.go index 42d9585c606..f2f0aae51e2 100644 --- a/pkg/testing/fetcher_artifact_test.go +++ b/pkg/testing/fetcher_artifact_test.go @@ -29,7 +29,7 @@ func TestArtifactFetcher_Default(t *testing.T) { af.doer = newFakeHttpClient(t) tmp := t.TempDir() - res, err := f.Fetch(context.Background(), "linux", "amd64", "8.12.0") + res, err := f.Fetch(context.Background(), "linux", "amd64", "8.12.0", "targz") require.NoError(t, err) err = res.Fetch(context.Background(), t, tmp) @@ -46,7 +46,7 @@ func TestArtifactFetcher_Snapshot(t *testing.T) { af.doer = newFakeHttpClient(t) tmp := t.TempDir() - res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT") + res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT", "targz") require.NoError(t, err) err = res.Fetch(context.Background(), t, tmp) @@ -64,7 +64,7 @@ func TestArtifactFetcher_SnapshotOnly(t *testing.T) { af.doer = newFakeHttpClient(t) tmp := t.TempDir() - res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0") + res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0", "targz") require.NoError(t, err) err = res.Fetch(context.Background(), t, tmp) @@ -82,7 +82,7 @@ func TestArtifactFetcher_Build(t *testing.T) { af.doer = newFakeHttpClient(t) tmp := t.TempDir() - res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT+l5snflwr") + res, err := f.Fetch(context.Background(), "linux", "amd64", "8.13.0-SNAPSHOT+l5snflwr", "targz") require.NoError(t, err) err = res.Fetch(context.Background(), t, tmp) diff --git a/pkg/testing/fetcher_http.go b/pkg/testing/fetcher_http.go index b56086b022e..5a30a6e4e86 100644 --- a/pkg/testing/fetcher_http.go +++ b/pkg/testing/fetcher_http.go @@ -45,8 +45,8 @@ func (h HttpFetcher) Name() string { return fmt.Sprintf("httpFetcher-%s", sanitizeFetcherName(h.baseURL)) } -func (h HttpFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) { - suffix, err := GetPackageSuffix(operatingSystem, architecture) +func (h HttpFetcher) Fetch(ctx context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) { + suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat) if err != nil { return nil, err } diff --git a/pkg/testing/fetcher_http_test.go b/pkg/testing/fetcher_http_test.go index 72f2d9c0a1f..869cd57095c 100644 --- a/pkg/testing/fetcher_http_test.go +++ b/pkg/testing/fetcher_http_test.go @@ -24,6 +24,7 @@ func TestHttpFetcher_Fetch(t *testing.T) { operatingSystem string architecture string version string + pkgFormat string } tests := []struct { name string @@ -38,6 +39,7 @@ func TestHttpFetcher_Fetch(t *testing.T) { operatingSystem: "linux", architecture: "arm64", version: "1.2.3", + pkgFormat: "targz", }, want: &httpFetcherResult{ baseURL: "https://artifacts.elastic.co/downloads/beats/elastic-agent/", @@ -52,6 +54,7 @@ func TestHttpFetcher_Fetch(t *testing.T) { operatingSystem: "windows", architecture: "amd64", version: "1.2.3", + pkgFormat: "zip", }, want: &httpFetcherResult{ baseURL: "http://somehost.somedomain/some/path/here", @@ -69,7 +72,7 @@ func TestHttpFetcher_Fetch(t *testing.T) { h := NewHttpFetcher(opts...) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - got, err := h.Fetch(ctx, tt.args.operatingSystem, tt.args.architecture, tt.args.version) + got, err := h.Fetch(ctx, tt.args.operatingSystem, tt.args.architecture, tt.args.version, tt.args.pkgFormat) if !tt.wantErr(t, err, fmt.Sprintf("Fetch(%v, %v, %v, %v)", ctx, tt.args.operatingSystem, tt.args.architecture, tt.args.version)) { return } diff --git a/pkg/testing/fetcher_local.go b/pkg/testing/fetcher_local.go index 4a8e4993d13..4962283e2cd 100644 --- a/pkg/testing/fetcher_local.go +++ b/pkg/testing/fetcher_local.go @@ -55,8 +55,8 @@ func (f *localFetcher) Name() string { } // Fetch fetches the Elastic Agent and places the resulting binary at the path. -func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architecture string, version string) (FetcherResult, error) { - suffix, err := GetPackageSuffix(operatingSystem, architecture) +func (f *localFetcher) Fetch(_ context.Context, operatingSystem string, architecture string, version string, packageFormat string) (FetcherResult, error) { + suffix, err := GetPackageSuffix(operatingSystem, architecture, packageFormat) if err != nil { return nil, err } diff --git a/pkg/testing/fetcher_local_test.go b/pkg/testing/fetcher_local_test.go index 78f2bd5b02c..1223b84ebb7 100644 --- a/pkg/testing/fetcher_local_test.go +++ b/pkg/testing/fetcher_local_test.go @@ -28,9 +28,13 @@ func TestLocalFetcher(t *testing.T) { snapshotContentHash := []byte("snapshot contents hash") noSnapshotContent := []byte("not snapshot contents") noSnapshotContentHash := []byte("not snapshot contents hash") + pkgFormat := "targz" + if runtime.GOOS == "windows" { + pkgFormat = "zip" + } testdata := t.TempDir() - suffix, err := GetPackageSuffix(runtime.GOOS, runtime.GOARCH) + suffix, err := GetPackageSuffix(runtime.GOOS, runtime.GOARCH, pkgFormat) require.NoError(t, err) snapshotPath := fmt.Sprintf("elastic-agent-%s-SNAPSHOT-%s", baseVersion, suffix) @@ -92,8 +96,12 @@ func TestLocalFetcher(t *testing.T) { tmp := t.TempDir() f := LocalFetcher(testdata, tc.opts...) + pkgFormat := "targz" + if runtime.GOOS == "windows" { + pkgFormat = "zip" + } got, err := f.Fetch( - context.Background(), runtime.GOOS, runtime.GOARCH, tc.version) + context.Background(), runtime.GOOS, runtime.GOARCH, tc.version, pkgFormat) require.NoError(t, err) err = got.Fetch(context.Background(), t, tmp) diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go index f4990f591c0..1f74eac5ad2 100644 --- a/pkg/testing/fixture.go +++ b/pkg/testing/fixture.go @@ -41,6 +41,7 @@ type Fixture struct { fetcher Fetcher operatingSystem string architecture string + packageFormat string logOutput bool allowErrs bool connectTimout time.Duration @@ -83,6 +84,14 @@ func WithOSArchitecture(operatingSystem string, architecture string) FixtureOpt } } +// WithPackageFormat changes the package format to use for the fixture. +// By default, targz is picked except for windows which uses zip +func WithPackageFormat(packageFormat string) FixtureOpt { + return func(f *Fixture) { + f.packageFormat = packageFormat + } +} + // WithLogOutput instructs the fixture to log all Elastic Agent output to the test log. // By default, the Elastic Agent output will not be logged to the test logger. func WithLogOutput() FixtureOpt { @@ -139,6 +148,10 @@ func NewFixture(t *testing.T, version string, opts ...FixtureOpt) (*Fixture, err if !ok { return nil, errors.New("unable to determine callers file path") } + pkgFormat := "targz" + if runtime.GOOS == "windows" { + pkgFormat = "zip" + } f := &Fixture{ t: t, version: version, @@ -146,6 +159,7 @@ func NewFixture(t *testing.T, version string, opts ...FixtureOpt) (*Fixture, err fetcher: ArtifactFetcher(), operatingSystem: runtime.GOOS, architecture: runtime.GOARCH, + packageFormat: pkgFormat, connectTimout: 15 * time.Second, // default to elastic-agent, can be changed by a set FixtureOpt below binaryName: "elastic-agent", @@ -253,6 +267,11 @@ func (f *Fixture) SrcPackage(ctx context.Context) (string, error) { return f.srcPackage, nil } +// PackageFormat returns the package format for the fixture +func (f *Fixture) PackageFormat() string { + return f.packageFormat +} + func ExtractArtifact(l Logger, artifactFile, outputDir string) error { filename := filepath.Base(artifactFile) _, ext, err := splitFileType(filename) @@ -271,6 +290,11 @@ func ExtractArtifact(l Logger, artifactFile, outputDir string) error { if err != nil { return fmt.Errorf("failed to unzip %s: %w", artifactFile, err) } + case ".deb", "rpm": + err := copy.Copy(artifactFile, filepath.Join(outputDir, filepath.Base(artifactFile))) + if err != nil { + return fmt.Errorf("failed to copy %s to %s: %w", artifactFile, outputDir, err) + } } l.Logf("Completed extraction of artifact %s to %s", filename, outputDir) return nil @@ -813,6 +837,9 @@ func (f *Fixture) binaryPath() string { workDir = filepath.Join(paths.DefaultBasePath, "Elastic", "Agent") } } + if f.packageFormat == "deb" { + workDir = "/usr/bin" + } defaultBin := "elastic-agent" if f.binaryName != "" { defaultBin = f.binaryName @@ -840,7 +867,7 @@ func (f *Fixture) fetch(ctx context.Context) (string, error) { cache.dir = dir } - res, err := f.fetcher.Fetch(ctx, f.operatingSystem, f.architecture, f.version) + res, err := f.fetcher.Fetch(ctx, f.operatingSystem, f.architecture, f.version, f.packageFormat) if err != nil { return "", err } diff --git a/pkg/testing/fixture_install.go b/pkg/testing/fixture_install.go index 651a1eac573..23edbde1ebe 100644 --- a/pkg/testing/fixture_install.go +++ b/pkg/testing/fixture_install.go @@ -13,6 +13,7 @@ import ( "io" "io/fs" "os" + "os/exec" "path/filepath" "runtime" "strconv" @@ -132,11 +133,32 @@ func (f *Fixture) Install(ctx context.Context, installOpts *InstallOpts, opts .. // check for running agents before installing, but proceed anyway assert.Empty(f.t, getElasticAgentProcesses(f.t), "there should be no running agent at beginning of Install()") - installArgs := []string{"install"} + switch f.packageFormat { + case "targz", "zip": + return f.installNoPkgManager(ctx, installOpts, opts) + case "deb": + return f.installDeb(ctx, installOpts, opts) + default: + return nil, fmt.Errorf("package format %s isn't supported yet", f.packageFormat) + } +} + +// installNoPkgManager installs the prepared Elastic Agent binary from +// the tgz or zip archive and registers a t.Cleanup function to +// uninstall the agent if it hasn't been uninstalled. It also takes +// care of collecting a diagnostics when AGENT_COLLECT_DIAG=true or +// the test has failed. +// It returns: +// - the combined output of Install command stdout and stderr +// - an error if any. +func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallOpts, opts []process.CmdOption) ([]byte, error) { + f.t.Logf("[test %s] Inside fixture installNoPkgManager function", f.t.Name()) if installOpts == nil { // default options when not provided installOpts = &InstallOpts{} } + + installArgs := []string{"install"} installOptsArgs, err := installOpts.toCmdArgs(f.operatingSystem) if err != nil { return nil, err @@ -339,6 +361,80 @@ func getProcesses(t *gotesting.T, regex string) []runningProcess { return processes } +// installDeb installs the prepared Elastic Agent binary from the deb +// package and registers a t.Cleanup function to uninstall the agent if +// it hasn't been uninstalled. It also takes care of collecting a +// diagnostics when AGENT_COLLECT_DIAG=true or the test has failed. +// It returns: +// - the combined output of Install command stdout and stderr +// - an error if any. +func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, opts []process.CmdOption) ([]byte, error) { + f.t.Logf("[test %s] Inside fixture installDeb function", f.t.Name()) + //Prepare so that the f.srcPackage string is populated + err := f.EnsurePrepared(ctx) + if err != nil { + return nil, fmt.Errorf("failed to prepare: %w", err) + } + + // sudo apt install the deb + out, err := exec.CommandContext(ctx, "sudo", "apt", "install", f.srcPackage).CombinedOutput() // #nosec G204 -- Need to pass in name of package + if err != nil { + return out, fmt.Errorf("apt install failed: %w output:%s", err, string(out)) + } + + f.t.Cleanup(func() { + f.t.Logf("[test %s] Inside fixture installDeb cleanup function", f.t.Name()) + uninstallCtx, uninstallCancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer uninstallCancel() + // stop elastic-agent, non fatal if error, might have been stopped before this. + f.t.Logf("running 'sudo systemctl stop elastic-agent'") + out, err := exec.CommandContext(uninstallCtx, "sudo", "systemctl", "stop", "elastic-agent").CombinedOutput() + if err != nil { + f.t.Logf("error systemctl stop elastic-agent: %s, output: %s", err, string(out)) + } + // apt-get purge elastic-agent + f.t.Logf("running 'sudo apt-get -y -q purge elastic-agent'") + out, err = exec.CommandContext(uninstallCtx, "sudo", "apt-get", "-y", "-q", "purge", "elastic-agent").CombinedOutput() + if err != nil { + f.t.Logf("failed to apt-get purge elastic-agent: %s, output: %s", err, string(out)) + f.t.FailNow() + } + }) + + // start elastic-agent + out, err = exec.CommandContext(ctx, "sudo", "systemctl", "start", "elastic-agent").CombinedOutput() + if err != nil { + return out, fmt.Errorf("systemctl start elastic-agent failed: %w", err) + } + + // apt install doesn't enroll, so need to do that + enrollArgs := []string{"elastic-agent", "enroll"} + if installOpts.Force { + enrollArgs = append(enrollArgs, "--force") + } + if installOpts.Insecure { + enrollArgs = append(enrollArgs, "--insecure") + } + if installOpts.ProxyURL != "" { + enrollArgs = append(enrollArgs, "--proxy-url="+installOpts.ProxyURL) + } + if installOpts.DelayEnroll { + enrollArgs = append(enrollArgs, "--delay-enroll") + } + if installOpts.EnrollOpts.URL != "" { + enrollArgs = append(enrollArgs, "--url", installOpts.EnrollOpts.URL) + } + if installOpts.EnrollOpts.EnrollmentToken != "" { + enrollArgs = append(enrollArgs, "--enrollment-token", installOpts.EnrollOpts.EnrollmentToken) + } + out, err = exec.CommandContext(ctx, "sudo", enrollArgs...).CombinedOutput() + if err != nil { + return out, fmt.Errorf("elastic-agent enroll failed: %w, output: %s args: %v", err, string(out), enrollArgs) + } + + return nil, nil +} + type UninstallOpts struct { Force bool // --force UninstallToken string @@ -359,6 +455,30 @@ func (i UninstallOpts) toCmdArgs() []string { // Uninstall uninstalls the installed Elastic Agent binary func (f *Fixture) Uninstall(ctx context.Context, uninstallOpts *UninstallOpts, opts ...process.CmdOption) ([]byte, error) { + switch f.packageFormat { + case "targz", "zip": + return f.uninstallNoPkgManager(ctx, uninstallOpts, opts) + case "deb": + return f.uninstallDeb(ctx, uninstallOpts, opts) + default: + return nil, fmt.Errorf("uninstall of package format '%s' not supported yet", f.packageFormat) + } +} + +func (f *Fixture) uninstallDeb(ctx context.Context, uninstallOpts *UninstallOpts, opts []process.CmdOption) ([]byte, error) { + // stop elastic-agent, non fatal if error, might have been stopped before this. + out, err := exec.CommandContext(ctx, "sudo", "systemctl", "stop", "elastic-agent").CombinedOutput() + if err != nil { + f.t.Logf("error systemctl stop elastic-agent: %s, output: %s", err, string(out)) + } + out, err = exec.CommandContext(ctx, "sudo", "apt-get", "-y", "-q", "purge", "elastic-agent").CombinedOutput() + if err != nil { + return out, fmt.Errorf("error removing apt: %w", err) + } + return out, nil +} + +func (f *Fixture) uninstallNoPkgManager(ctx context.Context, uninstallOpts *UninstallOpts, opts []process.CmdOption) ([]byte, error) { if !f.installed { return nil, ErrNotInstalled } diff --git a/pkg/testing/runner/debian.go b/pkg/testing/runner/debian.go index 268d42fb604..a3995efd602 100644 --- a/pkg/testing/runner/debian.go +++ b/pkg/testing/runner/debian.go @@ -87,7 +87,7 @@ func (DebianRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Log } // Copy places the required files on the host. -func (DebianRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, build Build) error { +func (DebianRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error { // copy the archive and extract it on the host logger.Logf("Copying repo") destRepoName := filepath.Base(repoArchive) @@ -97,12 +97,14 @@ func (DebianRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger } // remove build paths, on cases where the build path is different from agent. - for _, remoteBuildPath := range []string{build.Path, build.SHA512Path} { - relativeAgentDir := filepath.Join("agent", remoteBuildPath) - _, _, err := sshClient.Exec(ctx, "sudo", []string{"rm", "-rf", relativeAgentDir}, nil) - // doesn't need to be a fatal error. - if err != nil { - logger.Logf("error removing build dir %s: %w", relativeAgentDir, err) + for _, build := range builds { + for _, remoteBuildPath := range []string{build.Path, build.SHA512Path} { + relativeAgentDir := filepath.Join("agent", remoteBuildPath) + _, _, err := sshClient.Exec(ctx, "sudo", []string{"rm", "-rf", relativeAgentDir}, nil) + // doesn't need to be a fatal error. + if err != nil { + logger.Logf("error removing build dir %s: %w", relativeAgentDir, err) + } } } @@ -132,56 +134,58 @@ func (DebianRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger // determine if the build needs to be replaced on the host // if it already exists and the SHA512 are the same contents, then // there is no reason to waste time uploading the build - copyBuild := true - localSHA512, err := os.ReadFile(build.SHA512Path) - if err != nil { - return fmt.Errorf("failed to read local SHA52 contents %s: %w", build.SHA512Path, err) - } - hostSHA512Path := filepath.Base(build.SHA512Path) - hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path) - if err == nil { - if string(localSHA512) == string(hostSHA512) { - logger.Logf("Skipping copy agent build %s; already the same", filepath.Base(build.Path)) - copyBuild = false - } - } - - if copyBuild { - // ensure the existing copies are removed first - toRemove := filepath.Base(build.Path) - stdOut, errOut, err = sshClient.Exec(ctx, - "sudo", []string{"rm", "-f", toRemove}, nil) + for _, build := range builds { + copyBuild := true + localSHA512, err := os.ReadFile(build.SHA512Path) if err != nil { - return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", - toRemove, err, stdOut, errOut) + return fmt.Errorf("failed to read local SHA52 contents %s: %w", build.SHA512Path, err) } - - toRemove = filepath.Base(build.SHA512Path) - stdOut, errOut, err = sshClient.Exec(ctx, - "sudo", []string{"rm", "-f", toRemove}, nil) - if err != nil { - return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", - toRemove, err, stdOut, errOut) + hostSHA512Path := filepath.Base(build.SHA512Path) + hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path) + if err == nil { + if string(localSHA512) == string(hostSHA512) { + logger.Logf("Skipping copy agent build %s; already the same", filepath.Base(build.Path)) + copyBuild = false + } } - logger.Logf("Copying agent build %s", filepath.Base(build.Path)) - } - - for _, buildPath := range []string{build.Path, build.SHA512Path} { if copyBuild { - err = sshClient.Copy(buildPath, filepath.Base(buildPath)) + // ensure the existing copies are removed first + toRemove := filepath.Base(build.Path) + stdOut, errOut, err = sshClient.Exec(ctx, + "sudo", []string{"rm", "-f", toRemove}, nil) if err != nil { - return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err) + return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", + toRemove, err, stdOut, errOut) } + + toRemove = filepath.Base(build.SHA512Path) + stdOut, errOut, err = sshClient.Exec(ctx, + "sudo", []string{"rm", "-f", toRemove}, nil) + if err != nil { + return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", + toRemove, err, stdOut, errOut) + } + + logger.Logf("Copying agent build %s", filepath.Base(build.Path)) } - insideAgentDir := filepath.Join("agent", buildPath) - stdOut, errOut, err = sshClient.Exec(ctx, "mkdir", []string{"-p", filepath.Dir(insideAgentDir)}, nil) - if err != nil { - return fmt.Errorf("failed to create %s directory: %w (stdout: %s, stderr: %s)", filepath.Dir(insideAgentDir), err, stdOut, errOut) - } - stdOut, errOut, err = sshClient.Exec(ctx, "ln", []string{filepath.Base(buildPath), insideAgentDir}, nil) - if err != nil { - return fmt.Errorf("failed to hard link %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), insideAgentDir, err, stdOut, errOut) + + for _, buildPath := range []string{build.Path, build.SHA512Path} { + if copyBuild { + err = sshClient.Copy(buildPath, filepath.Base(buildPath)) + if err != nil { + return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err) + } + } + insideAgentDir := filepath.Join("agent", buildPath) + stdOut, errOut, err = sshClient.Exec(ctx, "mkdir", []string{"-p", filepath.Dir(insideAgentDir)}, nil) + if err != nil { + return fmt.Errorf("failed to create %s directory: %w (stdout: %s, stderr: %s)", filepath.Dir(insideAgentDir), err, stdOut, errOut) + } + stdOut, errOut, err = sshClient.Exec(ctx, "ln", []string{filepath.Base(buildPath), insideAgentDir}, nil) + if err != nil { + return fmt.Errorf("failed to hard link %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), insideAgentDir, err, stdOut, errOut) + } } } diff --git a/pkg/testing/runner/runner.go b/pkg/testing/runner/runner.go index f6a304c1de5..939cf108e46 100644 --- a/pkg/testing/runner/runner.go +++ b/pkg/testing/runner/runner.go @@ -24,6 +24,7 @@ import ( "golang.org/x/sync/errgroup" "k8s.io/utils/strings/slices" + "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" ) @@ -65,7 +66,7 @@ type OSRunner interface { // Prepare prepares the runner to actual run on the host. Prepare(ctx context.Context, sshClient SSHClient, logger Logger, arch string, goVersion string) error // Copy places the required files on the host. - Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, build Build) error + Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error // Run runs the actual tests and provides the result. Run(ctx context.Context, verbose bool, sshClient SSHClient, logger Logger, agentVersion string, prefix string, batch define.Batch, env map[string]string) (OSRunnerResult, error) // Diagnostics gathers any diagnostics from the host. @@ -361,7 +362,7 @@ func (r *Runner) runInstance(ctx context.Context, sshAuth ssh.AuthMethod, logger } // copy the required files (done every run) - err = batch.OS.Runner.Copy(ctx, client, logger, repoArchive, r.getBuild(batch)) + err = batch.OS.Runner.Copy(ctx, client, logger, repoArchive, r.getBuilds(batch)) if err != nil { logger.Logf("Failed to copy files instance: %s", err) return OSRunnerResult{}, fmt.Errorf("failed to copy files to instance %s: %w", instance.Name, err) @@ -419,12 +420,13 @@ func (r *Runner) validate() error { var requiredFiles []string for _, b := range r.batches { if !b.Skip { - build := r.getBuild(b) - if !slices.Contains(requiredFiles, build.Path) { - requiredFiles = append(requiredFiles, build.Path) - } - if !slices.Contains(requiredFiles, build.SHA512Path) { - requiredFiles = append(requiredFiles, build.SHA512Path) + for _, build := range r.getBuilds(b) { + if !slices.Contains(requiredFiles, build.Path) { + requiredFiles = append(requiredFiles, build.Path) + } + if !slices.Contains(requiredFiles, build.SHA512Path) { + requiredFiles = append(requiredFiles, build.SHA512Path) + } } } } @@ -443,29 +445,43 @@ func (r *Runner) validate() error { return nil } -// getBuild returns the build for the batch. -func (r *Runner) getBuild(b OSBatch) Build { - arch := b.OS.Arch - if arch == define.AMD64 { - arch = "x86_64" - } - ext := "tar.gz" - if b.OS.Type == define.Windows { - ext = "zip" +// getBuilds returns the build for the batch. +func (r *Runner) getBuilds(b OSBatch) []Build { + builds := []Build{} + formats := []string{"targz", "zip", "rpm", "deb"} + binaryName := "elastic-agent" + + // This is for testing beats in serverless environment + if strings.HasSuffix(r.cfg.BinaryName, "beat") { + formats = []string{"targz", "zip"} } - hashExt := ".sha512" - name := "elastic-agent" + if r.cfg.BinaryName != "" { - name = r.cfg.BinaryName + binaryName = r.cfg.BinaryName } - packageName := filepath.Join(r.cfg.BuildDir, fmt.Sprintf("%s-%s-%s-%s.%s", name, r.cfg.AgentVersion, b.OS.Type, arch, ext)) - return Build{ - Version: r.cfg.ReleaseVersion, - Type: b.OS.Type, - Arch: arch, - Path: packageName, - SHA512Path: packageName + hashExt, + + for _, f := range formats { + arch := b.OS.Arch + if arch == define.AMD64 { + arch = "x86_64" + } + suffix, err := testing.GetPackageSuffix(b.OS.Type, b.OS.Arch, f) + if err != nil { + // Means that OS type & Arch doesn't support that package format + continue + } + packageName := filepath.Join(r.cfg.BuildDir, fmt.Sprintf("%s-%s-%s", binaryName, r.cfg.AgentVersion, suffix)) + build := Build{ + Version: r.cfg.ReleaseVersion, + Type: b.OS.Type, + Arch: arch, + Path: packageName, + SHA512Path: packageName + ".sha512", + } + + builds = append(builds, build) } + return builds } // prepare prepares for the runner to run. diff --git a/pkg/testing/runner/windows.go b/pkg/testing/runner/windows.go index 8180ff9a4f3..2fcdc4f420c 100644 --- a/pkg/testing/runner/windows.go +++ b/pkg/testing/runner/windows.go @@ -69,7 +69,7 @@ func (WindowsRunner) Prepare(ctx context.Context, sshClient SSHClient, logger Lo } // Copy places the required files on the host. -func (WindowsRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, build Build) error { +func (WindowsRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logger, repoArchive string, builds []Build) error { // copy the archive and extract it on the host (tar exists and can extract zip on windows) logger.Logf("Copying repo") destRepoName := filepath.Base(repoArchive) @@ -101,55 +101,57 @@ func (WindowsRunner) Copy(ctx context.Context, sshClient SSHClient, logger Logge // determine if the build needs to be replaced on the host // if it already exists and the SHA512 are the same contents, then // there is no reason to waste time uploading the build - copyBuild := true - localSHA512, err := os.ReadFile(build.SHA512Path) - if err != nil { - return fmt.Errorf("failed to read local SHA52 contents %s: %w", build.SHA512Path, err) - } - hostSHA512Path := filepath.Base(build.SHA512Path) - hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path, WithContentFetchCommand("type")) - if err == nil { - if string(localSHA512) == string(hostSHA512) { - logger.Logf("Skipping copy agent build %s; already the same", filepath.Base(build.Path)) - copyBuild = false - } - } - - if copyBuild { - // ensure the existing copies are removed first - toRemove := filepath.Base(build.Path) - stdOut, errOut, err = sshClient.Exec(ctx, - "del", []string{toRemove, "/f", "/q"}, nil) + for _, build := range builds { + copyBuild := true + localSHA512, err := os.ReadFile(build.SHA512Path) if err != nil { - return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", - toRemove, err, stdOut, errOut) + return fmt.Errorf("failed to read local SHA52 contents %s: %w", build.SHA512Path, err) } - - toRemove = filepath.Base(build.SHA512Path) - stdOut, errOut, err = sshClient.Exec(ctx, - "del", []string{toRemove, "/f", "/q"}, nil) - if err != nil { - return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", - toRemove, err, stdOut, errOut) + hostSHA512Path := filepath.Base(build.SHA512Path) + hostSHA512, err := sshClient.GetFileContents(ctx, hostSHA512Path, WithContentFetchCommand("type")) + if err == nil { + if string(localSHA512) == string(hostSHA512) { + logger.Logf("Skipping copy agent build %s; already the same", filepath.Base(build.Path)) + copyBuild = false + } } - logger.Logf("Copying agent build %s", filepath.Base(build.Path)) - } - - for _, buildPath := range []string{build.Path, build.SHA512Path} { if copyBuild { - err = sshClient.Copy(buildPath, filepath.Base(buildPath)) + // ensure the existing copies are removed first + toRemove := filepath.Base(build.Path) + stdOut, errOut, err = sshClient.Exec(ctx, + "del", []string{toRemove, "/f", "/q"}, nil) if err != nil { - return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err) + return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", + toRemove, err, stdOut, errOut) } + + toRemove = filepath.Base(build.SHA512Path) + stdOut, errOut, err = sshClient.Exec(ctx, + "del", []string{toRemove, "/f", "/q"}, nil) + if err != nil { + return fmt.Errorf("failed to remove %q: %w (stdout: %q, stderr: %q)", + toRemove, err, stdOut, errOut) + } + + logger.Logf("Copying agent build %s", filepath.Base(build.Path)) } - insideAgentDir := filepath.Join("agent", buildPath) - // possible the build path already exists, 'mkdir' on windows will fail if it already exists - // error from this call is ignored because of it - _, _, _ = sshClient.Exec(ctx, "mkdir", []string{toWindowsPath(filepath.Dir(insideAgentDir))}, nil) - stdOut, errOut, err = sshClient.Exec(ctx, "mklink", []string{"/h", toWindowsPath(insideAgentDir), filepath.Base(buildPath)}, nil) - if err != nil { - return fmt.Errorf("failed to hard link %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), toWindowsPath(insideAgentDir), err, stdOut, errOut) + + for _, buildPath := range []string{build.Path, build.SHA512Path} { + if copyBuild { + err = sshClient.Copy(buildPath, filepath.Base(buildPath)) + if err != nil { + return fmt.Errorf("failed to SCP build %s: %w", filepath.Base(buildPath), err) + } + } + insideAgentDir := filepath.Join("agent", buildPath) + // possible the build path already exists, 'mkdir' on windows will fail if it already exists + // error from this call is ignored because of it + _, _, _ = sshClient.Exec(ctx, "mkdir", []string{toWindowsPath(filepath.Dir(insideAgentDir))}, nil) + stdOut, errOut, err = sshClient.Exec(ctx, "mklink", []string{"/h", toWindowsPath(insideAgentDir), filepath.Base(buildPath)}, nil) + if err != nil { + return fmt.Errorf("failed to hard link %s to %s: %w (stdout: %s, stderr: %s)", filepath.Base(buildPath), toWindowsPath(insideAgentDir), err, stdOut, errOut) + } } } diff --git a/testing/integration/groups_test.go b/testing/integration/groups_test.go index d6311cc7eb6..2c0f38b7e53 100644 --- a/testing/integration/groups_test.go +++ b/testing/integration/groups_test.go @@ -30,4 +30,7 @@ const ( // Upgrade group of tests. Used for testing upgrades. Upgrade = "upgrade" + + // Deb group of tests. Used for testing .deb packages install & upgrades + Deb = "deb" ) diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index 4254c10a5d1..5a31b21d95c 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -99,6 +99,74 @@ func TestLogIngestionFleetManaged(t *testing.T) { }) } +func TestDebLogIngestFleetManaged(t *testing.T) { + info := define.Require(t, define.Requirements{ + Group: Deb, + Stack: &define.Stack{}, + OS: []define.OS{ + { + Type: define.Linux, + Distro: "ubuntu", + }, + }, + Local: false, + Sudo: true, + }) + + ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) + defer cancel() + + agentFixture, err := define.NewFixture(t, define.Version(), atesting.WithPackageFormat("deb")) + require.NoError(t, err) + + // 1. Create a policy in Fleet with monitoring enabled. + // To ensure there are no conflicts with previous test runs against + // the same ESS stack, we add the current time at the end of the policy + // name. This policy does not contain any integration. + t.Log("Enrolling agent in Fleet with a test policy") + createPolicyReq := kibana.AgentPolicy{ + Name: fmt.Sprintf("test-policy-enroll-%d", time.Now().Unix()), + Namespace: info.Namespace, + Description: "test policy for agent enrollment", + MonitoringEnabled: []kibana.MonitoringEnabledOption{ + kibana.MonitoringEnabledLogs, + kibana.MonitoringEnabledMetrics, + }, + AgentFeatures: []map[string]interface{}{ + { + "name": "test_enroll", + "enabled": true, + }, + }, + } + + installOpts := atesting.InstallOpts{ + NonInteractive: true, + Force: true, + } + + // 2. Install the Elastic-Agent with the policy that + // was just created. + policy, err := tools.InstallAgentWithPolicy( + ctx, + t, + installOpts, + agentFixture, + info.KibanaClient, + createPolicyReq) + require.NoError(t, err) + t.Logf("created policy: %s", policy.ID) + check.ConnectedToFleet(ctx, t, agentFixture, 5*time.Minute) + + t.Run("Monitoring logs are shipped", func(t *testing.T) { + testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy) + }) + + t.Run("Normal logs with flattened data_stream are shipped", func(t *testing.T) { + testFlattenedDatastreamFleetPolicy(t, ctx, info, policy) + }) +} + func testMonitoringLogsAreShipped( t *testing.T, ctx context.Context, @@ -143,6 +211,7 @@ func testMonitoringLogsAreShipped( "add_cloud_metadata: received error failed requesting openstack metadata: Get \\\"https://169.254.169.254/2009-04-04/meta-data/instance-type\\\": dial tcp 169.254.169.254:443: connect: connection refused", // okay for the cloud metadata to not work "add_cloud_metadata: received error failed with http status code 404", // okay for the cloud metadata to not work "add_cloud_metadata: received error failed fetching EC2 Identity Document: operation error ec2imds: GetInstanceIdentityDocument, http response error StatusCode: 404, request to EC2 IMDS failed", // okay for the cloud metadata to not work + "failed to invoke rollback watcher: failed to start Upgrade Watcher: fork/exec /var/lib/elastic-agent/elastic-agent: no such file or directory", //on debian this happens probably need to fix. }) }) t.Logf("error logs: Got %d documents", len(docs.Hits.Hits)) diff --git a/testing/integration/upgrade_fleet_test.go b/testing/integration/upgrade_fleet_test.go index 8452605adfc..28d1e7d7e8e 100644 --- a/testing/integration/upgrade_fleet_test.go +++ b/testing/integration/upgrade_fleet_test.go @@ -146,7 +146,7 @@ func testFleetAirGappedUpgrade(t *testing.T, stack *define.Info, unprivileged bo err = upgradeTo.Prepare(ctx) require.NoError(t, err) - s := newArtifactsServer(ctx, t, latest) + s := newArtifactsServer(ctx, t, latest, upgradeTo.PackageFormat()) host := "artifacts.elastic.co" simulateAirGapedEnvironment(t, host) @@ -392,14 +392,14 @@ func simulateAirGapedEnvironment(t *testing.T, host string) { }) } -func newArtifactsServer(ctx context.Context, t *testing.T, version string) *httptest.Server { +func newArtifactsServer(ctx context.Context, t *testing.T, version string, packageFormat string) *httptest.Server { fileServerDir := t.TempDir() downloadAt := filepath.Join(fileServerDir, "downloads", "beats", "elastic-agent", "beats", "elastic-agent") err := os.MkdirAll(downloadAt, 0700) require.NoError(t, err, "could not create directory structure for file server") fetcher := atesting.ArtifactFetcher() - fr, err := fetcher.Fetch(ctx, runtime.GOOS, runtime.GOARCH, version) + fr, err := fetcher.Fetch(ctx, runtime.GOOS, runtime.GOARCH, version, packageFormat) require.NoErrorf(t, err, "could not prepare fetcher to download agent %s", version) err = fr.Fetch(ctx, t, downloadAt)