From 4c9e1458c6267a02ad9f04c12e9ac35b1452cbb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paolo=20Chil=C3=A0?= Date: Fri, 2 Feb 2024 17:19:11 +0100 Subject: [PATCH] Use package manifest for install (#4173) * use version in path for rpm and deb * install elastic-agent remapping paths --- dev-tools/mage/settings.go | 4 +- dev-tools/packaging/packages.yml | 8 +- .../templates/linux/postinstall.sh.tmpl | 3 +- internal/pkg/agent/install/install.go | 158 +++++++++++++++--- internal/pkg/agent/install/install_test.go | 127 ++++++++++++++ 5 files changed, 271 insertions(+), 29 deletions(-) diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index a087fd51f00..c94d13d834b 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -23,7 +23,7 @@ import ( "gopkg.in/yaml.v3" "github.com/magefile/mage/sh" - "golang.org/x/tools/go/vcs" + "golang.org/x/tools/go/vcs" //nolint:staticcheck // this deprecation will be handled in https://github.com/elastic/elastic-agent/issues/4138 "github.com/elastic/elastic-agent/dev-tools/mage/gotool" v1 "github.com/elastic/elastic-agent/pkg/api/v1" @@ -48,6 +48,7 @@ const ( // Mapped functions agentPackageVersionMappedFunc = "agent_package_version" agentManifestGeneratorMappedFunc = "manifest" + snapshotSuffix = "snapshot_suffix" ) // Common settings with defaults derived from files, CWD, and environment. @@ -108,6 +109,7 @@ var ( "contains": strings.Contains, agentPackageVersionMappedFunc: AgentPackageVersion, agentManifestGeneratorMappedFunc: PackageManifest, + snapshotSuffix: SnapshotSuffix, } ) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index c64a26bb0bc..b948dad3d16 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -62,19 +62,19 @@ shared: /etc/init.d/{{.BeatServiceName}}: template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/elastic-agent.init.sh.tmpl' mode: 0755 - /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}: + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}: source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} mode: 0755 - /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/package.version: + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/package.version: content: > {{ agent_package_version }} mode: 0644 - /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/components: + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/components: source: '{{.AgentDropPath}}/{{.GOOS}}-{{.AgentArchName}}.tar.gz/' mode: 0755 config_mode: 0644 skip_on_missing: true - /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/manifest.yaml: + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/manifest.yaml: mode: 0644 content: > {{ manifest }} diff --git a/dev-tools/packaging/templates/linux/postinstall.sh.tmpl b/dev-tools/packaging/templates/linux/postinstall.sh.tmpl index d96f21a8629..c1927d9d550 100644 --- a/dev-tools/packaging/templates/linux/postinstall.sh.tmpl +++ b/dev-tools/packaging/templates/linux/postinstall.sh.tmpl @@ -16,8 +16,9 @@ if test -L "$symlink"; then fi commit_hash="{{ commit_short }}" +version_dir="{{agent_package_version}}{{snapshot_suffix}}" -new_agent_dir="/var/lib/elastic-agent/data/elastic-agent-$commit_hash" +new_agent_dir="/var/lib/elastic-agent/data/elastic-agent-$version_dir-$commit_hash" # copy the state files if there was a previous agent install if ! [ -z "$old_agent_dir" ] && ! [ "$old_agent_dir" -ef "$new_agent_dir" ]; then diff --git a/internal/pkg/agent/install/install.go b/internal/pkg/agent/install/install.go index 2ff0ef7dacc..14b6c275c10 100644 --- a/internal/pkg/agent/install/install.go +++ b/internal/pkg/agent/install/install.go @@ -5,6 +5,7 @@ package install import ( + goerrors "errors" "fmt" "os" "path/filepath" @@ -20,6 +21,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/paths" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" "github.com/elastic/elastic-agent/internal/pkg/cli" + v1 "github.com/elastic/elastic-agent/pkg/api/v1" "github.com/elastic/elastic-agent/pkg/utils" ) @@ -109,35 +111,20 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p errors.M("directory", filepath.Dir(topPath))) } - // copy source into install path - // - // Try to detect if we are running with SSDs. If we are increase the copy concurrency, - // otherwise fall back to the default. - copyConcurrency := 1 - hasSSDs, detectHWErr := HasAllSSDs() - if detectHWErr != nil { - fmt.Fprintf(streams.Out, "Could not determine block hardware type, disabling copy concurrency: %s\n", detectHWErr) - } - if hasSSDs { - copyConcurrency = runtime.NumCPU() * 4 + manifest, err := readPackageManifest(dir) + if err != nil { + return utils.FileOwner{}, fmt.Errorf("reading package manifest: %w", err) } + pathMappings := manifest.Package.PathMappings + pt.Describe("Copying install files") - err = copy.Copy(dir, topPath, copy.Options{ - OnSymlink: func(_ string) copy.SymlinkAction { - return copy.Shallow - }, - Sync: true, - NumOfWorkers: int64(copyConcurrency), - }) + err = copyFiles(streams, pathMappings, dir, topPath) if err != nil { pt.Describe("Error copying files") - return utils.FileOwner{}, errors.New( - err, - fmt.Sprintf("failed to copy source directory (%s) to destination (%s)", dir, topPath), - errors.M("source", dir), errors.M("destination", topPath), - ) + return utils.FileOwner{}, err } + pt.Describe("Successfully copied files") // place shell wrapper, if present on platform @@ -229,6 +216,131 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p return ownership, nil } +func readPackageManifest(extractedPackageDir string) (*v1.PackageManifest, error) { + manifestFilePath := filepath.Join(extractedPackageDir, "manifest.yaml") + manifestFile, err := os.Open(manifestFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open package manifest file (%s): %w", manifestFilePath, err) + } + defer manifestFile.Close() + manifest, err := v1.ParseManifest(manifestFile) + if err != nil { + return nil, fmt.Errorf("failed to parse package manifest file %q contents: %w", manifestFilePath, err) + } + + return manifest, nil +} + +func copyFiles(streams *cli.IOStreams, pathMappings []map[string]string, srcDir string, topPath string) error { + // copy source into install path + // + // Try to detect if we are running with SSDs. If we are increase the copy concurrency, + // otherwise fall back to the default. + copyConcurrency := 1 + hasSSDs, detectHWErr := HasAllSSDs() + if detectHWErr != nil { + fmt.Fprintf(streams.Out, "Could not determine block hardware type, disabling copy concurrency: %s\n", detectHWErr) + } + if hasSSDs { + copyConcurrency = runtime.NumCPU() * 4 + } + + // these are needed to keep track of what we already copied + copiedFiles := map[string]struct{}{} + // collect any symlink we found that need remapping + symlinks := map[string]string{} + + var copyErrors []error + + // Start copying the remapped paths first + for _, pathMapping := range pathMappings { + for packagePath, installedPath := range pathMapping { + // flag the original path as handled + copiedFiles[packagePath] = struct{}{} + err := copy.Copy(filepath.Join(srcDir, packagePath), filepath.Join(topPath, installedPath), copy.Options{ + OnSymlink: func(_ string) copy.SymlinkAction { + return copy.Shallow + }, + Sync: true, + NumOfWorkers: int64(copyConcurrency), + }) + if err != nil { + return errors.New( + err, + fmt.Sprintf("failed to copy source directory (%s) to destination (%s)", packagePath, installedPath), + errors.M("source", packagePath), errors.M("destination", installedPath), + ) + } + } + } + + // copy the remaining files excluding overlaps with the mapped paths + err := copy.Copy(srcDir, topPath, copy.Options{ + OnSymlink: func(source string) copy.SymlinkAction { + target, err := os.Readlink(source) + if err != nil { + // error reading the link, not much choice to leave it unchanged and collect the error + copyErrors = append(copyErrors, fmt.Errorf("unable to read link %q for remapping", source)) + return copy.Skip + } + + // if we find a link, check if its target need to be remapped, in which case skip it for now and save it for + // later creation with the remapped target + for _, pathMapping := range pathMappings { + for srcPath, dstPath := range pathMapping { + if strings.HasPrefix(target, srcPath) { + newTarget := strings.Replace(target, srcPath, dstPath, 1) + rel, err := filepath.Rel(srcDir, source) + if err != nil { + copyErrors = append(copyErrors, fmt.Errorf("extracting relative path for %q using %q as base: %w", source, srcDir, err)) + return copy.Skip + } + symlinks[rel] = newTarget + return copy.Skip + } + } + } + + return copy.Shallow + }, + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + relPath, err := filepath.Rel(srcDir, src) + if err != nil { + return false, fmt.Errorf("calculating relative path for %s: %w", src, err) + } + // check if we already handled this path as part of the mappings: if we did, skip it + _, ok := copiedFiles[relPath] + return ok, nil + }, + Sync: true, + NumOfWorkers: int64(copyConcurrency), + }) + if err != nil { + return errors.New( + err, + fmt.Sprintf("failed to copy source directory (%s) to destination (%s)", srcDir, topPath), + errors.M("source", srcDir), errors.M("destination", topPath), + ) + } + + if len(copyErrors) > 0 { + return fmt.Errorf("errors encountered during copy from %q to %q: %w", srcDir, topPath, goerrors.Join(copyErrors...)) + } + + // Create the remapped symlinks + for src, target := range symlinks { + absSrcPath := filepath.Join(topPath, src) + err := os.Symlink(target, absSrcPath) + if err != nil { + return errors.New( + err, + fmt.Sprintf("failed to link source %q to destination %q", absSrcPath, target), + ) + } + } + return nil +} + // StartService starts the installed service. // // This should only be called after Install is successful. diff --git a/internal/pkg/agent/install/install_test.go b/internal/pkg/agent/install/install_test.go index dd73cac17a8..84812bad54f 100644 --- a/internal/pkg/agent/install/install_test.go +++ b/internal/pkg/agent/install/install_test.go @@ -5,11 +5,16 @@ package install import ( + "os" + "path/filepath" "testing" "github.com/jaypipes/ghw" "github.com/jaypipes/ghw/pkg/block" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent/internal/pkg/cli" ) func TestHasAllSSDs(t *testing.T) { @@ -57,3 +62,125 @@ func TestHasAllSSDs(t *testing.T) { }) } } + +var sampleManifestContent = ` +version: co.elastic.agent/v1 +kind: PackageManifest +package: + version: 8.13.0 + snapshot: true + versioned-home: elastic-agent-fb7370 + path-mappings: + - data/elastic-agent-fb7370: data/elastic-agent-8.13.0-SNAPSHOT-fb7370 + manifest.yaml: data/elastic-agent-8.13.0-SNAPSHOT-fb7370/manifest.yaml +` + +type testLogWriter struct { + t *testing.T +} + +func (tlw testLogWriter) Write(b []byte) (n int, err error) { + tlw.t.Log(b) + return len(b), nil +} + +func TestCopyFiles(t *testing.T) { + type fileType uint + + const ( + REGULAR fileType = iota + DIRECTORY + SYMLINK + ) + + type files struct { + fType fileType + path string + content []byte + } + + type testcase struct { + name string + setupFiles []files + expectedFiles []files + mappings []map[string]string + } + + testcases := []testcase{ + { + name: "simple install package mockup", + setupFiles: []files{ + {fType: REGULAR, path: "manifest.yaml", content: []byte(sampleManifestContent)}, + {fType: DIRECTORY, path: filepath.Join("data", "elastic-agent-fb7370"), content: nil}, + {fType: REGULAR, path: filepath.Join("data", "elastic-agent-fb7370", "elastic-agent"), content: []byte("this is an elastic-agent wannabe")}, + {fType: SYMLINK, path: "elastic-agent", content: []byte(filepath.Join("data", "elastic-agent-fb7370", "elastic-agent"))}, + }, + expectedFiles: []files{ + {fType: DIRECTORY, path: filepath.Join("data", "elastic-agent-8.13.0-SNAPSHOT-fb7370"), content: nil}, + {fType: REGULAR, path: filepath.Join("data", "elastic-agent-8.13.0-SNAPSHOT-fb7370", "manifest.yaml"), content: []byte(sampleManifestContent)}, + {fType: REGULAR, path: filepath.Join("data", "elastic-agent-8.13.0-SNAPSHOT-fb7370", "elastic-agent"), content: []byte("this is an elastic-agent wannabe")}, + {fType: SYMLINK, path: "elastic-agent", content: []byte(filepath.Join("data", "elastic-agent-8.13.0-SNAPSHOT-fb7370", "elastic-agent"))}, + }, + mappings: []map[string]string{ + { + "data/elastic-agent-fb7370": "data/elastic-agent-8.13.0-SNAPSHOT-fb7370", + "manifest.yaml": "data/elastic-agent-8.13.0-SNAPSHOT-fb7370/manifest.yaml", + }, + }}, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + tmpSrc := t.TempDir() + tmpDst := t.TempDir() + + for _, sf := range tc.setupFiles { + switch sf.fType { + case REGULAR: + err := os.WriteFile(filepath.Join(tmpSrc, sf.path), sf.content, 0o644) + require.NoErrorf(t, err, "error writing setup file %s in tempDir %s", sf.path, tmpSrc) + + case DIRECTORY: + err := os.MkdirAll(filepath.Join(tmpSrc, sf.path), 0o755) + require.NoErrorf(t, err, "error creating setup directory %s in tempDir %s", sf.path, tmpSrc) + + case SYMLINK: + err := os.Symlink(string(sf.content), filepath.Join(tmpSrc, sf.path)) + require.NoErrorf(t, err, "error creating symlink %s in tempDir %s", sf.path, tmpSrc) + } + } + outWriter := &testLogWriter{t: t} + ioStreams := &cli.IOStreams{ + In: nil, + Out: outWriter, + Err: outWriter, + } + err := copyFiles(ioStreams, tc.mappings, tmpSrc, tmpDst) + assert.NoError(t, err) + + for _, ef := range tc.expectedFiles { + switch ef.fType { + case REGULAR: + if assert.FileExistsf(t, filepath.Join(tmpDst, ef.path), "file %s does not exist in output directory %s", ef.path, tmpDst) { + // check contents + actualContent, err := os.ReadFile(filepath.Join(tmpDst, ef.path)) + if assert.NoErrorf(t, err, "error reading expected file %s content", ef.path) { + assert.Equal(t, ef.content, actualContent, "content of expected file %s does not match", ef.path) + } + } + + case DIRECTORY: + assert.DirExistsf(t, filepath.Join(tmpDst, ef.path), "directory %s does not exist in output directory %s", ef.path, tmpDst) + + case SYMLINK: + actualTarget, err := os.Readlink(filepath.Join(tmpDst, ef.path)) + if assert.NoErrorf(t, err, "error readling expected symlink %s in output dir %s", ef.path, tmpDst) { + assert.Equal(t, string(ef.content), actualTarget, "unexpected target for symlink %s", ef.path) + } + } + + } + }) + } + +}