Use package manifest for install (#4173)

* use version in path for rpm and deb
* install elastic-agent remapping paths

dev-tools/mage/settings.go

@@ -23,7 +23,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/magefile/mage/sh"
-	"golang.org/x/tools/go/vcs"
+	"golang.org/x/tools/go/vcs" //nolint:staticcheck // this deprecation will be handled in https://github.com/elastic/elastic-agent/issues/4138
 
 	"github.com/elastic/elastic-agent/dev-tools/mage/gotool"
 	v1 "github.com/elastic/elastic-agent/pkg/api/v1"
@@ -48,6 +48,7 @@ const (
 	// Mapped functions
 	agentPackageVersionMappedFunc    = "agent_package_version"
 	agentManifestGeneratorMappedFunc = "manifest"
+	snapshotSuffix                   = "snapshot_suffix"
 )
 
 // Common settings with defaults derived from files, CWD, and environment.
@@ -108,6 +109,7 @@ var (
 		"contains":                       strings.Contains,
 		agentPackageVersionMappedFunc:    AgentPackageVersion,
 		agentManifestGeneratorMappedFunc: PackageManifest,
+		snapshotSuffix:                   SnapshotSuffix,
 	}
 )
 

dev-tools/packaging/packages.yml

@@ -62,19 +62,19 @@ shared:
       /etc/init.d/{{.BeatServiceName}}:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/elastic-agent.init.sh.tmpl'
         mode: 0755
-      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}:
         source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
         mode: 0755
-      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/package.version:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/package.version:
         content: >
           {{ agent_package_version }}
         mode: 0644
-      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/components:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/components:
         source: '{{.AgentDropPath}}/{{.GOOS}}-{{.AgentArchName}}.tar.gz/'
         mode: 0755
         config_mode: 0644
         skip_on_missing: true
-      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/manifest.yaml:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/manifest.yaml:
         mode: 0644
         content: >
           {{ manifest }}

dev-tools/packaging/templates/linux/postinstall.sh.tmpl

@@ -16,8 +16,9 @@ if test -L "$symlink"; then
 fi
 
 commit_hash="{{ commit_short }}"
+version_dir="{{agent_package_version}}{{snapshot_suffix}}"
 
-new_agent_dir="/var/lib/elastic-agent/data/elastic-agent-$commit_hash"
+new_agent_dir="/var/lib/elastic-agent/data/elastic-agent-$version_dir-$commit_hash"
 
 # copy the state files if there was a previous agent install
 if ! [ -z "$old_agent_dir" ] && ! [ "$old_agent_dir" -ef "$new_agent_dir" ]; then

internal/pkg/agent/install/install.go

@@ -5,6 +5,7 @@
 package install
 
 import (
+	goerrors "errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -20,6 +21,7 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/errors"
 	"github.com/elastic/elastic-agent/internal/pkg/cli"
+	v1 "github.com/elastic/elastic-agent/pkg/api/v1"
 	"github.com/elastic/elastic-agent/pkg/utils"
 )
 
@@ -109,35 +111,20 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p
 			errors.M("directory", filepath.Dir(topPath)))
 	}
 
-	// copy source into install path
-	//
-	// Try to detect if we are running with SSDs. If we are increase the copy concurrency,
-	// otherwise fall back to the default.
-	copyConcurrency := 1
-	hasSSDs, detectHWErr := HasAllSSDs()
-	if detectHWErr != nil {
-		fmt.Fprintf(streams.Out, "Could not determine block hardware type, disabling copy concurrency: %s\n", detectHWErr)
-	}
-	if hasSSDs {
-		copyConcurrency = runtime.NumCPU() * 4
+	manifest, err := readPackageManifest(dir)
+	if err != nil {
+		return utils.FileOwner{}, fmt.Errorf("reading package manifest: %w", err)
 	}
 
+	pathMappings := manifest.Package.PathMappings
+
 	pt.Describe("Copying install files")
-	err = copy.Copy(dir, topPath, copy.Options{
-		OnSymlink: func(_ string) copy.SymlinkAction {
-			return copy.Shallow
-		},
-		Sync:         true,
-		NumOfWorkers: int64(copyConcurrency),
-	})
+	err = copyFiles(streams, pathMappings, dir, topPath)
 	if err != nil {
 		pt.Describe("Error copying files")
-		return utils.FileOwner{}, errors.New(
-			err,
-			fmt.Sprintf("failed to copy source directory (%s) to destination (%s)", dir, topPath),
-			errors.M("source", dir), errors.M("destination", topPath),
-		)
+		return utils.FileOwner{}, err
 	}
+
 	pt.Describe("Successfully copied files")
 
 	// place shell wrapper, if present on platform
@@ -229,6 +216,131 @@ func Install(cfgFile, topPath string, unprivileged bool, log *logp.Logger, pt *p
 	return ownership, nil
 }
 
+func readPackageManifest(extractedPackageDir string) (*v1.PackageManifest, error) {
+	manifestFilePath := filepath.Join(extractedPackageDir, "manifest.yaml")
+	manifestFile, err := os.Open(manifestFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open package manifest file (%s): %w", manifestFilePath, err)
+	}
+	defer manifestFile.Close()
+	manifest, err := v1.ParseManifest(manifestFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse package manifest file %q contents: %w", manifestFilePath, err)
+	}
+
+	return manifest, nil
+}
+
+func copyFiles(streams *cli.IOStreams, pathMappings []map[string]string, srcDir string, topPath string) error {
+	// copy source into install path
+	//
+	// Try to detect if we are running with SSDs. If we are increase the copy concurrency,
+	// otherwise fall back to the default.
+	copyConcurrency := 1
+	hasSSDs, detectHWErr := HasAllSSDs()
+	if detectHWErr != nil {
+		fmt.Fprintf(streams.Out, "Could not determine block hardware type, disabling copy concurrency: %s\n", detectHWErr)
+	}
+	if hasSSDs {
+		copyConcurrency = runtime.NumCPU() * 4
+	}
+
+	// these are needed to keep track of what we already copied
+	copiedFiles := map[string]struct{}{}
+	// collect any symlink we found that need remapping
+	symlinks := map[string]string{}
+
+	var copyErrors []error
+
+	// Start copying the remapped paths first
+	for _, pathMapping := range pathMappings {
+		for packagePath, installedPath := range pathMapping {
+			// flag the original path as handled
+			copiedFiles[packagePath] = struct{}{}
+			err := copy.Copy(filepath.Join(srcDir, packagePath), filepath.Join(topPath, installedPath), copy.Options{
+				OnSymlink: func(_ string) copy.SymlinkAction {
+					return copy.Shallow
+				},
+				Sync:         true,
+				NumOfWorkers: int64(copyConcurrency),
+			})
+			if err != nil {
+				return errors.New(
+					err,
+					fmt.Sprintf("failed to copy source directory (%s) to destination (%s)", packagePath, installedPath),
+					errors.M("source", packagePath), errors.M("destination", installedPath),
+				)
+			}
+		}
+	}
+
+	// copy the remaining files excluding overlaps with the mapped paths
+	err := copy.Copy(srcDir, topPath, copy.Options{
+		OnSymlink: func(source string) copy.SymlinkAction {
+			target, err := os.Readlink(source)
+			if err != nil {
+				// error reading the link, not much choice to leave it unchanged and collect the error
+				copyErrors = append(copyErrors, fmt.Errorf("unable to read link %q for remapping", source))
+				return copy.Skip
+			}
+
+			// if we find a link, check if its target need to be remapped, in which case skip it for now and save it for
+			// later creation with the remapped target
+			for _, pathMapping := range pathMappings {
+				for srcPath, dstPath := range pathMapping {
+					if strings.HasPrefix(target, srcPath) {
+						newTarget := strings.Replace(target, srcPath, dstPath, 1)
+						rel, err := filepath.Rel(srcDir, source)
+						if err != nil {
+							copyErrors = append(copyErrors, fmt.Errorf("extracting relative path for %q using %q as base: %w", source, srcDir, err))
+							return copy.Skip
+						}
+						symlinks[rel] = newTarget
+						return copy.Skip
+					}
+				}
+			}
+
+			return copy.Shallow
+		},
+		Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) {
+			relPath, err := filepath.Rel(srcDir, src)
+			if err != nil {
+				return false, fmt.Errorf("calculating relative path for %s: %w", src, err)
+			}
+			// check if we already handled this path as part of the mappings: if we did, skip it
+			_, ok := copiedFiles[relPath]
+			return ok, nil
+		},
+		Sync:         true,
+		NumOfWorkers: int64(copyConcurrency),
+	})
+	if err != nil {
+		return errors.New(
+			err,
+			fmt.Sprintf("failed to copy source directory (%s) to destination (%s)", srcDir, topPath),
+			errors.M("source", srcDir), errors.M("destination", topPath),
+		)
+	}
+
+	if len(copyErrors) > 0 {
+		return fmt.Errorf("errors encountered during copy from %q to %q: %w", srcDir, topPath, goerrors.Join(copyErrors...))
+	}
+
+	// Create the remapped symlinks
+	for src, target := range symlinks {
+		absSrcPath := filepath.Join(topPath, src)
+		err := os.Symlink(target, absSrcPath)
+		if err != nil {
+			return errors.New(
+				err,
+				fmt.Sprintf("failed to link source %q to destination %q", absSrcPath, target),
+			)
+		}
+	}
+	return nil
+}
+
 // StartService starts the installed service.
 //
 // This should only be called after Install is successful.

internal/pkg/agent/install/install_test.go

@@ -5,11 +5,16 @@
 package install
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/jaypipes/ghw"
 	"github.com/jaypipes/ghw/pkg/block"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-agent/internal/pkg/cli"
 )
 
 func TestHasAllSSDs(t *testing.T) {
@@ -57,3 +62,125 @@ func TestHasAllSSDs(t *testing.T) {
 		})
 	}
 }
+
+var sampleManifestContent = `
+version: co.elastic.agent/v1
+kind: PackageManifest
+package:
+    version: 8.13.0
+    snapshot: true
+    versioned-home: elastic-agent-fb7370
+    path-mappings:
+        - data/elastic-agent-fb7370: data/elastic-agent-8.13.0-SNAPSHOT-fb7370
+          manifest.yaml: data/elastic-agent-8.13.0-SNAPSHOT-fb7370/manifest.yaml
+`
+
+type testLogWriter struct {
+	t *testing.T
+}
+
+func (tlw testLogWriter) Write(b []byte) (n int, err error) {
+	tlw.t.Log(b)
+	return len(b), nil
+}
+
+func TestCopyFiles(t *testing.T) {
+	type fileType uint
+
+	const (
+		REGULAR fileType = iota
+		DIRECTORY
+		SYMLINK
+	)
+
+	type files struct {
+		fType   fileType
+		path    string
+		content []byte
+	}
+
+	type testcase struct {
+		name          string
+		setupFiles    []files
+		expectedFiles []files
+		mappings      []map[string]string
+	}
+
+	testcases := []testcase{
+		{
+			name: "simple install package mockup",
+			setupFiles: []files{
+				{fType: REGULAR, path: "manifest.yaml", content: []byte(sampleManifestContent)},
+				{fType: DIRECTORY, path: filepath.Join("data", "elastic-agent-fb7370"), content: nil},
+				{fType: REGULAR, path: filepath.Join("data", "elastic-agent-fb7370", "elastic-agent"), content: []byte("this is an elastic-agent wannabe")},
+				{fType: SYMLINK, path: "elastic-agent", content: []byte(filepath.Join("data", "elastic-agent-fb7370", "elastic-agent"))},
+			},
+			expectedFiles: []files{
+				{fType: DIRECTORY, path: filepath.Join("data", "elastic-agent-8.13.0-SNAPSHOT-fb7370"), content: nil},
+				{fType: REGULAR, path: filepath.Join("data", "elastic-agent-8.13.0-SNAPSHOT-fb7370", "manifest.yaml"), content: []byte(sampleManifestContent)},
+				{fType: REGULAR, path: filepath.Join("data", "elastic-agent-8.13.0-SNAPSHOT-fb7370", "elastic-agent"), content: []byte("this is an elastic-agent wannabe")},
+				{fType: SYMLINK, path: "elastic-agent", content: []byte(filepath.Join("data", "elastic-agent-8.13.0-SNAPSHOT-fb7370", "elastic-agent"))},
+			},
+			mappings: []map[string]string{
+				{
+					"data/elastic-agent-fb7370": "data/elastic-agent-8.13.0-SNAPSHOT-fb7370",
+					"manifest.yaml":             "data/elastic-agent-8.13.0-SNAPSHOT-fb7370/manifest.yaml",
+				},
+			}},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			tmpSrc := t.TempDir()
+			tmpDst := t.TempDir()
+
+			for _, sf := range tc.setupFiles {
+				switch sf.fType {
+				case REGULAR:
+					err := os.WriteFile(filepath.Join(tmpSrc, sf.path), sf.content, 0o644)
+					require.NoErrorf(t, err, "error writing setup file %s in tempDir %s", sf.path, tmpSrc)
+
+				case DIRECTORY:
+					err := os.MkdirAll(filepath.Join(tmpSrc, sf.path), 0o755)
+					require.NoErrorf(t, err, "error creating setup directory %s in tempDir %s", sf.path, tmpSrc)
+
+				case SYMLINK:
+					err := os.Symlink(string(sf.content), filepath.Join(tmpSrc, sf.path))
+					require.NoErrorf(t, err, "error creating symlink %s in tempDir %s", sf.path, tmpSrc)
+				}
+			}
+			outWriter := &testLogWriter{t: t}
+			ioStreams := &cli.IOStreams{
+				In:  nil,
+				Out: outWriter,
+				Err: outWriter,
+			}
+			err := copyFiles(ioStreams, tc.mappings, tmpSrc, tmpDst)
+			assert.NoError(t, err)
+
+			for _, ef := range tc.expectedFiles {
+				switch ef.fType {
+				case REGULAR:
+					if assert.FileExistsf(t, filepath.Join(tmpDst, ef.path), "file %s does not exist in output directory %s", ef.path, tmpDst) {
+						// check contents
+						actualContent, err := os.ReadFile(filepath.Join(tmpDst, ef.path))
+						if assert.NoErrorf(t, err, "error reading expected file %s content", ef.path) {
+							assert.Equal(t, ef.content, actualContent, "content of expected file %s does not match", ef.path)
+						}
+					}
+
+				case DIRECTORY:
+					assert.DirExistsf(t, filepath.Join(tmpDst, ef.path), "directory %s does not exist in output directory %s", ef.path, tmpDst)
+
+				case SYMLINK:
+					actualTarget, err := os.Readlink(filepath.Join(tmpDst, ef.path))
+					if assert.NoErrorf(t, err, "error readling expected symlink %s in output dir %s", ef.path, tmpDst) {
+						assert.Equal(t, string(ef.content), actualTarget, "unexpected target for symlink %s", ef.path)
+					}
+				}
+
+			}
+		})
+	}
+
+}