Feature/add agent version to installation directory name (#4193)

* Use package manifest for install (#4173)

* use version in path for rpm and deb
* install elastic-agent remapping paths

* Use package manifests for upgrade (#4174)

* map paths when upgrading using .tar.gz packages
* use structured output from unpack to identify old and new agents directories
* copy state directories and rotate symlink correctly

* Support version in watcher cleanup rollback (#4176)

* extract directory paths to allow unit testing (instead of nested paths.* calls)
* introduce tests for watcher cleanup and rollback
* close data directory after cleanup

* Reintroduce upgrade version check (#4177)

* Fix shutdownCallback() directories
* reintroduce version check in upgrade

* Use watcher that supports paths in update marker (#4178)

* Invoke agent watcher capable of handling new paths

* Adapt TestShutdownCallback to run multiple testcases

* Fix shutdownCallback to work with version in path

* Fix MacOS relink step

* Add commit hash to manifest

* document manifest-aware upgrade
---------

Co-authored-by: Craig MacKenzie <craig.mackenzie@elastic.co>

* small fixes based on PR feedback

* Add wait for watcher (#4229)

* Fix rollbackInstall order of execution and handle errors
* Add wait for watcher up and running

---------

Co-authored-by: Craig MacKenzie <craig.mackenzie@elastic.co>

changelog/fragments/1706703918-Add-the-full-version-number-to-the-installation-directory-name.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Add the full version number to the installation directory name
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component: agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

dev-tools/mage/settings.go

@@ -23,7 +23,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/magefile/mage/sh"
-	"golang.org/x/tools/go/vcs"
+	"golang.org/x/tools/go/vcs" //nolint:staticcheck // this deprecation will be handled in https://github.com/elastic/elastic-agent/issues/4138
 
 	"github.com/elastic/elastic-agent/dev-tools/mage/gotool"
 	v1 "github.com/elastic/elastic-agent/pkg/api/v1"
@@ -48,6 +48,7 @@ const (
 	// Mapped functions
 	agentPackageVersionMappedFunc    = "agent_package_version"
 	agentManifestGeneratorMappedFunc = "manifest"
+	snapshotSuffix                   = "snapshot_suffix"
 )
 
 // Common settings with defaults derived from files, CWD, and environment.
@@ -108,6 +109,7 @@ var (
 		"contains":                       strings.Contains,
 		agentPackageVersionMappedFunc:    AgentPackageVersion,
 		agentManifestGeneratorMappedFunc: PackageManifest,
+		snapshotSuffix:                   SnapshotSuffix,
 	}
 )
 
@@ -253,6 +255,7 @@ repo.RootDir                 = {{ repo.RootDir }}
 repo.ImportPath              = {{ repo.ImportPath }}
 repo.SubDir                  = {{ repo.SubDir }}
 agent_package_version        = {{ agent_package_version}}
+snapshot_suffix              = {{ snapshot_suffix }}
 `
 
 	return Expand(dumpTemplate)
@@ -309,6 +312,13 @@ func PackageManifest() (string, error) {
 		return "", fmt.Errorf("retrieving agent package version: %w", err)
 	}
 	m.Package.Version = packageVersion
+
+	hash, err := CommitHash()
+	if err != nil {
+		return "", fmt.Errorf("retrieving agent commit hash: %w", err)
+	}
+	m.Package.Hash = hash
+
 	commitHashShort, err := CommitHashShort()
 	if err != nil {
 		return "", fmt.Errorf("retrieving agent commit hash: %w", err)
@@ -318,7 +328,7 @@ func PackageManifest() (string, error) {
 	m.Package.VersionedHome = versionedHomePath
 	m.Package.PathMappings = []map[string]string{{}}
 	m.Package.PathMappings[0][versionedHomePath] = fmt.Sprintf("data/elastic-agent-%s%s-%s", m.Package.Version, SnapshotSuffix(), commitHashShort)
-	m.Package.PathMappings[0]["manifest.yaml"] = fmt.Sprintf("data/elastic-agent-%s%s-%s/manifest.yaml", m.Package.Version, SnapshotSuffix(), commitHashShort)
+	m.Package.PathMappings[0][v1.ManifestFileName] = fmt.Sprintf("data/elastic-agent-%s%s-%s/%s", m.Package.Version, SnapshotSuffix(), commitHashShort, v1.ManifestFileName)
 	yamlBytes, err := yaml.Marshal(m)
 	if err != nil {
 		return "", fmt.Errorf("marshaling manifest: %w", err)

dev-tools/packaging/package_test.go

@@ -201,9 +201,9 @@ func checkZip(t *testing.T, file string) {
 
 func checkManifestFileContents(t *testing.T, extractedPackageDir string) {
 	t.Log("Checking file manifest.yaml")
-	manifestReadCloser, err := os.Open(filepath.Join(extractedPackageDir, "manifest.yaml"))
+	manifestReadCloser, err := os.Open(filepath.Join(extractedPackageDir, v1.ManifestFileName))
 	if err != nil {
-		t.Errorf("opening manifest %s : %v", "manifest.yaml", err)
+		t.Errorf("opening manifest %s : %v", v1.ManifestFileName, err)
 	}
 	defer func(closer io.ReadCloser) {
 		err := closer.Close()
@@ -219,6 +219,9 @@ func checkManifestFileContents(t *testing.T, extractedPackageDir string) {
 	assert.Equal(t, v1.ManifestKind, m.Kind, "manifest specifies wrong kind")
 	assert.Equal(t, v1.VERSION, m.Version, "manifest specifies wrong api version")
 
+	assert.NotEmpty(t, m.Package.Version, "manifest version must not be empty")
+	assert.NotEmpty(t, m.Package.Hash, "manifest hash must not be empty")
+
 	if assert.NotEmpty(t, m.Package.PathMappings, "path mappings in manifest are empty") {
 		versionedHome := m.Package.VersionedHome
 		assert.DirExistsf(t, filepath.Join(extractedPackageDir, versionedHome), "versionedHome directory %q not found in %q", versionedHome, extractedPackageDir)

dev-tools/packaging/packages.yml

@@ -62,19 +62,19 @@ shared:
       /etc/init.d/{{.BeatServiceName}}:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/elastic-agent.init.sh.tmpl'
         mode: 0755
-      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}:
         source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
         mode: 0755
-      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/package.version:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/package.version:
         content: >
           {{ agent_package_version }}
         mode: 0644
-      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/components:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/components:
         source: '{{.AgentDropPath}}/{{.GOOS}}-{{.AgentArchName}}.tar.gz/'
         mode: 0755
         config_mode: 0644
         skip_on_missing: true
-      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/manifest.yaml:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{agent_package_version}}{{snapshot_suffix}}-{{ commit_short }}/manifest.yaml:
         mode: 0644
         content: >
           {{ manifest }}

dev-tools/packaging/templates/darwin/elastic-agent.tmpl

@@ -6,6 +6,9 @@ set -e
 symlink="/Library/Elastic/Agent/elastic-agent"
 
 if test -L "$symlink"; then
-    ln -sfn "data/elastic-agent-{{ commit_short }}/elastic-agent.app/Contents/MacOS/elastic-agent" "$symlink"
+    symlinkTarget="data/elastic-agent-{{ commit_short }}/elastic-agent.app/Contents/MacOS/elastic-agent"
+    if test -f "data/elastic-agent-{{ agent_package_version }}{{ snapshot_suffix }}-{{ commit_short }}/elastic-agent.app/Contents/MacOS/elastic-agent"; then
+        symlinkTarget="data/elastic-agent-{{ agent_package_version }}{{ snapshot_suffix }}-{{ commit_short }}/elastic-agent.app/Contents/MacOS/elastic-agent"
+    ln -sfn "$symlinkTarget" "$symlink"
 fi
 

dev-tools/packaging/templates/linux/postinstall.sh.tmpl

@@ -16,8 +16,9 @@ if test -L "$symlink"; then
 fi
 
 commit_hash="{{ commit_short }}"
+version_dir="{{agent_package_version}}{{snapshot_suffix}}"
 
-new_agent_dir="/var/lib/elastic-agent/data/elastic-agent-$commit_hash"
+new_agent_dir="/var/lib/elastic-agent/data/elastic-agent-$version_dir-$commit_hash"
 
 # copy the state files if there was a previous agent install
 if ! [ -z "$old_agent_dir" ] && ! [ "$old_agent_dir" -ef "$new_agent_dir" ]; then

docs/upgrades.md

@@ -61,3 +61,74 @@ sequenceDiagram
        end
     end
 ```
+
+### Introducing package manifest
+
+Starting from version 8.13.0 an additional file `manifest.yaml` is present in elastic-agent packages.
+The purpose of this file is to present some metadata and package information to be used during install/upgrade operations.
+
+The first enhancement that makes use of this package manifest is [#2579](https://github.com/elastic/elastic-agent/issues/2579)
+as we use the manifest to map the package directory structure (based on agent commit hash) into one that takes also the
+agent version into account. This allows releasing versions of the agent package where only the component versions change,
+with the agent commit unchanged.
+
+
+The [structure](../pkg/api/v1/manifest.go) of such manifest is defined in the [api/v1 package](../pkg/api/v1/).
+The manifest data is generated during packaging and the file is added to the package files. This is an example of a
+complete manifest:
+
+```yaml
+version: co.elastic.agent/v1
+kind: PackageManifest
+package:
+  version: 8.13.0
+  snapshot: true
+  hash: 15658b38b48ba4487afadc5563b1576b85ce0264
+  versioned-home: data/elastic-agent-15658b
+  path-mappings:
+    - data/elastic-agent-15658b: data/elastic-agent-8.13.0-SNAPSHOT-15658b
+      manifest.yaml: data/elastic-agent-8.13.0-SNAPSHOT-15658b/manifest.yaml
+```
+
+The package information describes the package version, whether it's a snapshot build, the elastic-agent commit hash it
+has been built from and where to find the versioned home of the elastic agent within the package.
+
+Another section lists the path mappings that must be applied by an elastic-agent that is aware of the package manifest
+(version >8.13.0): these path mappings allow the incoming agent version to have some control over where the files in
+package will be stored on disk.
+
+#### Upgrading without the manifest
+
+Legacy elastic-agent upgrade is a pretty straightforward affair:
+- Download the agent package to use for upgrade
+- Open the .zip or .tar.gz archive and iterate over the files
+  - Look for the elastic-agent commit file to retrieve the actual hash of the agent version we want to install
+  - Extract any package file under `/data` under the installed agent `/data` directory
+- After extraction check if the hash we read from the package matches with the one from the current agent:
+  - if it's the same hash the upgrade fails because we are trying to upgrade to the same version
+  - if we extracted a package with a different hash, the upgrade keeps going
+- Copy the elastic agent action store and components run directory into the new agent directories `elastic-agent-<hash>`
+- Rotate the symlink in the top directory to point to the new agent executable `data/elastic-agent-<hash>/elastic-agent`
+- Write the update marker containing the information about the new and old agent versions/hashes in `data` directory
+- Invoke the watcher `elastic-agent watch` command to ensure that the new version of agent works correctly after restart
+- Shutdown current agent and its command components, copy components state once again and restart
+
+#### Upgrading using the manifest
+
+Upgrading using the manifest allows for the new version to pass along some information about the package to the upgrading agent.
+The new process looks like this:
+- Download the elastic-agent package to use for upgrade
+- Extract package metadata from the new elastic-agent package (`version`, `snapshot` and `hash`):
+  - if the package has a manifest we extract `version` and `snapshot` flag as declared by the package manifest
+  - if there is no manifest for the package we extract `version` and `snapshot` from the version string passed to the upgrader
+  - the `hash` is always retrieved from the agent commit file (this is always present in the package)
+- compare the tuple of new `(version, snapshot, hash)` to the current `(version, snapshot, hash)`: if they are the same
+  the upgrade fails because we are trying to upgrade to the same version as current
+- Extract any package file (after mapping it using file mappings in manifest if present) that should go under `/data`.
+  Return the new versionedHome (where the new version of agent has its files, returned as path relative to the top directory)
+- Copy the elastic agent action store and components run directory into the new agent in `<versionedHome>/run`
+- Write the update marker containing the information about the new and old agent version, hash and home in `data` directory
+- Invoke the watcher `elastic-agent watch` command to ensure that the new version of agent works correctly after restart:
+  - we invoke the current agent binary if the new version < 8.13.0 (needed to make sure it supports the paths written in the update marker)
+  - we invoke the new agent binary if the new version > 8.13.0
+- Shutdown current agent and its command components, copy components state once again and restart

internal/pkg/agent/application/paths/common.go

@@ -180,11 +180,16 @@ func ExternalInputs() string {
 
 // Data returns the data directory for Agent
 func Data() string {
+	return DataFrom(Top())
+}
+
+// DataFrom returns the data directory for Agent using the passed directory as top path
+func DataFrom(topDirPath string) string {
 	if unversionedHome {
 		// unversioned means the topPath is the data path
-		return topPath
+		return topDirPath
 	}
-	return filepath.Join(Top(), "data")
+	return filepath.Join(topDirPath, "data")
 }
 
 // Run returns the run directory for Agent