Use package manifests for upgrade (#4174)

* map paths when upgrading using .tar.gz packages
* use structured output from unpack to identify old and new agents directories
* copy state directories and rotate symlink correctly

internal/pkg/agent/application/upgrade/rollback.go

@@ -33,9 +33,20 @@ const (
 )
 
 // Rollback rollbacks to previous version which was functioning before upgrade.
-func Rollback(ctx context.Context, log *logger.Logger, prevHash string, currentHash string) error {
+func Rollback(ctx context.Context, log *logger.Logger, prevVersionedHome, prevHash string, currentHash string) error {
+	symlinkPath := filepath.Join(paths.Top(), agentName)
+
+	var symlinkTarget string
+	if prevVersionedHome != "" {
+		symlinkTarget = paths.BinaryPath(filepath.Join(paths.Top(), prevVersionedHome), agentName)
+	} else {
+		// fallback for upgrades that didn't use the manifest and path remapping
+		hashedDir := fmt.Sprintf("%s-%s", agentName, prevHash)
+		// paths.BinaryPath properly derives the binary directory depending on the platform. The path to the binary for macOS is inside of the app bundle.
+		symlinkTarget = paths.BinaryPath(filepath.Join(paths.Top(), "data", hashedDir), agentName)
+	}
 	// change symlink
-	if err := ChangeSymlink(ctx, log, prevHash); err != nil {
+	if err := changeSymlinkInternal(log, symlinkPath, symlinkTarget); err != nil {
 		return err
 	}
 
@@ -51,11 +62,11 @@ func Rollback(ctx context.Context, log *logger.Logger, prevHash string, currentH
 	}
 
 	// cleanup everything except version we're rolling back into
-	return Cleanup(log, prevHash, true, true)
+	return Cleanup(log, prevVersionedHome, prevHash, true, true)
 }
 
 // Cleanup removes all artifacts and files related to a specified version.
-func Cleanup(log *logger.Logger, currentHash string, removeMarker bool, keepLogs bool) error {
+func Cleanup(log *logger.Logger, currentVersionedHome, currentHash string, removeMarker bool, keepLogs bool) error {
 	log.Infow("Cleaning up upgrade", "hash", currentHash, "remove_marker", removeMarker)
 	<-time.After(afterRestartDelay)
 
@@ -83,7 +94,16 @@ func Cleanup(log *logger.Logger, currentHash string, removeMarker bool, keepLogs
 	_ = os.Remove(prevSymlink)
 
 	dirPrefix := fmt.Sprintf("%s-", agentName)
-	currentDir := fmt.Sprintf("%s-%s", agentName, currentHash)
+	var currentDir string
+	if currentVersionedHome != "" {
+		currentDir, err = filepath.Rel("data", currentVersionedHome)
+		if err != nil {
+			return fmt.Errorf("extracting elastic-agent path relative to data directory from %s: %w", currentVersionedHome, err)
+		}
+	} else {
+		currentDir = fmt.Sprintf("%s-%s", agentName, currentHash)
+	}
+
 	for _, dir := range subdirs {
 		if dir == currentDir {
 			continue

internal/pkg/agent/application/upgrade/step_mark.go

@@ -6,6 +6,7 @@ package upgrade
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"time"
@@ -24,15 +25,22 @@ const markerFilename = ".update-marker"
 
 // UpdateMarker is a marker holding necessary information about ongoing upgrade.
 type UpdateMarker struct {
+	// Version represents the version the agent is upgraded to
+	Version string `json:"version" yaml:"version"`
 	// Hash agent is updated to
 	Hash string `json:"hash" yaml:"hash"`
+	// VersionedHome represents the path where the new agent is located relative to top path
+	VersionedHome string `json:"versioned_home" yaml:"versioned_home"`
+
 	//UpdatedOn marks a date when update happened
 	UpdatedOn time.Time `json:"updated_on" yaml:"updated_on"`
 
 	// PrevVersion is a version agent is updated from
 	PrevVersion string `json:"prev_version" yaml:"prev_version"`
 	// PrevHash is a hash agent is updated from
 	PrevHash string `json:"prev_hash" yaml:"prev_hash"`
+	// PrevVersionedHome represents the path where the old agent is located relative to top path
+	PrevVersionedHome string `json:"prev_versioned_home" yaml:"prev_versioned_home"`
 
 	// Acked is a flag marking whether or not action was acked
 	Acked  bool                    `json:"acked" yaml:"acked"`
@@ -83,42 +91,55 @@ func convertToActionUpgrade(a *MarkerActionUpgrade) *fleetapi.ActionUpgrade {
 }
 
 type updateMarkerSerializer struct {
-	Hash        string               `yaml:"hash"`
-	UpdatedOn   time.Time            `yaml:"updated_on"`
-	PrevVersion string               `yaml:"prev_version"`
-	PrevHash    string               `yaml:"prev_hash"`
-	Acked       bool                 `yaml:"acked"`
-	Action      *MarkerActionUpgrade `yaml:"action"`
-	Details     *details.Details     `yaml:"details"`
+	Version           string               `yaml:"version"`
+	Hash              string               `yaml:"hash"`
+	VersionedHome     string               `yaml:"versioned_home"`
+	UpdatedOn         time.Time            `yaml:"updated_on"`
+	PrevVersion       string               `yaml:"prev_version"`
+	PrevHash          string               `yaml:"prev_hash"`
+	PrevVersionedHome string               `yaml:"prev_versioned_home"`
+	Acked             bool                 `yaml:"acked"`
+	Action            *MarkerActionUpgrade `yaml:"action"`
+	Details           *details.Details     `yaml:"details"`
 }
 
 func newMarkerSerializer(m *UpdateMarker) *updateMarkerSerializer {
 	return &updateMarkerSerializer{
-		Hash:        m.Hash,
-		UpdatedOn:   m.UpdatedOn,
-		PrevVersion: m.PrevVersion,
-		PrevHash:    m.PrevHash,
-		Acked:       m.Acked,
-		Action:      convertToMarkerAction(m.Action),
-		Details:     m.Details,
+		Version:           m.Version,
+		Hash:              m.Hash,
+		VersionedHome:     m.VersionedHome,
+		UpdatedOn:         m.UpdatedOn,
+		PrevVersion:       m.PrevVersion,
+		PrevHash:          m.PrevHash,
+		PrevVersionedHome: m.PrevVersionedHome,
+		Acked:             m.Acked,
+		Action:            convertToMarkerAction(m.Action),
+		Details:           m.Details,
 	}
 }
 
 // markUpgrade marks update happened so we can handle grace period
-func (u *Upgrader) markUpgrade(_ context.Context, log *logger.Logger, hash string, action *fleetapi.ActionUpgrade, upgradeDetails *details.Details) error {
-	prevVersion := release.Version()
+func (u *Upgrader) markUpgrade(_ context.Context, log *logger.Logger, version, hash, versionedHome string, action *fleetapi.ActionUpgrade, upgradeDetails *details.Details) error {
+	prevVersion := release.VersionWithSnapshot()
 	prevHash := release.Commit()
+	prevVersionedHome, err := filepath.Rel(paths.Top(), paths.Home())
+	if err != nil {
+		return fmt.Errorf("calculating home path relative to top, home: %q top: %q : %w", paths.Home(), paths.Top(), err)
+	}
 	if len(prevHash) > hashLen {
 		prevHash = prevHash[:hashLen]
 	}
 
 	marker := &UpdateMarker{
-		Hash:        hash,
-		UpdatedOn:   time.Now(),
-		PrevVersion: prevVersion,
-		PrevHash:    prevHash,
-		Action:      action,
-		Details:     upgradeDetails,
+		Version:           version,
+		Hash:              hash,
+		VersionedHome:     versionedHome,
+		UpdatedOn:         time.Now(),
+		PrevVersion:       prevVersion,
+		PrevHash:          prevHash,
+		PrevVersionedHome: prevVersionedHome,
+		Action:            action,
+		Details:           upgradeDetails,
 	}
 
 	markerBytes, err := yaml.Marshal(newMarkerSerializer(marker))
@@ -183,13 +204,16 @@ func loadMarker(markerFile string) (*UpdateMarker, error) {
 	}
 
 	return &UpdateMarker{
-		Hash:        marker.Hash,
-		UpdatedOn:   marker.UpdatedOn,
-		PrevVersion: marker.PrevVersion,
-		PrevHash:    marker.PrevHash,
-		Acked:       marker.Acked,
-		Action:      convertToActionUpgrade(marker.Action),
-		Details:     marker.Details,
+		Version:           marker.Version,
+		Hash:              marker.Hash,
+		VersionedHome:     marker.VersionedHome,
+		UpdatedOn:         marker.UpdatedOn,
+		PrevVersion:       marker.PrevVersion,
+		PrevHash:          marker.PrevHash,
+		PrevVersionedHome: marker.PrevVersionedHome,
+		Acked:             marker.Acked,
+		Action:            convertToActionUpgrade(marker.Action),
+		Details:           marker.Details,
 	}, nil
 }
 
@@ -198,13 +222,16 @@ func loadMarker(markerFile string) (*UpdateMarker, error) {
 // file is immediately flushed to persistent storage.
 func SaveMarker(marker *UpdateMarker, shouldFsync bool) error {
 	makerSerializer := &updateMarkerSerializer{
-		Hash:        marker.Hash,
-		UpdatedOn:   marker.UpdatedOn,
-		PrevVersion: marker.PrevVersion,
-		PrevHash:    marker.PrevHash,
-		Acked:       marker.Acked,
-		Action:      convertToMarkerAction(marker.Action),
-		Details:     marker.Details,
+		Version:           marker.Version,
+		Hash:              marker.Hash,
+		VersionedHome:     marker.VersionedHome,
+		UpdatedOn:         marker.UpdatedOn,
+		PrevVersion:       marker.PrevVersion,
+		PrevHash:          marker.PrevHash,
+		PrevVersionedHome: marker.PrevVersionedHome,
+		Acked:             marker.Acked,
+		Action:            convertToMarkerAction(marker.Action),
+		Details:           marker.Details,
 	}
 	markerBytes, err := yaml.Marshal(makerSerializer)
 	if err != nil {

internal/pkg/agent/application/upgrade/step_relink.go

@@ -32,21 +32,26 @@ func ChangeSymlink(ctx context.Context, log *logger.Logger, targetHash string) e
 	// paths.BinaryPath properly derives the binary directory depending on the platform. The path to the binary for macOS is inside of the app bundle.
 	newPath := paths.BinaryPath(filepath.Join(paths.Top(), "data", hashedDir), agentName)
 
+	return changeSymlinkInternal(log, symlinkPath, newPath)
+}
+
+func changeSymlinkInternal(log *logger.Logger, symlinkPath, newTarget string) error {
+
 	// handle windows suffixes
 	if runtime.GOOS == windows {
 		symlinkPath += exe
-		newPath += exe
+		newTarget += exe
 	}
 
 	prevNewPath := prevSymlinkPath()
-	log.Infow("Changing symlink", "symlink_path", symlinkPath, "new_path", newPath, "prev_path", prevNewPath)
+	log.Infow("Changing symlink", "symlink_path", symlinkPath, "new_path", newTarget, "prev_path", prevNewPath)
 
 	// remove symlink to avoid upgrade failures
 	if err := os.Remove(prevNewPath); !os.IsNotExist(err) {
 		return err
 	}
 
-	if err := os.Symlink(newPath, prevNewPath); err != nil {
+	if err := os.Symlink(newTarget, prevNewPath); err != nil {
 		return errors.New(err, errors.TypeFilesystem, "failed to update agent symlink")
 	}