addressed comments

azurelinuxagent/common/conf.py

@@ -479,8 +479,8 @@ def get_autoupdate_enabled(conf=__conf__):
     return conf.get_switch("AutoUpdate.Enabled", True)
 
 
-def get_autoupdate_frequency(conf=__conf__):
-    return conf.get_int("Autoupdate.Frequency", 3600)
+def get_agentupdate_frequency(conf=__conf__):
+    return conf.get_int("AgentUpdate.Frequency", 3600)
 
 
 def get_enable_overprovisioning(conf=__conf__):

azurelinuxagent/common/protocol/extensions_goal_state_from_extensions_config.py

@@ -65,7 +65,9 @@ def _parse_extensions_config(self, xml_text, wire_client):
             is_vm_enabled_for_rsm_upgrades = findtext(ga_family, "IsVMEnabledForRSMUpgrades")
             uris_list = find(ga_family, "Uris")
             uris = findall(uris_list, "Uri")
-            family = VMAgentFamily(name, version)
+            family = VMAgentFamily(name)
+            if version is not None:
+                family.version = version
             if is_version_from_rsm is not None:
                 family.is_version_from_rsm = is_version_from_rsm.lower() == "true"
             if is_vm_enabled_for_rsm_upgrades is not None:

azurelinuxagent/common/protocol/extensions_goal_state_from_vm_settings.py

@@ -274,7 +274,9 @@ def _parse_agent_manifests(self, vm_settings):
             uris = family.get("uris")
             if uris is None:
                 uris = []
-            agent_family = VMAgentFamily(name, version)
+            agent_family = VMAgentFamily(name)
+            if version is not None:
+                agent_family.version = version
             if is_version_from_rsm is not None:
                 agent_family.is_version_from_rsm = is_version_from_rsm
             if is_vm_enabled_for_rsm_upgrades is not None:

azurelinuxagent/common/protocol/restapi.py

@@ -68,10 +68,10 @@ def __init__(self):
 
 
 class VMAgentFamily(object):
-    def __init__(self, name, version):
+    def __init__(self, name):
         self.name = name
         # This is the version as specified by the Goal State
-        self.version = version
+        self.version = None
         # Set to None if the property not specified in the GS and later computed True/False based on previous state in agent update
         self.is_version_from_rsm = None
         # Set to None if this property not specified in the GS and later computed True/False based on previous state in agent update

azurelinuxagent/ga/ga_version_updater.py

@@ -0,0 +1,157 @@
+# Microsoft Azure Linux Agent
+#
+# Copyright 2020 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requires Python 2.6+ and Openssl 1.0+
+
+import glob
+import os
+import shutil
+
+from azurelinuxagent.common import conf, logger
+from azurelinuxagent.common.exception import AgentUpdateError
+from azurelinuxagent.common.future import ustr
+from azurelinuxagent.common.protocol.extensions_goal_state import GoalStateSource
+from azurelinuxagent.common.utils import fileutil
+from azurelinuxagent.common.utils.flexible_version import FlexibleVersion
+from azurelinuxagent.common.version import AGENT_NAME, AGENT_DIR_PATTERN
+from azurelinuxagent.ga.guestagent import GuestAgent
+
+
+class VMEnabledRSMUpdates(TypeError):
+    """
+    Thrown when agent needs to switch to RSM update mode if vm turn on RSM updates
+    """
+
+
+class VMDisabledRSMUpdates(TypeError):
+    """
+    Thrown when agent needs to switch to self update mode if vm turn off RSM updates
+    """
+
+
+class GAVersionUpdater(object):
+
+    def __init__(self, gs_id):
+        self._gs_id = gs_id
+        self._version = FlexibleVersion("0.0.0.0")  # Initialize to zero and retrieve from goal state later stage
+        self._agent_manifest = None  # Initialize to None and fetch from goal state at different stage for different updater
+
+    def is_update_allowed_this_time(self):
+        """
+        This function checks if we allowed to update the agent.
+        return false when we don't allow updates.
+        """
+        raise NotImplementedError
+
+    def check_and_switch_updater_if_changed(self, agent_family, gs_id):
+        """
+        checks and raise the updater exception if we need to switch to self-update from rsm update or vice versa
+        @param agent_family: goal state agent family
+        @param gs_id: incarnation of the goal state
+        @return: VMDisabledRSMUpdates: raise when agent need to stop rsm updates and switch to self-update
+                 VMEnabledRSMUpdates: raise when agent need to switch to rsm update
+        """
+        raise NotImplementedError
+
+    def retrieve_agent_version(self, agent_family, goal_state):
+        """
+        This function fetches the agent version from the goal state for the given family.
+        @param agent_family: goal state agent family
+        @param goal_state: goal state
+        """
+        raise NotImplementedError
+
+    def is_retrieved_version_allowed_to_update(self, goal_state):
+        """
+        Checks all base condition if new version allow to update.
+        @param goal_state: goal state
+        @return: True if allowed to update else False
+        """
+        raise NotImplementedError
+
+    def log_new_agent_update_message(self):
+        """
+        This function logs the update message after we check agent allowed to update.
+        """
+        raise NotImplementedError
+
+    def purge_extra_agents_from_disk(self):
+        """
+        Method remove the extra agents from disk.
+        """
+        raise NotImplementedError
+
+    def proceed_with_update(self):
+        """
+        performs upgrade/downgrade
+        @return: AgentUpgradeExitException
+        """
+        raise NotImplementedError
+
+    @property
+    def version(self):
+        """
+        Return version
+        """
+        return self._version
+
+    def download_and_get_new_agent(self, protocol, agent_family, goal_state):
+        """
+        Function downloads the new agent and returns the downloaded version.
+        @param protocol: protocol object
+        @param agent_family: agent family
+        @param goal_state: goal state
+        @return: GuestAgent: downloaded agent
+        """
+        if self._agent_manifest is None:  # Fetch agent manifest if it's not already done
+            self._agent_manifest = goal_state.fetch_agent_manifest(agent_family.name, agent_family.uris)
+        package_to_download = self._get_agent_package_to_download(self._agent_manifest, self._version)
+        is_fast_track_goal_state = goal_state.extensions_goal_state.source == GoalStateSource.FastTrack
+        agent = GuestAgent.from_agent_package(package_to_download, protocol, is_fast_track_goal_state)
+        return agent
+
+    def _get_agent_package_to_download(self, agent_manifest, version):
+        """
+        Returns the package of the given Version found in the manifest. If not found, returns exception
+        """
+        for pkg in agent_manifest.pkg_list.versions:
+            if FlexibleVersion(pkg.version) == version:
+                # Found a matching package, only download that one
+                return pkg
+
+        raise AgentUpdateError("No matching package found in the agent manifest for version: {0} in goal state incarnation: {1}, "
+                        "skipping agent update".format(str(version), self._gs_id))
+
+    @staticmethod
+    def _purge_unknown_agents_from_disk(known_agents):
+        """
+        Remove from disk all directories and .zip files of unknown agents
+        """
+        path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME))
+
+        for agent_path in glob.iglob(path):
+            try:
+                name = fileutil.trim_ext(agent_path, "zip")
+                m = AGENT_DIR_PATTERN.match(name)
+                if m is not None and FlexibleVersion(m.group(1)) not in known_agents:
+                    if os.path.isfile(agent_path):
+                        logger.info(u"Purging outdated Agent file {0}", agent_path)
+                        os.remove(agent_path)
+                    else:
+                        logger.info(u"Purging outdated Agent directory {0}", agent_path)
+                        shutil.rmtree(agent_path)
+            except Exception as e:
+                logger.warn(u"Purging {0} raised exception: {1}", agent_path, ustr(e))

azurelinuxagent/ga/rsm_version_updater.py

@@ -0,0 +1,159 @@
+# Microsoft Azure Linux Agent
+#
+# Copyright 2020 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requires Python 2.6+ and Openssl 1.0+
+
+import datetime
+import glob
+import os
+
+from azurelinuxagent.common import conf, logger
+from azurelinuxagent.common.event import add_event, WALAEventOperation
+from azurelinuxagent.common.exception import AgentUpgradeExitException, AgentUpdateError
+from azurelinuxagent.common.utils.flexible_version import FlexibleVersion
+from azurelinuxagent.common.version import CURRENT_VERSION, AGENT_NAME
+from azurelinuxagent.ga.ga_version_updater import GAVersionUpdater, VMDisabledRSMUpdates
+from azurelinuxagent.ga.guestagent import GuestAgent
+
+
+class RSMVersionUpdater(GAVersionUpdater):
+    def __init__(self, gs_id, daemon_version, last_attempted_rsm_version_update_time):
+        super(RSMVersionUpdater, self).__init__(gs_id)
+        self._daemon_version = daemon_version
+        self._last_attempted_rsm_version_update_time = last_attempted_rsm_version_update_time
+
+    @staticmethod
+    def _get_all_agents_on_disk():
+        path = os.path.join(conf.get_lib_dir(), "{0}-*".format(AGENT_NAME))
+        return [GuestAgent.from_installed_agent(path=agent_dir) for agent_dir in glob.iglob(path) if
+                os.path.isdir(agent_dir)]
+
+    def _get_available_agents_on_disk(self):
+        available_agents = [agent for agent in self._get_all_agents_on_disk() if agent.is_available]
+        return sorted(available_agents, key=lambda agent: agent.version, reverse=True)
+
+    def is_update_allowed_this_time(self):
+        """
+        update is allowed once per (as specified in the conf.get_autoupdate_frequency())
+        If update allowed, we update the last_attempted_rsm_version_update_time to current time.
+        """
+        now = datetime.datetime.now()
+
+        if self._last_attempted_rsm_version_update_time != datetime.datetime.min:
+            next_attempt_time = self._last_attempted_rsm_version_update_time + datetime.timedelta(
+                seconds=conf.get_agentupdate_frequency())
+        else:
+            next_attempt_time = now
+
+        if next_attempt_time > now:
+            return False
+        self._last_attempted_rsm_version_update_time = now
+        # The time limit elapsed for us to allow updates.
+        return True
+
+    def check_and_switch_updater_if_changed(self, agent_family, gs_id):
+        """
+        Checks if there is a new goal state and decide if we need to continue with rsm update or switch to self-update.
+        Firstly it checks agent supports GA versioning or not. If not, we raise exception to switch to self-update.
+        if vm is enabled for RSM updates and continue with rsm update, otherwise we raise exception to switch to self-update.
+        if either isVersionFromRSM or isVMEnabledForRSMUpgrades is missing in the goal state, we ignore the update as we consider it as invalid goal state.
+        """
+        if self._gs_id != gs_id:
+            self._gs_id = gs_id
+            if not conf.get_enable_ga_versioning():
+                raise VMDisabledRSMUpdates()
+
+            if agent_family.is_vm_enabled_for_rsm_upgrades is None:
+                raise AgentUpdateError(
+                    "Received invalid goal state:{0}, missing isVMEnabledForRSMUpgrades property. So, skipping agent update".format(
+                        gs_id))
+            elif not agent_family.is_vm_enabled_for_rsm_upgrades:
+                raise VMDisabledRSMUpdates()
+            else:
+                if agent_family.is_version_from_rsm is None:
+                    raise AgentUpdateError(
+                        "Received invalid goal state:{0}, missing isVersionFromRSM property. So, skipping agent update".format(
+                            gs_id))
+
+    def retrieve_agent_version(self, agent_family, goal_state):
+        """
+        Get the agent version from the goal state
+        """
+        if agent_family.version is None and agent_family.is_vm_enabled_for_rsm_upgrades and agent_family.is_version_from_rsm:
+            raise AgentUpdateError(
+                "Received invalid goal state:{0}, missing version property. So, skipping agent update".format(self._gs_id))
+        self._version = FlexibleVersion(agent_family.version)
+
+    def is_retrieved_version_allowed_to_update(self, agent_family):
+        """
+        Once version retrieved from goal state, we check if we allowed to update for that version
+        allow update If new version not same as current version, not below than daemon version and if version is from rsm request
+        """
+
+        if not agent_family.is_version_from_rsm or self._version < self._daemon_version or self._version == CURRENT_VERSION:
+            return False
+
+        return True
+
+    def log_new_agent_update_message(self):
+        """
+        This function logs the update message after we check version allowed to update.
+        """
+        msg = "New agent version:{0} requested by RSM in Goal state {1}, will update the agent before processing the goal state.".format(
+            str(self._version), self._gs_id)
+        logger.info(msg)
+        add_event(op=WALAEventOperation.AgentUpgrade, message=msg, log_event=False)
+
+    def purge_extra_agents_from_disk(self):
+        """
+        Remove the agents( including rsm version if exists) from disk except current version. There is a chance that rsm version could exist and/or blacklisted
+        on previous update attempts. So we should remove it from disk in order to honor current rsm version update.
+        """
+        known_agents = [CURRENT_VERSION]
+        self._purge_unknown_agents_from_disk(known_agents)
+
+    def proceed_with_update(self):
+        """
+        upgrade/downgrade to the new version.
+        Raises: AgentUpgradeExitException
+        """
+        if self._version < CURRENT_VERSION:
+            # In case of a downgrade, we mark the current agent as bad version to avoid starting it back up ever again
+            # (the expectation here being that if we get request to a downgrade,
+            # there's a good reason for not wanting the current version).
+            prefix = "downgrade"
+            try:
+                # We should always have an agent directory for the CURRENT_VERSION
+                agents_on_disk = self._get_available_agents_on_disk()
+                current_agent = next(agent for agent in agents_on_disk if agent.version == CURRENT_VERSION)
+                msg = "Marking the agent {0} as bad version since a downgrade was requested in the GoalState, " \
+                      "suggesting that we really don't want to execute any extensions using this version".format(
+                    CURRENT_VERSION)
+                logger.info(msg)
+                add_event(op=WALAEventOperation.AgentUpgrade, message=msg, log_event=False)
+                current_agent.mark_failure(is_fatal=True, reason=msg)
+            except StopIteration:
+                logger.warn(
+                    "Could not find a matching agent with current version {0} to blacklist, skipping it".format(
+                        CURRENT_VERSION))
+        else:
+            # In case of an upgrade, we don't need to exclude anything as the daemon will automatically
+            # start the next available highest version which would be the target version
+            prefix = "upgrade"
+        raise AgentUpgradeExitException(
+            "Agent completed all update checks, exiting current process to {0} to the new Agent version {1}".format(
+                prefix,
+                self._version))