agent persist firewall scenario (#2983)

* agent persist firewall scenario

* address comments

* new comments

tests_e2e/orchestrator/lib/agent_test_loader.py

@@ -65,6 +65,8 @@ class TestSuiteInfo(object):
     template: str
     # skip test suite if the test not supposed to run on specific clouds
     skip_on_clouds: List[str]
+    # skip test suite if test suite not suppose to run on specific images
+    skip_on_images: List[str]
 
     def __str__(self):
         return self.name
@@ -168,6 +170,12 @@ def _parse_image(image: str) -> str:
                 if suite_skip_cloud not in ["AzureCloud", "AzureChinaCloud", "AzureUSGovernment"]:
                     raise Exception(f"Invalid cloud {suite_skip_cloud} for in {suite.name}")
 
+            # if the suite specifies skip images, validate that images used in our tests
+            for suite_skip_image in suite.skip_on_images:
+                if suite_skip_image not in self.images:
+                    raise Exception(f"Invalid image reference in test suite {suite.name}: Can't find {suite_skip_image} in images.yml")
+
+
     @staticmethod
     def _load_test_suites(test_suites: str) -> List[TestSuiteInfo]:
         #
@@ -205,6 +213,8 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
             owns_vm: true
             install_test_agent: true
             template: "bvts/template.py"
+            skip_on_clouds: "AzureChinaCloud"
+            skip_on_images: "ubuntu_2004"
 
         * name     - A string used to identify the test suite
         * tests    - A list of the tests in the suite. Each test can be specified by a string (the path for its source code relative to
@@ -231,7 +241,9 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
         * skip_on_clouds - [Optional; string or list of strings] If given, the test suite will be skipped in the specified cloud(e.g. "AzureCloud").
                     If not specified, the test suite will be executed in all the clouds that we use. This is useful
                     if you want to skip a test suite validation in a particular cloud when certain feature is not available in that cloud.
-
+        # skip_on_images - [Optional; string or list of strings] If given, the test suite will be skipped on the specified images or image sets(e.g. "ubuntu_2004").
+                    If not specified, the test suite will be executed on all the images that we use. This is useful
+                    if you want to skip a test suite validation on a particular images or image sets when certain feature is not available on that image.
         """
         test_suite: Dict[str, Any] = AgentTestLoader._load_file(description_file)
 
@@ -286,6 +298,15 @@ def _load_test_suite(description_file: Path) -> TestSuiteInfo:
         else:
             test_suite_info.skip_on_clouds = []
 
+        skip_on_images = test_suite.get("skip_on_images")
+        if skip_on_images is not None:
+            if isinstance(skip_on_images, str):
+                test_suite_info.skip_on_images = [skip_on_images]
+            else:
+                test_suite_info.skip_on_images = skip_on_images
+        else:
+            test_suite_info.skip_on_images = []
+
         return test_suite_info
 
     @staticmethod

tests_e2e/orchestrator/lib/agent_test_suite_combinator.py

@@ -140,6 +140,7 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
         runbook_images = self._get_runbook_images(loader)
 
         skip_test_suites: List[str] = []
+        skip_test_suites_images: List[str] = []
         for test_suite_info in loader.test_suites:
             if self.runbook.cloud in test_suite_info.skip_on_clouds:
                 skip_test_suites.append(test_suite_info.name)
@@ -149,7 +150,14 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
             else:
                 images_info: List[VmImageInfo] = self._get_test_suite_images(test_suite_info, loader)
 
+            skip_images_info: List[VmImageInfo] = self._get_test_suite_skip_images(test_suite_info, loader)
+            if len(skip_images_info) > 0:
+                skip_test_suite_image = f"{test_suite_info.name}: {','.join([i.urn for i in skip_images_info])}"
+                skip_test_suites_images.append(skip_test_suite_image)
+
             for image in images_info:
+                if image in skip_images_info:
+                    continue
                 # 'image.urn' can actually be the URL to a VHD if the runbook provided it in the 'image' parameter
                 if self._is_vhd(image.urn):
                     marketplace_image = ""
@@ -238,6 +246,9 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
         if len(skip_test_suites) > 0:
             self._log.info("Skipping test suites %s", skip_test_suites)
 
+        if len(skip_test_suites_images) > 0:
+            self._log.info("Skipping test suits run on images \n %s", '\n'.join([f"\t{skip}" for skip in skip_test_suites_images]))
+
         return environments
 
     def create_existing_vm_environment(self) -> Dict[str, Any]:
@@ -440,6 +451,20 @@ def _get_test_suite_images(suite: TestSuiteInfo, loader: AgentTestLoader) -> Lis
                 unique[i.urn] = i
         return [v for k, v in unique.items()]
 
+    @staticmethod
+    def _get_test_suite_skip_images(suite: TestSuiteInfo, loader: AgentTestLoader) -> List[VmImageInfo]:
+        """
+        Returns images that need to be skipped by the suite.
+
+        A test suite may reference multiple image sets and sets can intersect; this method eliminates any duplicates.
+        """
+        skip_unique: Dict[str, VmImageInfo] = {}
+        for image in suite.skip_on_images:
+            image_list = loader.images[image]
+            for i in image_list:
+                skip_unique[i.urn] = i
+        return [v for k, v in skip_unique.items()]
+
     def _get_location(self, suite_info: TestSuiteInfo, image: VmImageInfo) -> str:
         """
         Returns the location on which the test VM for the given test suite and image should be created.

tests_e2e/orchestrator/runbook.yml

@@ -29,7 +29,7 @@ variable:
   # Test suites to execute
   #
   - name: test_suites
-    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline, ext_sequencing"
+    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline, ext_sequencing, agent_persist_firewall"
 
   #
   # Parameters used to create test VMs

tests_e2e/orchestrator/scripts/agent-service

@@ -43,11 +43,13 @@ if command -v systemctl &> /dev/null; then
   service-stop()   { systemctl stop $1; }
   service-restart()  { systemctl restart $1; }
   service-start()  { systemctl start $1; }
+  service-disable() { systemctl disable $1; }
 else
   service-status() { service $1 status; }
   service-stop()   { service $1 stop; }
   service-restart()   { service $1 restart; }
   service-start()  { service $1 start; }
+  service-disable() { service $1 disable; }
 fi
 
 python=$(get-agent-python)
@@ -83,3 +85,8 @@ if [[ "$cmd" == "status" ]]; then
     echo "Service status..."
     service-status $service_name
 fi
+
+if [[ "$cmd" == "disable" ]]; then
+    echo "Disabling service..."
+    service-disable $service_name
+fi

tests_e2e/test_suites/agent_persist_firewall.yml

@@ -0,0 +1,19 @@
+#
+# Iptable rules that agent add not persisted on reboot. So we use firewalld service if distro supports it otherwise agent creates custom service and only runs on boot before network up.
+# so that attacker will not have room to contact the wireserver
+# This test verifies that either of the service is active. Ensure those rules are added on boot and working as expected.
+#
+name: "AgentPersistFirewall"
+tests:
+  - "agent_persist_firewall/agent_persist_firewall.py"
+images:
+  - "endorsed"
+  - "endorsed-arm64"
+owns_vm: true  # This vm cannot be shared with other tests because it modifies the firewall rules and agent status.
+# agent persist firewall service not running on flatcar distro since agent can't install custom service due to read only filesystem.
+# so skipping the test run on flatcar distro.
+# (2023-11-14T19:04:13.738695Z ERROR ExtHandler ExtHandler Unable to setup the persistent firewall rules: [Errno 30] Read-only file system: '/lib/systemd/system/waagent-network-setup.service)
+skip_on_images:
+  - "flatcar"
+  - "flatcar_arm64"
+  - "debian_9"  # TODO: Reboot is slow on debian_9. Need to investigate further.

tests_e2e/tests/agent_persist_firewall/agent_persist_firewall.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from tests_e2e.tests.lib.agent_test import AgentVmTest
+from tests_e2e.tests.lib.agent_test_context import AgentVmTestContext
+from tests_e2e.tests.lib.logging import log
+from tests_e2e.tests.lib.ssh_client import SshClient
+
+
+class AgentPersistFirewallTest(AgentVmTest):
+    """
+    This test verifies agent setup persist firewall rules using custom network setup service or firewalld service. Ensure those rules are added on boot and working as expected.
+    """
+
+    def __init__(self, context: AgentVmTestContext):
+        super().__init__(context)
+        self._ssh_client: SshClient = self._context.create_ssh_client()
+
+    def run(self):
+        self._test_setup()
+        # Test case 1: After test agent install, verify firewalld or network.setup is running
+        self._verify_persist_firewall_service_running()
+        # Test case 2: Perform reboot and ensure firewall rules added on boot and working as expected
+        self._context.vm.restart(wait_for_boot=True, ssh_client=self._ssh_client)
+        self._verify_persist_firewall_service_running()
+        self._verify_firewall_rules_on_boot("first_boot")
+        # Test case 3: Disable the agent(so that agent won't get started after reboot)
+        # perform reboot and ensure firewall rules added on boot even after agent is disabled
+        self._disable_agent()
+        self._context.vm.restart(wait_for_boot=True, ssh_client=self._ssh_client)
+        self._verify_persist_firewall_service_running()
+        self._verify_firewall_rules_on_boot("second_boot")
+        # Test case 4: perform firewalld rules deletion and ensure deleted rules added back to rule set after agent start
+        self._verify_firewall_rules_readded()
+
+    def _test_setup(self):
+        log.info("Doing test setup")
+        self._run_remote_test(self._ssh_client, f"agent_persist_firewall-test_setup {self._context.username}",
+                              use_sudo=True)
+        log.info("Successfully completed test setup\n")
+
+    def _verify_persist_firewall_service_running(self):
+        log.info("Verifying persist firewall service is running")
+        self._run_remote_test(self._ssh_client, "agent_persist_firewall-verify_persist_firewall_service_running.py",
+                              use_sudo=True)
+        log.info("Successfully verified persist firewall service is running\n")
+
+    def _verify_firewall_rules_on_boot(self, boot_name):
+        log.info("Verifying firewall rules on {0}".format(boot_name))
+        self._run_remote_test(self._ssh_client, f"agent_persist_firewall-verify_firewall_rules_on_boot.py --user {self._context.username} --boot_name {boot_name}",
+                              use_sudo=True)
+        log.info("Successfully verified firewall rules on {0}".format(boot_name))
+
+    def _disable_agent(self):
+        log.info("Disabling agent")
+        self._run_remote_test(self._ssh_client, "agent-service disable", use_sudo=True)
+        log.info("Successfully disabled agent\n")
+
+    def _verify_firewall_rules_readded(self):
+        log.info("Verifying firewall rules readded")
+        self._run_remote_test(self._ssh_client, "agent_persist_firewall-verify_firewalld_rules_readded.py",
+                              use_sudo=True)
+        log.info("Successfully verified firewall rules readded\n")