Add test for NIC Passthrough

Signed-off-by: Smit Gardhariya <sgardhariya@microsoft.com>

lisa/sut_orchestrator/baremetal/cluster/idrac.py

@@ -214,11 +214,13 @@ def _wait_for_completion(self, response: Any, timeout: int = 600) -> None:
                 task = response.monitor(self.redfish_instance)
 
         if response.status not in [200, 202, 204]:
+            print(f"response: {response}")
             raise LisaException("Failed to complete task! - status:", response.status)
 
     def _insert_virtual_media(self, iso_http_url: str) -> None:
         self._log.debug("Inserting virtual media...")
         body = {"Image": iso_http_url}
+        print(f"body: {body}, iso_http_url: {iso_http_url}")
         response = self.redfish_instance.post(
             "/redfish/v1/Managers/iDRAC.Embedded.1/VirtualMedia/CD/Actions/"
             "VirtualMedia.InsertMedia",

lisa/sut_orchestrator/libvirt/libvirt_device_pool.py

@@ -186,24 +186,78 @@ def create_device_pool(
             vendor_id=vendor_id,
             device_id=device_id,
         )
-        primary_nic_iommu = self.get_primary_nic_id()
-        for item in device_list:
-            device = DeviceAddressSchema()
-            domain, bus, slot, fn = self._parse_pci_address_str(addr=item.slot)
-            device.domain = domain
-            device.bus = bus
-            device.slot = slot
-            device.function = fn
+        bdf_list = [i.slot for i in device_list]
+        self._create_pool(pool_type, bdf_list)
+
+    def create_device_pool_from_pci_addresses(
+        self,
+        pool_type: HostDevicePoolType,
+        pci_addr_list: List[str],
+    ) -> None:
+        self.available_host_devices[pool_type] = {}
+        for bdf in pci_addr_list:
+            domain, bus, slot, fn = self._parse_pci_address_str(bdf)
+            device = self._get_pci_address_instance(domain, bus, slot, fn)
             iommu_group = self._get_device_iommu_group(device)
-            is_vfio_pci = self._is_driver_vfio_pci(device)
 
-            if not is_vfio_pci and iommu_group not in primary_nic_iommu:
+            # Get all the devices of that iommu group
+            iommu_path = f"/sys/kernel/iommu_groups/{iommu_group}/devices"
+            bdf_list = [i.strip() for i in self.host_node.tools[Ls].list(iommu_path)]
+            bdf_list.append(bdf.strip())  # append the given device in list
+
+            self._create_pool(pool_type, bdf_list)
+
+    def _create_pool(
+        self,
+        pool_type: HostDevicePoolType,
+        bdf_list: List[str],
+    ) -> None:
+        iommu_grp_of_used_devices = []
+        primary_nic_iommu = self.get_primary_nic_id()
+        for bdf in bdf_list:
+            domain, bus, slot, fn = self._parse_pci_address_str(bdf)
+            dev = self._get_pci_address_instance(domain, bus, slot, fn)
+            is_vfio_pci = self._is_driver_vfio_pci(dev)
+            iommu_group = self._get_device_iommu_group(dev)
+
+            if iommu_group in iommu_grp_of_used_devices:
+                # No need to add this device in pool as one of the devices for this
+                # iommu group is in use
+                continue
+
+            if is_vfio_pci:
+                # Do not consider any device for pool if any device of same iommu group
+                # is already assigned
+                pool = self.available_host_devices.get(pool_type, {})
+                pool.pop(iommu_group, [])
+                self.available_host_devices[pool_type] = pool
+                iommu_grp_of_used_devices.append(iommu_group)
+            elif (
+                iommu_group not in primary_nic_iommu
+                and iommu_group not in iommu_grp_of_used_devices
+            ):
                 pool = self.available_host_devices.get(pool_type, {})
                 devices = pool.get(iommu_group, [])
-                devices.append(device)
+                if dev not in devices:
+                    devices.append(dev)
                 pool[iommu_group] = devices
                 self.available_host_devices[pool_type] = pool
 
+    def _get_pci_address_instance(
+        self,
+        domain: str,
+        bus: str,
+        slot: str,
+        fn: str,
+    ) -> DeviceAddressSchema:
+        device = DeviceAddressSchema()
+        device.domain = domain
+        device.bus = bus
+        device.slot = slot
+        device.function = fn
+
+        return device
+
     def _add_device_passthrough_xml(
         self,
         devices: ET.Element,

lisa/sut_orchestrator/libvirt/platform.py

@@ -820,6 +820,10 @@ def _fill_nodes_metadata(self, environment: Environment, log: Logger) -> None:
             )
 
             node_context = get_node_context(node)
+            # if self.host_node.is_remote:
+            #     node_context.host_node = remote_node
+            # else:
+            #     node_context.host_node = self.host_node
             if node_context.init_system == InitSystem.CLOUD_INIT:
                 # Ensure cloud-init completes its setup.
                 node.execute(

lisa/sut_orchestrator/libvirt/schema.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import List, Optional, Union
+from typing import Any, List, Optional, Union
 
 from dataclasses_json import dataclass_json
 
@@ -48,6 +48,16 @@ class DeviceAddressSchema:
     slot: str = ""
     function: str = ""
 
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, DeviceAddressSchema):
+            return (
+                self.domain == other.domain
+                and self.bus == other.bus
+                and self.slot == other.slot
+                and self.function == other.function
+            )
+        return False
+
 
 # QEMU orchestrator's global configuration options.
 @dataclass_json()

lisa/sut_orchestrator/util/device_pool.py

@@ -1,6 +1,10 @@
-from typing import Any, List, Optional
+from typing import Any, List, Optional, cast
 
-from lisa.sut_orchestrator.util.schema import HostDevicePoolSchema, HostDevicePoolType
+from lisa.sut_orchestrator.util.schema import (
+    HostDevicePoolSchema,
+    HostDevicePoolType,
+    VendorDeviceIdIdentifier,
+)
 from lisa.util import LisaException
 
 
@@ -16,6 +20,13 @@ def create_device_pool(
     ) -> None:
         raise NotImplementedError()
 
+    def create_device_pool_from_pci_addresses(
+        self,
+        pool_type: HostDevicePoolType,
+        pci_addr_list: List[str],
+    ) -> None:
+        raise NotImplementedError()
+
     def get_primary_nic_id(self) -> List[str]:
         raise NotImplementedError()
 
@@ -44,22 +55,40 @@ def configure_device_passthrough_pool(
                         f"Pool type '{pool_type}' is not supported by platform"
                     )
             for config in device_configs:
-                vendor_device_list = config.devices
-                if len(vendor_device_list) > 1:
-                    raise LisaException(
-                        "Device Pool does not support more than one "
-                        "vendor/device id list for given pool type"
-                    )
+                devices = config.devices
+                if isinstance(devices, list) and all(
+                    isinstance(d, VendorDeviceIdIdentifier) for d in devices
+                ):
+                    if len(devices) > 1:
+                        raise LisaException(
+                            "Device Pool does not support more than one "
+                            "vendor/device id list for given pool type"
+                        )
+
+                    vendor_device_id = devices[0]
+                    assert vendor_device_id.vendor_id.strip()
+                    vendor_id = vendor_device_id.vendor_id.strip()
 
-                vendor_device_id = vendor_device_list[0]
-                assert vendor_device_id.vendor_id.strip()
-                vendor_id = vendor_device_id.vendor_id.strip()
+                    assert vendor_device_id.device_id.strip()
+                    device_id = vendor_device_id.device_id.strip()
 
-                assert vendor_device_id.device_id.strip()
-                device_id = vendor_device_id.device_id.strip()
+                    self.create_device_pool(
+                        pool_type=config.type,
+                        vendor_id=vendor_id,
+                        device_id=device_id,
+                    )
+                elif isinstance(devices, dict):
+                    bdf_list = devices.get("pci_bdf", [])
+                    assert bdf_list, "Key not found: 'pci_bdf'"
+                    pci_addr_list: List[str] = cast(List[str], bdf_list)
 
-                self.create_device_pool(
-                    pool_type=config.type,
-                    vendor_id=vendor_id,
-                    device_id=device_id,
-                )
+                    # Create pool from the list of PCI addresses
+                    self.create_device_pool_from_pci_addresses(
+                        pool_type=config.type,
+                        pci_addr_list=pci_addr_list,
+                    )
+                else:
+                    raise LisaException(
+                        f"Unknown device identifier of type: {type(devices)}"
+                        f", value: {devices}"
+                    )

lisa/sut_orchestrator/util/schema.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import List
+from typing import List, Union, cast
 
 from dataclasses_json import dataclass_json
 
@@ -12,17 +12,26 @@ class HostDevicePoolType(Enum):
 
 @dataclass_json()
 @dataclass
-class DeviceIdentifier:
+class VendorDeviceIdIdentifier:
     vendor_id: str = ""
     device_id: str = ""
 
 
+@dataclass_json()
+@dataclass
+class PciAddressIdentifier:
+    # list of bdf like 0000:3b:00.0 - <domain>:<bus>:<slot>.<fn>
+    pci_bdf: List[str] = field(default_factory=list)
+
+
 # Configuration options for device-passthrough for the VM.
 @dataclass_json()
 @dataclass
 class HostDevicePoolSchema:
     type: HostDevicePoolType = HostDevicePoolType.PCI_NIC
-    devices: List[DeviceIdentifier] = field(default_factory=list)
+    devices: Union[List[VendorDeviceIdIdentifier], PciAddressIdentifier] = field(
+        default_factory=lambda: cast(List[VendorDeviceIdIdentifier], [])
+    )
 
 
 @dataclass_json()

lisa/tools/iperf3.py

@@ -119,6 +119,7 @@ def run_as_server_async(
         use_json_format: bool = False,
         one_connection_only: bool = False,
         daemon: bool = True,
+        interface_ip: str = "",
     ) -> Process:
         # -s: run iperf3 as server mode
         # -D: run iperf3 as a daemon
@@ -135,6 +136,8 @@ def run_as_server_async(
             cmd += f" -f {report_unit} "
         if port:
             cmd += f" -p {port} "
+        if interface_ip:
+            cmd += f" -B {interface_ip}"
         process = self.node.execute_async(
             f"{self.command} {cmd}", shell=True, sudo=True
         )

lisa/tools/netperf.py

@@ -31,10 +31,17 @@ def can_install(self) -> bool:
     def dependencies(self) -> List[Type[Tool]]:
         return [Gcc, Git, Make, Texinfo]
 
-    def run_as_server(self, port: int = 30000, daemon: bool = True) -> None:
+    def run_as_server(
+        self,
+        port: int = 30000,
+        daemon: bool = True,
+        interface_ip: str = "",
+    ) -> None:
         cmd = f"netserver -p {port} "
         if not daemon:
             cmd += " -D "
+        if interface_ip:
+            cmd += f" -L {interface_ip}"
         self.node.execute(
             cmd,
             sudo=True,
@@ -50,9 +57,11 @@ def run_as_client_async(
         test_name: str = "TCP_RR",
         seconds: int = 150,
         time_unit: int = 1,
+        interface_ip: str = "",
         send_recv_offset: str = "THROUGHPUT, THROUGHPUT_UNITS, MIN_LATENCY, MAX_LATENCY, MEAN_LATENCY, REQUEST_SIZE, RESPONSE_SIZE, STDDEV_LATENCY",  # noqa: E501
     ) -> Process:
         # -H: Specify the target machine and/or local ip and family
+        # -L: Specify the IP for client interface to be used
         # -p: Specify netserver port number and/or local port
         # -t: Specify test to perform
         # -n: Set the number of processors for CPU util
@@ -61,8 +70,11 @@ def run_as_client_async(
         #     initial guess for units per second. A negative value for time will make
         #     heavy use of the system's timestamping functionality
         # -O: Set the remote send,recv buffer offset
-        cmd = (
-            f"-H {server_ip} -p {port} -t {test_name} -n {core_count} -l {seconds}"
+        cmd: str = ""
+        if interface_ip:
+            cmd += f" -L {interface_ip}"
+        cmd += (
+            f" -H {server_ip} -p {port} -t {test_name} -n {core_count} -l {seconds}"
             f" -D {time_unit} -- -O '{send_recv_offset}'"
         )
         process = self.node.execute_async(f"{self.command} {cmd}", sudo=True)