Merge branch 'master' into SubnetAddressSpace_K8sSwiftv2

Signed-off-by: Kshitija Murudi <kmurudi@microsoft.com>

.pipelines/multitenancy/swiftv2-e2e-step-template.yaml

@@ -34,7 +34,7 @@ steps:
         kubectl get po -owide -A
         echo "Apply the pod network yaml to start the delegation"
         less test/integration/manifests/swiftv2/podnetwork.yaml
-        envsubst '${SUBNET_TOKEN},${SUBNET_GUID},${SUBNET_RESOURCE_ID},${VNET_GUID}' < test/integration/manifests/swiftv2/podnetwork.yaml | kubectl apply -f -
+        envsubst '${SUBNET_TOKEN},${SUBNET_RESOURCE_ID},${VNET_GUID}' < test/integration/manifests/swiftv2/podnetwork.yaml | kubectl apply -f -
         echo "Check the podnetwork yaml file"
         less test/integration/manifests/swiftv2/podnetwork.yaml
         kubectl get pn

.pipelines/pipeline.yaml

@@ -525,7 +525,7 @@ stages:
         clusterType: swiftv2-multitenancy-cluster-up
         clusterName: "mtacluster"
         nodePoolName: "mtapool"
-        vmSize: Standard_D4_v2
+        vmSize: $(SWIFTV2_MT_CLUSTER_SKU)
         dependsOn: "test"
         dummyClusterName: "swiftv2dummy"
         dummyClusterType: "swiftv2-dummy-cluster-up"

cns/NetworkContainerContract.go

@@ -481,6 +481,8 @@ type PodIpInfo struct {
 	SkipDefaultRoutes bool
 	// Routes to configure on interface
 	Routes []Route
+	// PnpId is set for backend interfaces, Pnp Id identifies VF. Plug and play id(pnp) is also called as PCI ID
+	PnPID string
 }
 
 type HostIPInfo struct {
@@ -499,12 +501,14 @@ type IPConfigRequest struct {
 
 // Same as IPConfigRequest except that DesiredIPAddresses is passed in as a slice
 type IPConfigsRequest struct {
-	DesiredIPAddresses       []string        `json:"desiredIPAddresses"`
-	PodInterfaceID           string          `json:"podInterfaceID"`
-	InfraContainerID         string          `json:"infraContainerID"`
-	OrchestratorContext      json.RawMessage `json:"orchestratorContext"`
-	Ifname                   string          `json:"ifname"`                   // Used by delegated IPAM
-	SecondaryInterfacesExist bool            `json:"secondaryInterfacesExist"` // will be set by SWIFT v2 validator func
+	DesiredIPAddresses           []string        `json:"desiredIPAddresses"`
+	PodInterfaceID               string          `json:"podInterfaceID"`
+	InfraContainerID             string          `json:"infraContainerID"`
+	OrchestratorContext          json.RawMessage `json:"orchestratorContext"`
+	Ifname                       string          `json:"ifname"`                   // Used by delegated IPAM
+	SecondaryInterfacesExist     bool            `json:"secondaryInterfacesExist"` // will be set by SWIFT v2 validator func
+	BackendInterfaceExist        bool            `json:"BackendInterfaceExist"`    // will be set by SWIFT v2 validator func
+	BackendInterfaceMacAddresses []string        `json:"BacknendInterfaceMacAddress"`
 }
 
 // IPConfigResponse is used in CNS IPAM mode as a response to CNI ADD

cns/configuration/env.go

@@ -14,10 +14,11 @@ const (
 	// LabelNodeSwiftV2 is the Node label for Swift V2
 	LabelNodeSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled"
 	// LabelPodSwiftV2 is the Pod label for Swift V2
-	LabelPodSwiftV2   = "kubernetes.azure.com/pod-network"
-	EnvPodCIDRs       = "POD_CIDRs"
-	EnvServiceCIDRs   = "SERVICE_CIDRs"
-	EnvInfraVNETCIDRs = "INFRA_VNET_CIDRs"
+	LabelPodSwiftV2                = "kubernetes.azure.com/pod-network"
+	LabelPodNetworkInstanceSwiftV2 = "kubernetes.azure.com/pod-network-instance"
+	EnvPodCIDRs                    = "POD_CIDRs"
+	EnvServiceCIDRs                = "SERVICE_CIDRs"
+	EnvInfraVNETCIDRs              = "INFRA_VNET_CIDRs"
 )
 
 // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment.

cns/middlewares/k8sSwiftV2.go

@@ -81,15 +81,18 @@ func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
 		// Set routes for the pod
 		for i := range ipConfigsResp.PodIPInfo {
 			ipInfo := &ipConfigsResp.PodIPInfo[i]
-			err = k.setRoutes(ipInfo)
-			if err != nil {
-				return &cns.IPConfigsResponse{
-					Response: cns.Response{
-						ReturnCode: types.FailedToAllocateIPConfig,
-						Message:    fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
-					},
-					PodIPInfo: []cns.PodIpInfo{},
-				}, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name())
+			// Backend nics doesn't need routes to be set
+			if ipInfo.NICType != cns.BackendNIC {
+				err = k.setRoutes(ipInfo)
+				if err != nil {
+					return &cns.IPConfigsResponse{
+						Response: cns.Response{
+							ReturnCode: types.FailedToAllocateIPConfig,
+							Message:    fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
+						},
+						PodIPInfo: []cns.PodIpInfo{},
+					}, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name())
+				}
 			}
 		}
 		return ipConfigsResp, nil
@@ -114,8 +117,10 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
 	}
 
 	// check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready
-	if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok {
-		req.SecondaryInterfacesExist = true
+	_, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2]
+	_, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2]
+	if swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel {
+
 		// Check if the MTPNC CRD exists for the pod, if not, return error
 		mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
 		mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
@@ -126,8 +131,27 @@ func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
 		if !mtpnc.IsReady() {
 			return nil, types.UnexpectedError, errMTPNCNotReady.Error()
 		}
+		// If primary Ip is set in status field, it indicates the presence of secondary interfaces
+		if mtpnc.Status.PrimaryIP != "" {
+			req.SecondaryInterfacesExist = true
+		}
+		interfaceInfos := mtpnc.Status.InterfaceInfos
+		for _, interfaceInfo := range interfaceInfos {
+			if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC {
+				if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" {
+					return nil, types.UnexpectedError, errMTPNCNotReady.Error()
+				}
+				req.BackendInterfaceExist = true
+				req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress)
+
+			}
+			if interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC {
+				req.SecondaryInterfacesExist = true
+			}
+		}
 	}
 	logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist)
+	logger.Printf("[SWIFTv2Middleware] pod %s has backend interface : %v", podInfo.Name(), req.BackendInterfaceExist)
 	// retrieve podinfo from orchestrator context
 	return podInfo, types.Success, ""
 }

cns/middlewares/k8sSwiftV2_linux_test.go

@@ -31,6 +31,12 @@ var (
 
 	testPod7GUID = "123e4567-e89b-12d3-a456-426614174000"
 	testPod7Info = cns.NewPodInfo("123e45-eth0", testPod7GUID, "testpod7", "testpod7namespace")
+
+	testPod8GUID = "2006cad4-e54d-472e-863d-c4bac66200a7"
+	testPod8Info = cns.NewPodInfo("2006cad4-eth0", testPod8GUID, "testpod8", "testpod8namespace")
+
+	testPod9GUID = "2006cad4-e54d-472e-863d-c4bac66200a7"
+	testPod9Info = cns.NewPodInfo("2006cad4-eth0", testPod9GUID, "testpod9", "testpod9namespace")
 )
 
 func TestMain(m *testing.M) {
@@ -141,6 +147,35 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) {
 	assert.Equal(t, err, "")
 	assert.Equal(t, respCode, types.Success)
 	assert.Equal(t, happyReq.SecondaryInterfacesExist, true)
+
+	happyReq2 := &cns.IPConfigsRequest{
+		PodInterfaceID:   testPod8Info.InterfaceID(),
+		InfraContainerID: testPod8Info.InfraContainerID(),
+	}
+
+	b, _ = testPod8Info.OrchestratorContext()
+	happyReq2.OrchestratorContext = b
+	happyReq2.SecondaryInterfacesExist = false
+
+	_, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq2)
+	assert.Equal(t, err, "")
+	assert.Equal(t, respCode, types.Success)
+	assert.Equal(t, happyReq.SecondaryInterfacesExist, true)
+
+	happyReq3 := &cns.IPConfigsRequest{
+		PodInterfaceID:   testPod9Info.InterfaceID(),
+		InfraContainerID: testPod9Info.InfraContainerID(),
+	}
+
+	b, _ = testPod9Info.OrchestratorContext()
+	happyReq3.OrchestratorContext = b
+	happyReq3.SecondaryInterfacesExist = false
+
+	_, respCode, err = middleware.validateIPConfigsRequest(context.TODO(), happyReq3)
+	assert.Equal(t, err, "")
+	assert.Equal(t, respCode, types.Success)
+	assert.Equal(t, happyReq3.SecondaryInterfacesExist, false)
+	assert.Equal(t, happyReq3.BackendInterfaceExist, true)
 }
 
 func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) {
@@ -373,11 +408,8 @@ func TestNICTypeConfigSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if len(ipInfos) != 1 {
-		t.Fatalf("expected 1 ipInfo, got %d", len(ipInfos))
-	}
-	if ipInfos[0].NICType != cns.BackendNIC {
-		t.Errorf("expected NIC type %v, got %v", cns.BackendNIC, ipInfos[0].NICType)
+	if len(ipInfos) != 0 {
+		t.Fatalf("expected 0 ipInfo, got %d", len(ipInfos))
 	}
 }
 
@@ -404,7 +436,7 @@ func TestGetSWIFTv2IPConfigMultiInterfaceSuccess(t *testing.T) {
 	assert.Equal(t, err, nil)
 	// Ensure that the length of ipInfos matches the number of InterfaceInfos
 	// Adjust this according to the test setup in mock client
-	expectedInterfaceCount := 3
+	expectedInterfaceCount := 2
 	assert.Equal(t, len(ipInfos), expectedInterfaceCount)
 
 	for _, ipInfo := range ipInfos {

cns/middlewares/mock/mockClient.go

@@ -54,6 +54,18 @@ func NewClient() *Client {
 	testPod7.Labels = make(map[string]string)
 	testPod7.Labels[configuration.LabelPodSwiftV2] = podNetwork
 
+	testPod8 := v1.Pod{}
+	testPod8.Labels = make(map[string]string)
+	testPod8.Labels[configuration.LabelPodSwiftV2] = podNetwork
+
+	testPod9 := v1.Pod{}
+	testPod9.Labels = make(map[string]string)
+	testPod9.Labels[configuration.LabelPodSwiftV2] = podNetwork
+
+	testPod10 := v1.Pod{}
+	testPod10.Labels = make(map[string]string)
+	testPod10.Labels[configuration.LabelPodNetworkInstanceSwiftV2] = podNetwork
+
 	testInterfaceInfos1 := v1alpha1.InterfaceInfo{
 		NCID:            "testncid",
 		PrimaryIP:       "192.168.0.1/32",
@@ -110,22 +122,72 @@ func NewClient() *Client {
 		},
 	}
 
+	testMTPNC8 := v1alpha1.MultitenantPodNetworkConfig{
+		Status: v1alpha1.MultitenantPodNetworkConfigStatus{
+			PrimaryIP:  "192.168.0.1/32",
+			MacAddress: "00:00:00:00:00:00",
+			GatewayIP:  "10.0.0.1",
+			NCID:       "testncid",
+			InterfaceInfos: []v1alpha1.InterfaceInfo{
+				{
+					PrimaryIP:  "192.168.0.1/32",
+					MacAddress: "00:00:00:00:00:00",
+					GatewayIP:  "10.0.0.1",
+					NCID:       "testncid",
+					DeviceType: v1alpha1.DeviceTypeVnetNIC,
+				},
+				{
+					PrimaryIP:  "192.168.0.1/32",
+					MacAddress: "00:00:00:00:00:00",
+					GatewayIP:  "10.0.0.1",
+					NCID:       "testncid",
+					DeviceType: v1alpha1.DeviceTypeInfiniBandNIC,
+				},
+			},
+		},
+	}
+	// Mtpnc with just Infiniband interface
+	testMTPNC9 := v1alpha1.MultitenantPodNetworkConfig{
+		Status: v1alpha1.MultitenantPodNetworkConfigStatus{
+			InterfaceInfos: []v1alpha1.InterfaceInfo{
+				{
+					PrimaryIP:  "192.168.0.1/32",
+					MacAddress: "00:00:00:00:00:00",
+					GatewayIP:  "10.0.0.1",
+					NCID:       "testncid",
+					DeviceType: v1alpha1.DeviceTypeInfiniBandNIC,
+				},
+			},
+		},
+	}
+
+	// Mtpnc with just Infiniband interface
+	testMTPNC10 := v1alpha1.MultitenantPodNetworkConfig{
+		Status: v1alpha1.MultitenantPodNetworkConfigStatus{},
+	}
+
 	return &Client{
 		mtPodCache: map[string]*v1.Pod{
-			"testpod1namespace/testpod1": &testPod1,
-			"testpod3namespace/testpod3": &testPod3,
-			"testpod4namespace/testpod4": &testPod4,
-			"testpod5namespace/testpod5": &testPod5,
-			"testpod6namespace/testpod6": &testPod6,
-			"testpod7namespace/testpod7": &testPod7,
+			"testpod1namespace/testpod1":   &testPod1,
+			"testpod3namespace/testpod3":   &testPod3,
+			"testpod4namespace/testpod4":   &testPod4,
+			"testpod5namespace/testpod5":   &testPod5,
+			"testpod6namespace/testpod6":   &testPod6,
+			"testpod7namespace/testpod7":   &testPod7,
+			"testpod8namespace/testpod8":   &testPod8,
+			"testpod9namespace/testpod9":   &testPod9,
+			"testpod10namespace/testpod10": &testPod10,
 		},
 		mtpncCache: map[string]*v1alpha1.MultitenantPodNetworkConfig{
-			"testpod1namespace/testpod1": &testMTPNC1,
-			"testpod2namespace/testpod2": &testMTPNC2,
-			"testpod4namespace/testpod4": &testMTPNC4,
-			"testpod5namespace/testpod5": &testMTPNC3,
-			"testpod6namespace/testpod6": &testMTPNC5,
-			"testpod7namespace/testpod7": &testMTPNCMulti,
+			"testpod1namespace/testpod1":   &testMTPNC1,
+			"testpod2namespace/testpod2":   &testMTPNC2,
+			"testpod4namespace/testpod4":   &testMTPNC4,
+			"testpod5namespace/testpod5":   &testMTPNC3,
+			"testpod6namespace/testpod6":   &testMTPNC5,
+			"testpod7namespace/testpod7":   &testMTPNCMulti,
+			"testpod8namespace/testpod8":   &testMTPNC8,
+			"testpod9namespace/testpod9":   &testMTPNC9,
+			"testpod10namespace/testpod10": &testMTPNC10,
 		},
 	}
 }

cns/restserver/ipam.go

@@ -47,9 +47,30 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context
 		}, errors.New("failed to validate ip config request")
 	}
 
+	var podIPInfoResult []cns.PodIpInfo
+	if ipconfigsRequest.BackendInterfaceExist {
+		for _, bNICMacAddress := range ipconfigsRequest.BackendInterfaceMacAddresses {
+			PnPID, err := service.getPNPIDFromMacAddress(ctx, bNICMacAddress)
+			if err != nil {
+				return &cns.IPConfigsResponse{
+					Response: cns.Response{
+						ReturnCode: types.FailedToAllocateIPConfig,
+						Message:    fmt.Sprintf("BackendNIC allocation failed: %v, config request is %v", err, ipconfigsRequest),
+					},
+					PodIPInfo: []cns.PodIpInfo{},
+				}, err
+			}
+			podBackendInfo := cns.PodIpInfo{
+				MacAddress: bNICMacAddress,
+				NICType:    cns.BackendNIC,
+				PnPID:      PnPID,
+			}
+			podIPInfoResult = append(podIPInfoResult, podBackendInfo)
+		}
+	}
+
 	// record a pod requesting an IP
 	service.podsPendingIPAssignment.Push(podInfo.Key())
-
 	podIPInfo, err := requestIPConfigsHelper(service, ipconfigsRequest) //nolint:contextcheck // appease linter for revert PR
 	if err != nil {
 		return &cns.IPConfigsResponse{
@@ -83,11 +104,12 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context
 		}
 	}
 
+	podIPInfoResult = append(podIPInfoResult, podIPInfo...)
 	return &cns.IPConfigsResponse{
 		Response: cns.Response{
 			ReturnCode: types.Success,
 		},
-		PodIPInfo: podIPInfo,
+		PodIPInfo: podIPInfoResult,
 	}, nil
 }