Skip to content

Commit

Permalink
Merge pull request ovn-kubernetes#4917 from qinqon/kubevirt-localnet-…
Browse files Browse the repository at this point in the history 
…improve-live-migration

kubevirt, localnet: Reduce live migration downtime.
  • Loading branch information
@trozet
trozet authored Feb 5, 2025
2 parents 73c48d8 + d40191e commit a7ca957
Show file tree
Hide file tree
Showing 8 changed files with 144 additions and 60 deletions.
4 changes: 3 additions & 1 deletion go-controller/pkg/kubevirt/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,9 @@ func IsPodOwnedByVirtualMachine(pod *corev1.Pod) bool {

// IsPodAllowedForMigration determines whether a given pod is eligible for live migration
func IsPodAllowedForMigration(pod *corev1.Pod, netInfo util.NetInfo) bool {
return IsPodOwnedByVirtualMachine(pod) && netInfo.TopologyType() == ovntypes.Layer2Topology
return IsPodOwnedByVirtualMachine(pod) &&
(netInfo.TopologyType() == ovntypes.Layer2Topology ||
netInfo.TopologyType() == ovntypes.LocalnetTopology)
}

func isTargetPodReady(targetPod *corev1.Pod) bool {
Expand Down
23 changes: 14 additions & 9 deletions go-controller/pkg/ovn/base_network_controller_pods.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -566,15 +566,25 @@ func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *kapi.Pod, nadName
return nil, nil, nil, false, err
}

// set addresses on the port
// LSP addresses in OVN are a single space-separated value
lsp.Enabled = enable
if lsp.Enabled != nil {
customFields = append(customFields, libovsdbops.LogicalSwitchPortEnabled)
}

addresses = []string{podAnnotation.MAC.String()}
for _, podIfAddr := range podAnnotation.IPs {
addresses[0] = addresses[0] + " " + podIfAddr.IP.String()
}

lsp.Addresses = addresses
customFields = append(customFields, libovsdbops.LogicalSwitchPortAddresses)
// Skip address configuration if LSP is disabled since it will install
// l2 look up flows that harms some topologies
if lsp.Enabled == nil || *lsp.Enabled {
// set addresses on the port
// LSP addresses in OVN are a single space-separated value

lsp.Addresses = addresses
customFields = append(customFields, libovsdbops.LogicalSwitchPortAddresses)
}

// add external ids
lsp.ExternalIDs = map[string]string{"namespace": pod.Namespace, "pod": "true"}
Expand Down Expand Up @@ -602,11 +612,6 @@ func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *kapi.Pod, nadName
if len(lsp.Options) != 0 {
customFields = append(customFields, libovsdbops.LogicalSwitchPortOptions)
}

lsp.Enabled = enable
if lsp.Enabled != nil {
customFields = append(customFields, libovsdbops.LogicalSwitchPortEnabled)
}
ops, err = libovsdbops.CreateOrUpdateLogicalSwitchPortsOnSwitchWithCustomFieldsOps(bnc.nbClient, nil, ls, customFields, lsp)
if err != nil {
return nil, nil, nil, false,
Expand Down
39 changes: 30 additions & 9 deletions go-controller/pkg/ovn/base_network_controller_secondary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,9 @@ func (bsnc *BaseSecondaryNetworkController) addLogicalPortToNetworkForNAD(pod *c
}

if shouldHandleLiveMigration &&
kubevirtLiveMigrationStatus.IsTargetDomainReady() {
kubevirtLiveMigrationStatus.IsTargetDomainReady() &&
// At localnet there is no source pod remote LSP so it should be skipped
(bsnc.TopologyType() != types.LocalnetTopology || bsnc.isPodScheduledinLocalZone(kubevirtLiveMigrationStatus.SourcePod)) {
ops, err = bsnc.disableLiveMigrationSourceLSPOps(kubevirtLiveMigrationStatus, nadName, ops)
if err != nil {
return fmt.Errorf("failed to create LSP ops for source pod during Live-migration status: %w", err)
Expand Down Expand Up @@ -444,7 +446,7 @@ func (bsnc *BaseSecondaryNetworkController) removePodForSecondaryNetwork(pod *co
}

var alreadyProcessed bool
for nadName := range podNetworks {
for nadName, podAnnotation := range podNetworks {
if !bsnc.HasNAD(nadName) {
continue
}
Expand All @@ -471,7 +473,7 @@ func (bsnc *BaseSecondaryNetworkController) removePodForSecondaryNetwork(pod *co
}

if kubevirt.IsPodAllowedForMigration(pod, bsnc.GetNetInfo()) {
if err = bsnc.enableSourceLSPFailedLiveMigration(pod, nadName); err != nil {
if err = bsnc.enableSourceLSPFailedLiveMigration(pod, nadName, podAnnotation.MAC, podAnnotation.IPs); err != nil {
return err
}
}
Expand Down Expand Up @@ -896,15 +898,34 @@ func (bsnc *BaseSecondaryNetworkController) requireDHCP(pod *corev1.Pod) bool {
bsnc.TopologyType() == types.Layer2Topology
}

func (bsnc *BaseSecondaryNetworkController) setPodLogicalSwitchPortEnabledField(
pod *corev1.Pod, nadName string, ops []ovsdb.Operation, enabled bool) ([]ovsdb.Operation, *nbdb.LogicalSwitchPort, error) {
func (bsnc *BaseSecondaryNetworkController) setPodLogicalSwitchPortAddressesAndEnabledField(
pod *corev1.Pod, nadName string, mac string, ips []string, enabled bool, ops []ovsdb.Operation) ([]ovsdb.Operation, *nbdb.LogicalSwitchPort, error) {
lsp := &nbdb.LogicalSwitchPort{Name: bsnc.GetLogicalPortName(pod, nadName)}
lsp.Enabled = ptr.To(enabled)
customFields := []libovsdbops.ModelUpdateField{
libovsdbops.LogicalSwitchPortEnabled,
libovsdbops.LogicalSwitchPortAddresses,
}
if !enabled {
lsp.Addresses = nil
} else {
if len(mac) == 0 || len(ips) == 0 {
return nil, nil, fmt.Errorf("failed to configure addresses for lsp, missing mac and ips for pod %s", pod.Name)
}

// Remove length
for i, ip := range ips {
ips[i] = strings.Split(ip, "/")[0]
}

lsp.Addresses = []string{
strings.Join(append([]string{mac}, ips...), " "),
}
}
switchName, err := bsnc.getExpectedSwitchName(pod)
if err != nil {
return nil, nil, fmt.Errorf("failed to fetch switch name for pod %s: %w", pod.Name, err)
}
customFields := []libovsdbops.ModelUpdateField{libovsdbops.LogicalSwitchPortEnabled}
ops, err = libovsdbops.UpdateLogicalSwitchPortsOnSwitchWithCustomFieldsOps(bsnc.nbClient, ops, &nbdb.LogicalSwitch{Name: switchName}, customFields, lsp)
if err != nil {
return nil, nil, fmt.Errorf("failed updating logical switch port %+v on switch %s: %w", *lsp, switchName, err)
Expand All @@ -916,11 +937,11 @@ func (bsnc *BaseSecondaryNetworkController) disableLiveMigrationSourceLSPOps(
kubevirtLiveMigrationStatus *kubevirt.LiveMigrationStatus,
nadName string, ops []ovsdb.Operation) ([]ovsdb.Operation, error) {
// closing the sourcePod lsp to ensure traffic goes to the now ready targetPod.
ops, _, err := bsnc.setPodLogicalSwitchPortEnabledField(kubevirtLiveMigrationStatus.SourcePod, nadName, ops, false)
ops, _, err := bsnc.setPodLogicalSwitchPortAddressesAndEnabledField(kubevirtLiveMigrationStatus.SourcePod, nadName, "", nil, false, ops)
return ops, err
}

func (bsnc *BaseSecondaryNetworkController) enableSourceLSPFailedLiveMigration(pod *corev1.Pod, nadName string) error {
func (bsnc *BaseSecondaryNetworkController) enableSourceLSPFailedLiveMigration(pod *corev1.Pod, nadName string, mac string, ips []string) error {
kubevirtLiveMigrationStatus, err := kubevirt.DiscoverLiveMigrationStatus(bsnc.watchFactory, pod)
if err != nil {
return fmt.Errorf("failed to discover Live-migration status after pod termination: %w", err)
Expand All @@ -931,7 +952,7 @@ func (bsnc *BaseSecondaryNetworkController) enableSourceLSPFailedLiveMigration(p
return nil
}
// make sure sourcePod lsp is enabled if migration failed after DomainReady was set.
ops, sourcePodLsp, err := bsnc.setPodLogicalSwitchPortEnabledField(kubevirtLiveMigrationStatus.SourcePod, nadName, nil, true)
ops, sourcePodLsp, err := bsnc.setPodLogicalSwitchPortAddressesAndEnabledField(kubevirtLiveMigrationStatus.SourcePod, nadName, mac, ips, true, nil)
if err != nil {
return fmt.Errorf("failed to set source Pod lsp to enabled after migration failed: %w", err)
}
Expand Down
27 changes: 23 additions & 4 deletions go-controller/pkg/ovn/multihoming_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,9 @@ func (em *secondaryNetworkExpectationMachine) expectedLogicalSwitchesAndPortsWit
lsp := newExpectedSwitchPort(lspUUID, portName, podAddr, pod, ocInfo.bnc, nad)
if expectedPodLspEnabled != nil {
lsp.Enabled = expectedPodLspEnabled[pod.podName]
if lsp.Enabled != nil && !*lsp.Enabled {
lsp.Addresses = nil
}
}

if pod.noIfaceIdVer {
Expand Down Expand Up @@ -429,6 +432,13 @@ func enableICFeatureConfig() *config.OVNKubernetesFeatureConfig {
return featConfig
}

func enableNonICFeatureConfig() *config.OVNKubernetesFeatureConfig {
featConfig := minimalFeatureConfig()
featConfig.EnableInterconnect = false
featConfig.EnablePersistentIPs = true
return featConfig
}

type testConfigOpt = func(*testConfiguration)

func icClusterTestConfiguration(opts ...testConfigOpt) testConfiguration {
Expand All @@ -443,7 +453,9 @@ func icClusterTestConfiguration(opts ...testConfigOpt) testConfiguration {
}

func nonICClusterTestConfiguration(opts ...testConfigOpt) testConfiguration {
config := testConfiguration{}
config := testConfiguration{
configToOverride: enableNonICFeatureConfig(),
}
for _, opt := range opts {
opt(&config)
}
Expand All @@ -464,8 +476,14 @@ func newMultiHomedKubevirtPod(vmName string, liveMigrationInfo liveMigrationPodI
func newMultiHomedPod(testPod testPod, multiHomingConfigs ...secondaryNetInfo) *v1.Pod {
pod := newPod(testPod.namespace, testPod.podName, testPod.nodeName, testPod.podIP)
var secondaryNetworks []nadapi.NetworkSelectionElement
if len(pod.Annotations) == 0 {
pod.Annotations = map[string]string{}
}
for _, multiHomingConf := range multiHomingConfigs {
if multiHomingConf.isPrimary {
if multiHomingConf.ipamClaimReference != "" {
pod.Annotations[util.OvnUDNIPAMClaimName] = multiHomingConf.ipamClaimReference
}
continue // these will be automatically plugged in
}
nadNamePair := strings.Split(multiHomingConf.nadName, "/")
Expand All @@ -476,13 +494,14 @@ func newMultiHomedPod(testPod testPod, multiHomingConfigs ...secondaryNetInfo) *
attachmentName = nadNamePair[1]
}
nse := nadapi.NetworkSelectionElement{
Name: attachmentName,
Namespace: ns,
Name: attachmentName,
Namespace: ns,
IPAMClaimReference: multiHomingConf.ipamClaimReference,
}
secondaryNetworks = append(secondaryNetworks, nse)
}
serializedNetworkSelectionElements, _ := json.Marshal(secondaryNetworks)
pod.Annotations = map[string]string{nadapi.NetworkAttachmentAnnot: string(serializedNetworkSelectionElements)}
pod.Annotations[nadapi.NetworkAttachmentAnnot] = string(serializedNetworkSelectionElements)
if config.OVNKubernetesFeature.EnableInterconnect {
dummyOVNNetAnnotations := dummyOVNPodNetworkAnnotations(testPod.secondaryPodInfos, multiHomingConfigs)
if dummyOVNNetAnnotations != "{}" {
Expand Down
8 changes: 7 additions & 1 deletion go-controller/pkg/ovn/ovn_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/onsi/ginkgo/v2"

ipamclaimsapi "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1"
fakeipamclaimclient "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1/apis/clientset/versioned/fake"
mnpapi "github.com/k8snetworkplumbingwg/multi-networkpolicy/pkg/apis/k8s.cni.cncf.io/v1beta1"
mnpfake "github.com/k8snetworkplumbingwg/multi-networkpolicy/pkg/client/clientset/versioned/fake"
Expand Down Expand Up @@ -127,6 +128,7 @@ func (o *FakeOVN) start(objects ...runtime.Object) {
egressServiceObjects := []runtime.Object{}
apbExternalRouteObjects := []runtime.Object{}
anpObjects := []runtime.Object{}
ipamClaimObjects := []runtime.Object{}
v1Objects := []runtime.Object{}
nads := []nettypes.NetworkAttachmentDefinition{}
nadClient := fakenadclient.NewSimpleClientset()
Expand Down Expand Up @@ -158,6 +160,8 @@ func (o *FakeOVN) start(objects ...runtime.Object) {
apbExternalRouteObjects = append(apbExternalRouteObjects, object)
case *anpapi.AdminNetworkPolicyList:
anpObjects = append(anpObjects, object)
case *ipamclaimsapi.IPAMClaimList:
ipamClaimObjects = append(ipamClaimObjects, object)
default:
v1Objects = append(v1Objects, object)
}
Expand All @@ -172,7 +176,7 @@ func (o *FakeOVN) start(objects ...runtime.Object) {
MultiNetworkPolicyClient: mnpfake.NewSimpleClientset(multiNetworkPolicyObjects...),
EgressServiceClient: egressservicefake.NewSimpleClientset(egressServiceObjects...),
AdminPolicyRouteClient: adminpolicybasedroutefake.NewSimpleClientset(apbExternalRouteObjects...),
IPAMClaimsClient: fakeipamclaimclient.NewSimpleClientset(),
IPAMClaimsClient: fakeipamclaimclient.NewSimpleClientset(ipamClaimObjects...),
NetworkAttchDefClient: nadClient,
UserDefinedNetworkClient: udnclientfake.NewSimpleClientset(),
}
Expand Down Expand Up @@ -410,6 +414,7 @@ func NewOvnController(
EgressServiceClient: ovnClient.EgressServiceClient,
APBRouteClient: ovnClient.AdminPolicyRouteClient,
EgressQoSClient: ovnClient.EgressQoSClient,
IPAMClaimsClient: ovnClient.IPAMClaimsClient,
},
wf,
recorder,
Expand Down Expand Up @@ -525,6 +530,7 @@ func (o *FakeOVN) NewSecondaryNetworkController(netattachdef *nettypes.NetworkAt
Kube: kube.Kube{KClient: o.fakeClient.KubeClient},
EIPClient: o.fakeClient.EgressIPClient,
EgressFirewallClient: o.fakeClient.EgressFirewallClient,
IPAMClaimsClient: o.fakeClient.IPAMClaimsClient,
},
o.watcher,
o.fakeRecorder,
Expand Down
Loading

0 comments on commit a7ca957

Please sign in to comment.