diff --git a/charts/vsphere-cpi-1.26.2.tgz b/charts/vsphere-cpi-1.26.2.tgz new file mode 100644 index 000000000..23f36611f Binary files /dev/null and b/charts/vsphere-cpi-1.26.2.tgz differ diff --git a/charts/vsphere-cpi/Chart.yaml b/charts/vsphere-cpi/Chart.yaml index 7842de835..4e2fa7eca 100644 --- a/charts/vsphere-cpi/Chart.yaml +++ b/charts/vsphere-cpi/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 -appVersion: 1.26.1 +appVersion: 1.26.2 description: A Helm chart for vSphere Cloud Provider Interface Manager (CPI) name: vsphere-cpi -version: 1.26.1 +version: 1.26.2 keywords: - vsphere - vmware diff --git a/charts/vsphere-cpi/README.md b/charts/vsphere-cpi/README.md index 1ef826d34..0edd6bf2f 100644 --- a/charts/vsphere-cpi/README.md +++ b/charts/vsphere-cpi/README.md @@ -149,7 +149,7 @@ helm repo add vsphere-cpi https://kubernetes.github.io/cloud-provider-vsphere helm repo update # Package CPI Chart -VERSION=1.26.1 +VERSION=1.26.2 cd charts helm package vsphere-cpi --version $VERSION --app-version $VERSION diff --git a/charts/vsphere-cpi/values.yaml b/charts/vsphere-cpi/values.yaml index 58694d2f1..f3a3c612b 100644 --- a/charts/vsphere-cpi/values.yaml +++ b/charts/vsphere-cpi/values.yaml @@ -59,7 +59,7 @@ serviceAccount: daemonset: annotations: {} image: gcr.io/cloud-provider-vsphere/cpi/release/manager - tag: v1.26.1 + tag: v1.26.2 pullPolicy: IfNotPresent dnsPolicy: ClusterFirst cmdline: diff --git a/cluster/images/controller-manager/Dockerfile b/cluster/images/controller-manager/Dockerfile index c14d20b73..9ae3a4a44 100644 --- a/cluster/images/controller-manager/Dockerfile +++ b/cluster/images/controller-manager/Dockerfile @@ -33,7 +33,7 @@ ARG DISTROLESS_IMAGE=gcr.io/distroless/static-debian11@sha256:a01d47d4036cae5a67 FROM ${GOLANG_IMAGE} as builder # This build arg is the version to embed in the CPI binary -ARG VERSION=1.26.1 +ARG VERSION=1.26.2 # This build arg controls the GOPROXY setting ARG GOPROXY diff --git a/docs/book/tutorials/disable-node-deletion.yaml b/docs/book/tutorials/disable-node-deletion.yaml index c60b0b706..c30f13d92 100644 --- a/docs/book/tutorials/disable-node-deletion.yaml +++ b/docs/book/tutorials/disable-node-deletion.yaml @@ -233,7 +233,7 @@ spec: priorityClassName: system-node-critical containers: - name: vsphere-cloud-controller-manager - image: gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.26.1 + image: gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.26.2 args: - --cloud-provider=vsphere - --v=2 diff --git a/index.yaml b/index.yaml index 40ac54626..046fcf4e8 100644 --- a/index.yaml +++ b/index.yaml @@ -1,9 +1,28 @@ apiVersion: v1 entries: vsphere-cpi: + - apiVersion: v2 + appVersion: 1.26.2 + created: "2023-05-30T11:20:44.316207-07:00" + description: A Helm chart for vSphere Cloud Provider Interface Manager (CPI) + digest: c599a10ef8d398fb492bffe1f1159d3525db363bd24d4f2e82dd73a5f42b39ff + home: https://github.com/kubernetes/cloud-provider-vsphere + icon: https://raw.githubusercontent.com/kubernetes/cloud-provider-vsphere/master/docs/vmware_logo.png + keywords: + - vsphere + - vmware + - cloud + - provider + - cpi + name: vsphere-cpi + sources: + - https://github.com/kubernetes/cloud-provider-vsphere + urls: + - https://kubernetes.github.io/cloud-provider-vsphere/charts/vsphere-cpi-1.26.2.tgz + version: 1.26.2 - apiVersion: v2 appVersion: 1.26.1 - created: "2023-04-28T14:51:00.826463-07:00" + created: "2023-05-30T11:20:44.315301-07:00" description: A Helm chart for vSphere Cloud Provider Interface Manager (CPI) digest: 50498be332c5119d16eee9aa6850ac36af16376448842c3d0474a1ada12dbf9e home: https://github.com/kubernetes/cloud-provider-vsphere @@ -22,7 +41,7 @@ entries: version: 1.26.1 - apiVersion: v2 appVersion: 1.26.0 - created: "2023-04-28T14:51:00.825587-07:00" + created: "2023-05-30T11:20:44.314472-07:00" description: A Helm chart for vSphere Cloud Provider Interface Manager (CPI) digest: 2ecf416700c819f408eedf9263d08dbc04cd61969576be13d8935f15605f4a69 home: https://github.com/kubernetes/cloud-provider-vsphere @@ -41,7 +60,7 @@ entries: version: 1.26.0 - apiVersion: v2 appVersion: 1.25.0 - created: "2023-04-28T14:51:00.824498-07:00" + created: "2023-05-30T11:20:44.313625-07:00" description: A Helm chart for vSphere Cloud Provider Interface Manager (CPI) digest: 3d48df49fdfb8bda6b5cec1d9e9f566183a150c8c01e111eec5e22ab1f557e31 home: https://github.com/kubernetes/cloud-provider-vsphere @@ -60,7 +79,7 @@ entries: version: 1.25.0 - apiVersion: v2 appVersion: 1.24.2 - created: "2023-04-28T14:51:00.823242-07:00" + created: "2023-05-30T11:20:44.312773-07:00" description: A Helm chart for vSphere Cloud Provider Interface Manager (CPI) digest: d0bfaf8a081128f9477b8d8ff00079f8506d4aa2ea899f55511d06400179b159 home: https://github.com/kubernetes/cloud-provider-vsphere @@ -79,7 +98,7 @@ entries: version: 1.24.2 - apiVersion: v2 appVersion: 1.24.1 - created: "2023-04-28T14:51:00.822124-07:00" + created: "2023-05-30T11:20:44.311829-07:00" description: A Helm chart for vSphere Cloud Provider Interface Manager (CPI) digest: 6e9dec3e4c6aaeed0d7f95aa2beade986d5a0a2261509b11479ceb1a51312096 home: https://github.com/kubernetes/cloud-provider-vsphere @@ -96,4 +115,4 @@ entries: urls: - https://kubernetes.github.io/cloud-provider-vsphere/charts/vsphere-cpi-1.24.1.tgz version: 1.24.1 -generated: "2023-04-28T14:51:00.820459-07:00" +generated: "2023-05-30T11:20:44.310546-07:00" diff --git a/pkg/cloudprovider/vsphere/nodemanager.go b/pkg/cloudprovider/vsphere/nodemanager.go index 068dd28d3..b39a5c663 100644 --- a/pkg/cloudprovider/vsphere/nodemanager.go +++ b/pkg/cloudprovider/vsphere/nodemanager.go @@ -18,11 +18,14 @@ package vsphere import ( "context" + "encoding/base64" "errors" "fmt" "net" + "sort" "strings" + "gopkg.in/yaml.v2" v1 "k8s.io/api/core/v1" ccfg "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/config" vcfg "k8s.io/cloud-provider-vsphere/pkg/common/config" @@ -49,6 +52,15 @@ var ( ErrVMNotFound = errors.New("VM not found") ) +type cloudInitConfig struct { + Network struct { + Ethernets map[string]struct { + Name string `yaml:"set-name"` + Addresses []string `yaml:"addresses"` + } `yaml:"ethernets"` + } `yaml:"network"` +} + func newNodeManager(cfg *ccfg.CPIConfig, cm *cm.ConnectionManager) *NodeManager { return &NodeManager{ nodeNameMap: make(map[string]*NodeInfo), @@ -191,7 +203,7 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error { } var oVM mo.VirtualMachine - err = vmDI.VM.Properties(ctx, vmDI.VM.Reference(), []string{"guest", "summary"}, &oVM) + err = vmDI.VM.Properties(ctx, vmDI.VM.Reference(), []string{"guest", "summary", "config"}, &oVM) if err != nil { klog.Errorf("Error collecting properties for vm=%+v in vc=%s and datacenter=%s: %v", vmDI.VM, vmDI.VcServer, vmDI.DataCenter.Name(), err) @@ -291,10 +303,17 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error { return fmt.Errorf("unable to find suitable IP address for node after filtering out localhost IPs") } + sortedNonLocalhostIPs, err := sortStaticallyConfiguredAddressesFirst(oVM.Config.ExtraConfig, nonLocalhostIPs) + if err != nil { + klog.Errorf("Error sorting statically configured addresses for vm=%+v in vc=%s and datacenter=%s: %v", + vmDI.VM, vmDI.VcServer, vmDI.DataCenter.Name(), err) + return err + } + for _, ipFamily := range ipFamilies { - klog.V(6).Infof("ipFamily: %q nonLocalhostIPs: %q", ipFamily, nonLocalhostIPs) + klog.V(6).Infof("ipFamily: %q nonLocalhostIPs: %q", ipFamily, sortedNonLocalhostIPs) discoveredInternal, discoveredExternal := discoverIPs( - nonLocalhostIPs, + sortedNonLocalhostIPs, ipFamily, internalNetworkSubnets, externalNetworkSubnets, @@ -652,3 +671,59 @@ func (nm *NodeManager) getNodeNameByUUID(UUID string) string { } return "" } + +func guestInfoMetadata(extraConfig []types.BaseOptionValue) (string, string) { + var guestInfo, encoding string + for _, option := range extraConfig { + value := option.GetOptionValue() + switch value.Key { + case "guestinfo.metadata": + guestInfo, _ = value.Value.(string) + case "guestinfo.metadata.encoding": + encoding, _ = value.Value.(string) + } + } + return guestInfo, encoding +} + +// sortStaticallyConfiguredAddressesFirst prefers addresses that are from the +// guestInfo but only if they are on a NIC already. It preserves the order in which +// the addresses appear in the guestInfo. For addresses not found in the guestInfo, +// it preserves the order in which they appear in nonlocalhostIPs. +func sortStaticallyConfiguredAddressesFirst(extraConfig []types.BaseOptionValue, nonLocalhostIPs []*ipAddrNetworkName) ([]*ipAddrNetworkName, error) { + guestInfo, encoding := guestInfoMetadata(extraConfig) + + if guestInfo != "" && encoding == "base64" { + value, err := base64.StdEncoding.DecodeString(guestInfo) + if err != nil { + return nil, err + } + + guestInfo := &cloudInitConfig{} + err = yaml.Unmarshal(value, guestInfo) + if err != nil { + return nil, err + } + + // Map of guestInfo IP -> index that describes the order they appear in the guestInfo + guestInfoAddresses := make(map[string]int) + for _, eth := range guestInfo.Network.Ethernets { + for _, address := range eth.Addresses { + ip := net.ParseIP(strings.Split(address, "/")[0]) + guestInfoAddresses[ip.String()] = len(guestInfoAddresses) + } + } + + // Sort nonlocalhostIPs by the following comparator for two IP addresses: a and b + // if a is statically configured, but b is not then a should be prioritized before b + // if b is statically configured, but a is not then a should not be prioritized before b + // if a and b are both statically configured, then use the index from the guest info + sort.SliceStable(nonLocalhostIPs, func(i, j int) bool { + aIndex, aFound := guestInfoAddresses[nonLocalhostIPs[i].ipAddr] + bIndex, bFound := guestInfoAddresses[nonLocalhostIPs[j].ipAddr] + + return aFound && !bFound || aFound && bFound && aIndex < bIndex + }) + } + return nonLocalhostIPs, nil +} diff --git a/pkg/cloudprovider/vsphere/nodemanager_test.go b/pkg/cloudprovider/vsphere/nodemanager_test.go index c1eb3b430..aafc1d161 100644 --- a/pkg/cloudprovider/vsphere/nodemanager_test.go +++ b/pkg/cloudprovider/vsphere/nodemanager_test.go @@ -18,6 +18,8 @@ package vsphere import ( "context" + "encoding/base64" + "fmt" "net" "strings" "testing" @@ -230,6 +232,7 @@ func TestDiscoverNodeIPs(t *testing.T) { ipFamilyPriority []string cpiConfig *ccfg.CPIConfig networks []vimtypes.GuestNicInfo + guestinfo string } testcases := []struct { testName string @@ -1543,6 +1546,235 @@ func TestDiscoverNodeIPs(t *testing.T) { }, expectedErrorSubstring: "unable to find suitable IP address for node", }, + { + testName: "IPv6_guestInfoWithDHCP", + setup: testSetup{ + ipFamilyPriority: []string{"ipv6"}, + guestinfo: guestInfoWithIPv6DHCP(), + cpiConfig: nil, + networks: []vimtypes.GuestNicInfo{ + { + Network: "VM Network", + IpAddress: []string{ + "fe80::1", + "fd01:1234::1", + "fd01:cccc::1", + }, + }, + }, + }, + expectedIPs: []v1.NodeAddress{ + {Type: "InternalIP", Address: "fd01:1234::1"}, + {Type: "ExternalIP", Address: "fd01:1234::1"}, + }, + }, + { + testName: "StaticAddresses_IPv6_usesStaticAddressForExternalInternal", + setup: testSetup{ + ipFamilyPriority: []string{"ipv6"}, + guestinfo: guestInfoWithAddresses("fd01:cccc::1/128"), + cpiConfig: nil, + networks: []vimtypes.GuestNicInfo{ + { + Network: "VM Network", + IpAddress: []string{ + "fe80::1", + "fd01:1234::1", + "fd01:cccc::1", + }, + }, + }, + }, + expectedIPs: []v1.NodeAddress{ + {Type: "InternalIP", Address: "fd01:cccc::1"}, + {Type: "ExternalIP", Address: "fd01:cccc::1"}, + }, + }, + { + testName: "StaticAddresses_IPv4_usesStaticAddressForExternalInternal", + setup: testSetup{ + ipFamilyPriority: []string{"ipv4"}, + guestinfo: guestInfoWithAddresses("192.168.1.12/64"), + cpiConfig: nil, + networks: []vimtypes.GuestNicInfo{ + { + Network: "VM Network", + IpAddress: []string{ + "192.168.1.10", + "192.168.1.12", + }, + }, + }, + }, + expectedIPs: []v1.NodeAddress{ + {Type: "InternalIP", Address: "192.168.1.12"}, + {Type: "ExternalIP", Address: "192.168.1.12"}, + }, + }, + { + testName: "StaticAddresses_prioritizesOrderFromAddresses", + setup: testSetup{ + ipFamilyPriority: []string{"ipv4"}, + guestinfo: guestInfoWithAddresses("192.168.1.12/64,192.168.1.10/64"), + cpiConfig: nil, + networks: []vimtypes.GuestNicInfo{ + { + Network: "VM Network", + IpAddress: []string{ + "192.168.1.10", + "192.168.1.12", + }, + }, + }, + }, + expectedIPs: []v1.NodeAddress{ + {Type: "InternalIP", Address: "192.168.1.12"}, + {Type: "ExternalIP", Address: "192.168.1.12"}, + }, + }, + { + testName: "StaticAddresses_usesTheStaticAddressInTheNetworkCIDR", + setup: testSetup{ + ipFamilyPriority: []string{"ipv4"}, + guestinfo: guestInfoWithAddresses("10.10.10.10/64,192.168.1.12/64"), + cpiConfig: &ccfg.CPIConfig{ + Nodes: ccfg.Nodes{ + InternalNetworkSubnetCIDR: "192.168.0.0/16", + ExternalNetworkSubnetCIDR: "192.168.0.0/16", + }, + }, + networks: []vimtypes.GuestNicInfo{ + { + Network: "VM Network", + IpAddress: []string{ + "192.168.1.10", + "192.168.1.12", + "10.10.10.10", + }, + }, + }, + }, + expectedIPs: []v1.NodeAddress{ + {Type: "InternalIP", Address: "192.168.1.12"}, + {Type: "ExternalIP", Address: "192.168.1.12"}, + }, + }, + { + testName: "StaticAddresses_ignoresStaticAddressWhenWithinExcludeCIDR", + setup: testSetup{ + ipFamilyPriority: []string{"ipv4"}, + guestinfo: guestInfoWithAddresses("192.168.1.12/64,10.10.10.10/64"), + cpiConfig: &ccfg.CPIConfig{ + Nodes: ccfg.Nodes{ + ExcludeInternalNetworkSubnetCIDR: "192.168.0.0/16", + ExcludeExternalNetworkSubnetCIDR: "192.168.0.0/16", + }, + }, + networks: []vimtypes.GuestNicInfo{ + { + Network: "VM Network", + IpAddress: []string{ + "192.168.1.10", + "192.168.1.12", + "10.10.10.10", + }, + }, + }, + }, + expectedIPs: []v1.NodeAddress{ + {Type: "InternalIP", Address: "10.10.10.10"}, + {Type: "ExternalIP", Address: "10.10.10.10"}, + }, + }, + { + testName: "StaticAddresses_usesTheStaticAddressInTheConfiguredNetworkName", + setup: testSetup{ + ipFamilyPriority: []string{"ipv4"}, + guestinfo: guestInfoWithAddresses("192.168.1.8/64,192.168.1.12/64,10.10.10.10/64"), + cpiConfig: &ccfg.CPIConfig{ + Nodes: ccfg.Nodes{ + InternalVMNetworkName: "VM Network", + ExternalVMNetworkName: "VM Network", + }, + }, + networks: []vimtypes.GuestNicInfo{ + { + Network: "internal_net", + IpAddress: []string{ + "192.168.1.8", + }, + }, + { + Network: "VM Network", + IpAddress: []string{ + "192.168.1.10", + "192.168.1.12", + "10.10.10.10", + }, + }, + }, + }, + expectedIPs: []v1.NodeAddress{ + {Type: "InternalIP", Address: "192.168.1.12"}, + {Type: "ExternalIP", Address: "192.168.1.12"}, + }, + }, + { + testName: "StaticAddresses_addressesAreNotAssignedToTheNIC", + setup: testSetup{ + ipFamilyPriority: []string{"ipv4"}, + guestinfo: guestInfoWithAddresses("192.168.1.12/64,10.10.10.10/64"), + cpiConfig: nil, + networks: []vimtypes.GuestNicInfo{ + { + Network: "VM Network", + IpAddress: []string{ + "192.168.1.8", + }, + }, + }, + }, + expectedIPs: []v1.NodeAddress{ + {Type: "InternalIP", Address: "192.168.1.8"}, + {Type: "ExternalIP", Address: "192.168.1.8"}, + }, + }, + { + testName: "StaticAddresses_IPv6_handlesShorthandVsLonghandAddrs", + setup: testSetup{ + ipFamilyPriority: []string{"ipv6"}, + guestinfo: guestInfoWithAddresses("fd01:1:2:2919:abba:0000:0000:401/128"), + cpiConfig: nil, + networks: []vimtypes.GuestNicInfo{ + { + Network: "VM Network", + IpAddress: []string{ + "fd00::1", + "fd01:1:2:2919:abba::401", + }, + }, + }, + }, + expectedIPs: []v1.NodeAddress{ + {Type: "InternalIP", Address: "fd01:1:2:2919:abba::401"}, + {Type: "ExternalIP", Address: "fd01:1:2:2919:abba::401"}, + }, + }, + { + testName: "StaticAddresses_errorsOnInvalidGuestInfoFormat", + setup: testSetup{ + guestinfo: "not-valid-yaml this should error", + networks: []vimtypes.GuestNicInfo{ + { + Network: "VM Network", + IpAddress: []string{ + "192.168.1.10", + }, + }, + }, + }, + expectedErrorSubstring: "cannot unmarshal", + }, } for _, testcase := range testcases { @@ -1559,6 +1791,18 @@ func TestDiscoverNodeIPs(t *testing.T) { vm := simulator.Map.Any("VirtualMachine").(*simulator.VirtualMachine) vm.Guest.HostName = strings.ToLower(vm.Name) // simulator.SearchIndex.FindByDnsName matches against the guest.hostName property vm.Guest.Net = testcase.setup.networks + if testcase.setup.guestinfo != "" { + vm.Config.ExtraConfig = []vimtypes.BaseOptionValue{ + &vimtypes.OptionValue{ + Key: "guestinfo.metadata", + Value: base64.StdEncoding.EncodeToString([]byte(testcase.setup.guestinfo)), + }, + &vimtypes.OptionValue{ + Key: "guestinfo.metadata.encoding", + Value: "base64", + }, + } + } name := vm.Name @@ -1883,3 +2127,41 @@ func TestExcludeLocalhostIPs(t *testing.T) { t.Errorf("failure: expected ipAddr to equal fd00:100:64::1, but was %s", actual[1].ipAddr) } } + +func guestInfoWithIPv6DHCP() string { + return `instance-id: "tkg-mgmt-vc" +local-hostname: "tkg-mgmt-vc" +wait-on-network: + ipv4: false + ipv6: false +network: + version: 2 + ethernets: + id0: + match: + macaddress: "00:11:22" + set-name: "eth0" + wakeonlan: true + dhcp4: false + dhcp6: true` +} + +func guestInfoWithAddresses(addresses string) string { + return fmt.Sprintf(`instance-id: "tkg-mgmt-vc" +local-hostname: "tkg-mgmt-vc" +wait-on-network: + ipv4: false + ipv6: false +network: + version: 2 + ethernets: + id0: + addresses: [%s] + match: + macaddress: "00:11:22" + set-name: "eth0" + wakeonlan: true + dhcp4: false + dhcp6: false`, + addresses) +} diff --git a/pkg/common/vclib/connection.go b/pkg/common/vclib/connection.go index 6938a69b5..a2a1c9230 100644 --- a/pkg/common/vclib/connection.go +++ b/pkg/common/vclib/connection.go @@ -31,6 +31,10 @@ import ( klog "k8s.io/klog/v2" ) +const ( + userAgentName = "k8s-cloud-provider-vsphere" +) + // VSphereConnection contains information for connecting to vCenter type VSphereConnection struct { Client *vim25.Client @@ -177,6 +181,7 @@ func (connection *VSphereConnection) NewClient(ctx context.Context) (*vim25.Clie klog.Errorf("Failed to create new client. err: %+v", err) return nil, err } + client.UserAgent = userAgentName err = connection.login(ctx, client) if err != nil { return nil, err diff --git a/releases/v1.26/vsphere-cloud-controller-manager.yaml b/releases/v1.26/vsphere-cloud-controller-manager.yaml index 8f5fa9e2d..b9b30b678 100644 --- a/releases/v1.26/vsphere-cloud-controller-manager.yaml +++ b/releases/v1.26/vsphere-cloud-controller-manager.yaml @@ -234,7 +234,7 @@ spec: priorityClassName: system-node-critical containers: - name: vsphere-cloud-controller-manager - image: gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.26.1 + image: gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.26.2 args: - --cloud-provider=vsphere - --v=2