Skip to content

Commit

Permalink
Update KubeSchedulerConfiguration to api v1 for k8s version 1.25+
Browse files Browse the repository at this point in the history
  • Loading branch information
@olavangad-px
olavangad-px committed Jan 19, 2024
1 parent e9190e0 commit 787c798
Show file tree
Hide file tree
Showing 16 changed files with 3,022 additions and 71 deletions.
91 changes: 69 additions & 22 deletions pkg/controller/storagecluster/stork.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
schedcomp "k8s.io/component-base/config/v1alpha1"
schedconfig "k8s.io/kube-scheduler/config/v1beta3"
schedconfigapi "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta3"
schedconfig "k8s.io/kube-scheduler/config/v1"
schedconfigbeta3 "k8s.io/kube-scheduler/config/v1beta3"
schedconfigapi "k8s.io/kubernetes/pkg/scheduler/apis/config/v1"
schedconfigapibeta3 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta3"
"sigs.k8s.io/yaml"
)

Expand All @@ -53,9 +55,12 @@ const (
// https://github.com/kubernetes/kubernetes/blob/release-1.21/pkg/scheduler/scheduler.go#L306
policyDecoderChangeVersion = "1.17.0"
// Stork scheduler cannot run with kube-scheduler image > v1.22
pinnedStorkSchedulerVersion = "1.21.4"
minK8sVersionForPinnedStorkScheduler = "1.22.0"
minK8sVersionForKubeSchedulerConfiguration = "1.23.0"
pinnedStorkSchedulerVersion = "1.21.4"
minK8sVersionForPinnedStorkScheduler = "1.22.0"
// kubescheduler.config.k8s.io/v1 is GA'ed in k8s 1.25, so for 1.23 <= ver < 1.25
// we should continue using kubescheduler.config.k8s.io/v1beta3
minK8sVersionForKubeSchedulerV1BetaConfiguration = "1.23.0"
minK8sVersionForKubeSchedulerV1Configuration = "1.25.0"
)

const (
Expand Down Expand Up @@ -256,22 +261,55 @@ func (c *Controller) createStorkConfigMap(
},
}

//KubeSchedulerConfiguration is beta in 1.23 and GA in 1.25
leaderElect := true
schedulerName := storkDeploymentName
kubeSchedulerConfiguration := schedconfig.KubeSchedulerConfiguration{

leaderElectionConfiguration := schedcomp.LeaderElectionConfiguration{
LeaderElect: &leaderElect,
ResourceNamespace: clusterNamespace,
ResourceName: storkSchedDeploymentName,
LeaseDuration: metav1.Duration{Duration: 15 * time.Second},
RenewDeadline: metav1.Duration{Duration: 10 * time.Second},
RetryPeriod: metav1.Duration{Duration: 2 * time.Second},
ResourceLock: "leases",
}

kubeSchedulerConfigurationV1Beta := schedconfigbeta3.KubeSchedulerConfiguration{
TypeMeta: metav1.TypeMeta{
Kind: "KubeSchedulerConfiguration",
APIVersion: "kubescheduler.config.k8s.io/v1beta3",
},
LeaderElection: schedcomp.LeaderElectionConfiguration{
LeaderElect: &leaderElect,
ResourceNamespace: clusterNamespace,
ResourceName: storkSchedDeploymentName,
LeaseDuration: metav1.Duration{Duration: 15 * time.Second},
RenewDeadline: metav1.Duration{Duration: 10 * time.Second},
RetryPeriod: metav1.Duration{Duration: 2 * time.Second},
ResourceLock: "leases",
LeaderElection: leaderElectionConfiguration,
Profiles: []schedconfigbeta3.KubeSchedulerProfile{
{
SchedulerName: &schedulerName,
},
},
Extenders: []schedconfigbeta3.Extender{
{
URLPrefix: fmt.Sprintf(
"http://%s.%s:%d",
storkServiceName, clusterNamespace, storkServicePort,
),
FilterVerb: "filter",
PrioritizeVerb: "prioritize",
Weight: 5,
EnableHTTPS: false,
NodeCacheCapable: false,
HTTPTimeout: metav1.Duration{Duration: 5 * time.Minute},
},
},
}
// Auto fill the default configuration params
schedconfigapibeta3.SetDefaults_KubeSchedulerConfiguration(&kubeSchedulerConfigurationV1Beta)

kubeSchedulerConfigurationV1 := schedconfig.KubeSchedulerConfiguration{
TypeMeta: metav1.TypeMeta{
Kind: "KubeSchedulerConfiguration",
APIVersion: "kubescheduler.config.k8s.io/v1",
},
LeaderElection: leaderElectionConfiguration,
Profiles: []schedconfig.KubeSchedulerProfile{
{
SchedulerName: &schedulerName,
Expand All @@ -292,19 +330,28 @@ func (c *Controller) createStorkConfigMap(
},
},
}

// Auto fill the default configuration params
schedconfigapi.SetDefaults_KubeSchedulerConfiguration(&kubeSchedulerConfiguration)
schedconfigapi.SetDefaults_KubeSchedulerConfiguration(&kubeSchedulerConfigurationV1)

k8sMinVersionForKubeSchedulerConfiguration, err := version.NewVersion(minK8sVersionForKubeSchedulerConfiguration)
k8sMinVersionForKubeSchedulerV1BetaConfiguration, err := version.NewVersion(minK8sVersionForKubeSchedulerV1BetaConfiguration)
if err != nil {
logrus.WithError(err).Errorf("Could not parse version %s", k8sMinVersionForKubeSchedulerV1BetaConfiguration)
return err

Check warning on line 339 in pkg/controller/storagecluster/stork.go

View check run for this annotation

Codecov / codecov/patch

pkg/controller/storagecluster/stork.go#L338-L339 

Added lines #L338 - L339 were not covered by tests
}
k8sMinVersionForKubeSchedulerV1Configuration, err := version.NewVersion(minK8sVersionForKubeSchedulerV1Configuration)
if err != nil {
logrus.WithError(err).Errorf("Could not parse version %s", k8sMinVersionForKubeSchedulerConfiguration)
logrus.WithError(err).Errorf("Could not parse version %s", k8sMinVersionForKubeSchedulerV1Configuration)

Check warning on line 343 in pkg/controller/storagecluster/stork.go

View check run for this annotation

Codecov / codecov/patch

pkg/controller/storagecluster/stork.go#L343 

Added line #L343 was not covered by tests
return err
}

var policyConfig []byte
var dataKey string
if c.kubernetesVersion.GreaterThanOrEqual(k8sMinVersionForKubeSchedulerConfiguration) {
policyConfig, err = yaml.Marshal(kubeSchedulerConfiguration)
if c.kubernetesVersion.GreaterThanOrEqual(k8sMinVersionForKubeSchedulerV1BetaConfiguration) {
if c.kubernetesVersion.GreaterThanOrEqual(k8sMinVersionForKubeSchedulerV1Configuration) {
policyConfig, err = yaml.Marshal(kubeSchedulerConfigurationV1)
} else {
policyConfig, err = yaml.Marshal(kubeSchedulerConfigurationV1Beta)
}
if err != nil {
logrus.WithError(err).Errorf("Could not encode policy object")
return err
Expand Down Expand Up @@ -882,9 +929,9 @@ func (c *Controller) createStorkSchedDeployment(
return err
}

k8sMinVersionForKubeSchedulerConfiguration, err := version.NewVersion(minK8sVersionForKubeSchedulerConfiguration)
k8sMinVersionForKubeSchedulerConfiguration, err := version.NewVersion(minK8sVersionForKubeSchedulerV1BetaConfiguration)
if err != nil {
logrus.WithError(err).Errorf("Could not parse version %s", k8sMinVersionForKubeSchedulerConfiguration)
logrus.WithError(err).Errorf("Could not parse version %s", minK8sVersionForKubeSchedulerV1BetaConfiguration)

Check warning on line 934 in pkg/controller/storagecluster/stork.go

View check run for this annotation

Codecov / codecov/patch

pkg/controller/storagecluster/stork.go#L934 

Added line #L934 was not covered by tests
return err
}

Expand Down
139 changes: 90 additions & 49 deletions pkg/controller/storagecluster/stork_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ import (
fakek8sclient "k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/tools/record"
schedcomp "k8s.io/component-base/config/v1alpha1"
schedconfig "k8s.io/kube-scheduler/config/v1beta3"
schedconfig "k8s.io/kube-scheduler/config/v1"
schedconfigbeta3 "k8s.io/kube-scheduler/config/v1beta3"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"

Expand All @@ -42,6 +43,7 @@ func TestStorkInstallation(t *testing.T) {
testStorkInstallation(t, "1.18.0")
testStorkInstallation(t, "1.23.0")
testStorkInstallation(t, "1.25.0")
testStorkInstallation(t, "1.29.0")
}

func testStorkInstallation(t *testing.T, k8sVersionStr string) {
Expand Down Expand Up @@ -104,49 +106,24 @@ func testStorkInstallation(t *testing.T, k8sVersionStr string) {

require.NoError(t, err)

k8sMinVersionForKubeSchedulerConfiguration, err := version.NewVersion(minK8sVersionForKubeSchedulerConfiguration)
k8sMinVersionForKubeSchedulerV1Configuration, err := version.NewVersion(minK8sVersionForKubeSchedulerV1Configuration)
require.NoError(t, err)
k8sMinVersionForKubeSchedulerV1BetaConfiguration, err := version.NewVersion(minK8sVersionForKubeSchedulerV1BetaConfiguration)
require.NoError(t, err)

if k8sVersion.GreaterThanOrEqual(k8sMinVersionForKubeSchedulerConfiguration) {
if k8sVersion.GreaterThanOrEqual(k8sMinVersionForKubeSchedulerV1BetaConfiguration) {
// Stork ConfigMap
leaderElect := true
schedulerName := storkDeploymentName
expectedKubeSchedulerConfiguration := schedconfig.KubeSchedulerConfiguration{
TypeMeta: metav1.TypeMeta{
Kind: "KubeSchedulerConfiguration",
APIVersion: "kubescheduler.config.k8s.io/v1beta3",
},
LeaderElection: schedcomp.LeaderElectionConfiguration{
LeaderElect: &leaderElect,
ResourceNamespace: "kube-test",
ResourceName: storkSchedDeploymentName,
LeaseDuration: metav1.Duration{Duration: 15 * time.Second},
RenewDeadline: metav1.Duration{Duration: 10 * time.Second},
RetryPeriod: metav1.Duration{Duration: 2 * time.Second},
ResourceLock: "leases",
},
Profiles: []schedconfig.KubeSchedulerProfile{
{
SchedulerName: &schedulerName,
},
},
Extenders: []schedconfig.Extender{
{
URLPrefix: fmt.Sprintf(
"http://%s.%s:%d",
storkServiceName, "kube-test", storkServicePort,
),
FilterVerb: "filter",
PrioritizeVerb: "prioritize",
Weight: 5,
EnableHTTPS: false,
NodeCacheCapable: false,
HTTPTimeout: metav1.Duration{Duration: 5 * time.Minute},
},
},
leaderElectionConfiguration := schedcomp.LeaderElectionConfiguration{
LeaderElect: &leaderElect,
ResourceNamespace: "kube-test",
ResourceName: storkSchedDeploymentName,
LeaseDuration: metav1.Duration{Duration: 15 * time.Second},
RenewDeadline: metav1.Duration{Duration: 10 * time.Second},
RetryPeriod: metav1.Duration{Duration: 2 * time.Second},
ResourceLock: "leases",
}

var actualKubeSchedulerConfiguration = schedconfig.KubeSchedulerConfiguration{}
storkConfigMap := &v1.ConfigMap{}
err = testutil.Get(k8sClient, storkConfigMap, storkConfigMapName, cluster.Namespace)
require.NoError(t, err)
Expand All @@ -155,13 +132,77 @@ func testStorkInstallation(t *testing.T, k8sVersionStr string) {
require.Len(t, storkConfigMap.OwnerReferences, 1)
require.Equal(t, cluster.Name, storkConfigMap.OwnerReferences[0].Name)

err = yaml.Unmarshal([]byte(storkConfigMap.Data["stork-config.yaml"]), &actualKubeSchedulerConfiguration)
require.NoError(t, err)

require.True(t, reflect.DeepEqual(expectedKubeSchedulerConfiguration.TypeMeta, actualKubeSchedulerConfiguration.TypeMeta))
require.Equal(t, expectedKubeSchedulerConfiguration.Profiles[0].SchedulerName, actualKubeSchedulerConfiguration.Profiles[0].SchedulerName)
require.True(t, reflect.DeepEqual(expectedKubeSchedulerConfiguration.LeaderElection, actualKubeSchedulerConfiguration.LeaderElection))
require.True(t, reflect.DeepEqual(expectedKubeSchedulerConfiguration.Extenders[0], actualKubeSchedulerConfiguration.Extenders[0]))
if k8sVersion.GreaterThanOrEqual(k8sMinVersionForKubeSchedulerV1Configuration) {
expectedKubeSchedulerConfiguration := schedconfig.KubeSchedulerConfiguration{
TypeMeta: metav1.TypeMeta{
Kind: "KubeSchedulerConfiguration",
APIVersion: "kubescheduler.config.k8s.io/v1",
},
LeaderElection: leaderElectionConfiguration,
Profiles: []schedconfig.KubeSchedulerProfile{
{
SchedulerName: &schedulerName,
},
},
Extenders: []schedconfig.Extender{
{
URLPrefix: fmt.Sprintf(
"http://%s.%s:%d",
storkServiceName, "kube-test", storkServicePort,
),
FilterVerb: "filter",
PrioritizeVerb: "prioritize",
Weight: 5,
EnableHTTPS: false,
NodeCacheCapable: false,
HTTPTimeout: metav1.Duration{Duration: 5 * time.Minute},
},
},
}
var actualKubeSchedulerConfiguration = schedconfig.KubeSchedulerConfiguration{}
err = yaml.Unmarshal([]byte(storkConfigMap.Data["stork-config.yaml"]), &actualKubeSchedulerConfiguration)
require.NoError(t, err)

require.True(t, reflect.DeepEqual(expectedKubeSchedulerConfiguration.TypeMeta, actualKubeSchedulerConfiguration.TypeMeta))
require.Equal(t, expectedKubeSchedulerConfiguration.Profiles[0].SchedulerName, actualKubeSchedulerConfiguration.Profiles[0].SchedulerName)
require.True(t, reflect.DeepEqual(expectedKubeSchedulerConfiguration.LeaderElection, actualKubeSchedulerConfiguration.LeaderElection))
require.True(t, reflect.DeepEqual(expectedKubeSchedulerConfiguration.Extenders[0], actualKubeSchedulerConfiguration.Extenders[0]))
} else {
expectedKubeSchedulerConfiguration := schedconfigbeta3.KubeSchedulerConfiguration{
TypeMeta: metav1.TypeMeta{
Kind: "KubeSchedulerConfiguration",
APIVersion: "kubescheduler.config.k8s.io/v1beta3",
},
LeaderElection: leaderElectionConfiguration,
Profiles: []schedconfigbeta3.KubeSchedulerProfile{
{
SchedulerName: &schedulerName,
},
},
Extenders: []schedconfigbeta3.Extender{
{
URLPrefix: fmt.Sprintf(
"http://%s.%s:%d",
storkServiceName, "kube-test", storkServicePort,
),
FilterVerb: "filter",
PrioritizeVerb: "prioritize",
Weight: 5,
EnableHTTPS: false,
NodeCacheCapable: false,
HTTPTimeout: metav1.Duration{Duration: 5 * time.Minute},
},
},
}
var actualKubeSchedulerConfiguration = schedconfigbeta3.KubeSchedulerConfiguration{}
err = yaml.Unmarshal([]byte(storkConfigMap.Data["stork-config.yaml"]), &actualKubeSchedulerConfiguration)
require.NoError(t, err)

require.True(t, reflect.DeepEqual(expectedKubeSchedulerConfiguration.TypeMeta, actualKubeSchedulerConfiguration.TypeMeta))
require.Equal(t, expectedKubeSchedulerConfiguration.Profiles[0].SchedulerName, actualKubeSchedulerConfiguration.Profiles[0].SchedulerName)
require.True(t, reflect.DeepEqual(expectedKubeSchedulerConfiguration.LeaderElection, actualKubeSchedulerConfiguration.LeaderElection))
require.True(t, reflect.DeepEqual(expectedKubeSchedulerConfiguration.Extenders[0], actualKubeSchedulerConfiguration.Extenders[0]))
}
} else {
// Stork ConfigMap
expectedPolicy := SchedulerPolicy{
Expand Down Expand Up @@ -313,7 +354,7 @@ func testStorkInstallation(t *testing.T, k8sVersionStr string) {
require.Equal(t, expectedStorkDeployment.Spec, storkDeployment.Spec)

// Sched Scheduler Deployment
if k8sVersion.GreaterThanOrEqual(k8sMinVersionForKubeSchedulerConfiguration) {
if k8sVersion.GreaterThanOrEqual(k8sMinVersionForKubeSchedulerV1BetaConfiguration) {
expectedSchedDeployment := testutil.GetExpectedDeployment(t, "storkSchedKubeSchedConfigDeployment.yaml")
schedDeployment := &appsv1.Deployment{}
err = testutil.Get(k8sClient, schedDeployment, storkSchedDeploymentName, cluster.Namespace)
Expand All @@ -327,7 +368,7 @@ func testStorkInstallation(t *testing.T, k8sVersionStr string) {
schedDeployment.Spec.Template.Spec.Containers[0].Resources.Requests = nil
require.Equal(t, expectedSchedDeployment.Labels, schedDeployment.Labels)
expectedSchedDeployment.Spec.Template.Spec.Containers[0].Image = strings.Replace(
expectedSchedDeployment.Spec.Template.Spec.Containers[0].Image, minK8sVersionForKubeSchedulerConfiguration, k8sVersionStr, -1)
expectedSchedDeployment.Spec.Template.Spec.Containers[0].Image, minK8sVersionForKubeSchedulerV1BetaConfiguration, k8sVersionStr, -1)
require.Equal(t, expectedSchedDeployment.Spec, schedDeployment.Spec)
} else {
expectedSchedDeployment := testutil.GetExpectedDeployment(t, "storkSchedDeployment.yaml")
Expand Down Expand Up @@ -496,7 +537,7 @@ func TestStorkSchedulerK8SVersions(t *testing.T) {
schedDeployment.Spec.Template.Spec.Containers[0].Resources.Requests = nil
require.Equal(t, expectedSchedDeployment.Labels, schedDeployment.Labels)
expectedSchedDeployment.Spec.Template.Spec.Containers[0].Image = strings.Replace(
expectedSchedDeployment.Spec.Template.Spec.Containers[0].Image, minK8sVersionForKubeSchedulerConfiguration, k8sVersionStr, -1)
expectedSchedDeployment.Spec.Template.Spec.Containers[0].Image, minK8sVersionForKubeSchedulerV1BetaConfiguration, k8sVersionStr, -1)
require.Equal(t, expectedSchedDeployment.Spec, schedDeployment.Spec)

k8sVersionStr = "1.25.0"
Expand All @@ -515,7 +556,7 @@ func TestStorkSchedulerK8SVersions(t *testing.T) {
schedDeployment.Spec.Template.Spec.Containers[0].Resources.Requests = nil
require.Equal(t, expectedSchedDeployment.Labels, schedDeployment.Labels)
expectedSchedDeployment.Spec.Template.Spec.Containers[0].Image = strings.Replace(
expectedSchedDeployment.Spec.Template.Spec.Containers[0].Image, minK8sVersionForKubeSchedulerConfiguration, k8sVersionStr, -1)
expectedSchedDeployment.Spec.Template.Spec.Containers[0].Image, minK8sVersionForKubeSchedulerV1BetaConfiguration, k8sVersionStr, -1)
require.Equal(t, expectedSchedDeployment.Spec, schedDeployment.Spec)
}

Expand Down
21 changes: 21 additions & 0 deletions vendor/k8s.io/kube-scheduler/config/v1/doc.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 787c798

Please sign in to comment.