From 635f77e06d2a0d644b45ccdcaadf04653a9cbe87 Mon Sep 17 00:00:00 2001 From: Omer Tuchfeld Date: Mon, 4 Mar 2024 00:19:13 +0100 Subject: [PATCH] Use the new recert additional trust bundle options Incomplete, still testing and need to add a bit more code # Background / Context Recert recently added ([1], [2]) some options that allow changing the cluster's trust bundle (it's recommended you read the PRs for more background about this). # Issue / Requirement / Reason for change The lifecycle-agent doesn't make use of the new options added to recert # Solution / Feature Overview Change the lifecycle-agent to use the new options added to recert # Implementation Details Multiple new fields have been added. - `AdditionalTrustBundle` in `SeedReconfiguration`. This represents the trust bundle to be used for seed-reconfiguration. This contains the user-ca-bundle contents, the proxy configmap name, and the proxy configmap contents. - `AdditionalTrustBundle` in `SeedClusterInfo`. This represents the state of the trust bundle in the seed cluster. This is simply booleans indicating the presence or lack there-of of the user-ca-bundle and the proxy configmap name (only if it actually has contents, a configmap with no contents is considered invalid OCP configuration). This is useful for when we want to verify that the seed is compatible with our desired `SeedReconfiguration`. - `RecertConfig` will now use the new `CryptoDirs` and `CryptoFiles` fields to specify the directories and files that should be considered part of the cluster's crypto material. Along with the `ClusterCustomizationDirs` and `ClusterCustomizationFiles` fields that specify the directories and files involved in cluster customization. Since these no longer overlap when it comes to customizing the trust bundle, we must use these new fields instead of the old common `StaticDirs` and `StaticFiles` fields. [1] https://github.com/rh-ecosystem-edge/recert/pull/110 [2] https://github.com/rh-ecosystem-edge/recert/pull/140 --- api/seedreconfig/seedreconfig.go | 14 ++ controllers/prep_handlers.go | 41 ++++++ internal/clusterconfig/clusterconfig.go | 130 +++++++++++-------- internal/clusterconfig/clusterconfig_test.go | 26 +--- internal/common/consts.go | 4 +- internal/recert/recert.go | 94 ++++++++++---- lca-cli/postpivot/postpivot.go | 12 -- lca-cli/seedclusterinfo/seedclusterinfo.go | 18 ++- lca-cli/seedcreator/seedcreator.go | 29 ++++- utils/client_helper.go | 61 +++++++++ 10 files changed, 309 insertions(+), 120 deletions(-) diff --git a/api/seedreconfig/seedreconfig.go b/api/seedreconfig/seedreconfig.go index 899136f78..7b476c0fc 100644 --- a/api/seedreconfig/seedreconfig.go +++ b/api/seedreconfig/seedreconfig.go @@ -132,6 +132,8 @@ type SeedReconfiguration struct { // upgraded cluster. In IBI, a fake install-config should be generated by // the IBIO. This parameter is required when the proxy parameters are set. InstallConfig string `json:"install_config,omitempty"` + + AdditionalTrustBundle AdditionalTrustBundle `json:"additionalTrustBundle,omitempty"` } type KubeConfigCryptoRetention struct { @@ -174,3 +176,15 @@ type Proxy struct { // +optional NoProxy string `json:"noProxy,omitempty"` } + +type AdditionalTrustBundle struct { + // The contents of the "user-ca-bundle" configmap in the "openshift-config" namepace + UserCaBundle string `json:"userCaBundle"` + + // The Proxy CR trustedCA configmap nam + ProxyConfigmapName string `json:"proxyConfigmapName"` + + // The contents of the ProxyConfigmapName configmap. Must equal + // UserCaBundle if ProxyConfigmapName is "user-ca-bundle" + ProxyConfigmapBundle string `json:"proxyConfigmapBundle"` +} diff --git a/controllers/prep_handlers.go b/controllers/prep_handlers.go index d8c9d2fc1..81f0d2071 100644 --- a/controllers/prep_handlers.go +++ b/controllers/prep_handlers.go @@ -106,6 +106,22 @@ func GetSeedImage(c client.Client, ctx context.Context, ibu *lcav1alpha1.ImageBa return fmt.Errorf("checking seed image compatibility: %w", err) } + HasUserCaBundle, ProxyConfigmapName, err := lcautils.GetClusterAdditionalTrustBundleState(ctx, c) + if err != nil { + return fmt.Errorf("failed to get cluster additional trust bundle state: %w", err) + } + + if seedInfo.AdditionalTrustBundle != nil { + if err := checkSeedImageAdditionalTrustBundleCompatibility(*seedInfo.AdditionalTrustBundle, HasUserCaBundle, ProxyConfigmapName); err != nil { + return fmt.Errorf("checking seed image additional trust bundle compatibility: %w", err) + } + } else { + // For the sake of backwards compatibility, we allow older seed images + // that don't have information about the additional trust bundle. This + // means that upgrade will fail at the recert stage if there's a + // mismatch between the seed and the seed reconfiguration data. + } + return nil } @@ -196,6 +212,31 @@ func checkSeedImageProxyCompatibility(seedHasProxy, hasProxy bool) error { return nil } +// checkSeedImageAdditionalTrustBundleCompatibility checks for proxy +// configuration compatibility of the seed image vs the current cluster. If the +// seed image has a proxy and the cluster being upgraded doesn't, we cannot +// proceed as recert does not support proxy rename under those conditions. +// Similarly, we cannot proceed if the cluster being upgraded has a proxy but +// the seed image doesn't. +func checkSeedImageAdditionalTrustBundleCompatibility(seedAdditionalTrustBundle seedclusterinfo.AdditionalTrustBundle, HasUserCaBundle bool, ProxyConfigmapName string) error { + if seedAdditionalTrustBundle.HasUserCaBundle && !HasUserCaBundle { + return fmt.Errorf("seed image has an %s/%s configmap but the cluster being upgraded does not, this combination is not supported", + common.OpenshiftConfigNamespace, common.ClusterAdditionalTrustBundleName) + } + + if !seedAdditionalTrustBundle.HasUserCaBundle && HasUserCaBundle { + return fmt.Errorf("seed image does not have an %s/%s configmap but the cluster being upgraded does, this combination is not supported", + common.OpenshiftConfigNamespace, common.ClusterAdditionalTrustBundleName) + } + + if seedAdditionalTrustBundle.ProxyConfigmapName != ProxyConfigmapName { + return fmt.Errorf("seed image Proxy trustedCA configmap (oc get proxy -oyaml) %s mismatches cluster's name %s, this combination is not supported", + seedAdditionalTrustBundle.ProxyConfigmapName, ProxyConfigmapName) + } + + return nil +} + // validateSeedOcpVersion rejects upgrade request if seed image version is not higher than current cluster (target) OCP version func (r *ImageBasedUpgradeReconciler) validateSeedOcpVersion(seedOcpVersion string) error { // get target OCP version diff --git a/internal/clusterconfig/clusterconfig.go b/internal/clusterconfig/clusterconfig.go index f7e889dac..e3da1ff9b 100644 --- a/internal/clusterconfig/clusterconfig.go +++ b/internal/clusterconfig/clusterconfig.go @@ -10,7 +10,6 @@ import ( v1 "github.com/openshift/api/config/v1" operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -33,16 +32,11 @@ import ( const ( manifestDir = "manifests" - proxyName = "cluster" - pullSecretName = "pull-secret" idmsFileName = "image-digest-mirror-set.json" icspsFileName = "image-content-source-policy-list.json" - caBundleCMName = "user-ca-bundle" - caBundleFileName = caBundleCMName + ".json" - // ssh authorized keys file created by mco from ssh machine configs sshKeyFile = "/home/core/.ssh/authorized_keys.d/ignition" ) @@ -84,9 +78,6 @@ func (r *UpgradeClusterConfigGather) FetchClusterConfig(ctx context.Context, ost if err := r.fetchClusterInfo(ctx, clusterConfigPath); err != nil { return err } - if err := r.fetchCABundle(ctx, manifestsDir, clusterConfigPath); err != nil { - return err - } if err := r.fetchICSPs(ctx, manifestsDir); err != nil { return err } @@ -144,9 +135,73 @@ func (r *UpgradeClusterConfigGather) GetKubeadminPasswordHash(ctx context.Contex return kubeadminPasswordHash, nil } +type AdditionalTrustBundle struct { + // The contents of the "user-ca-bundle" configmap in the "openshift-config" namepace + UserCaBundle string `json:"userCaBundle"` + + // The Proxy CR trustedCA configmap name + ProxyConfigmapName string `json:"proxyConfigmapName"` + + // The contents of the ProxyConfigmapName configmap. Must equal + // UserCaBundle if ProxyConfigmapName is "user-ca-bundle" + ProxyConfigmapBundle string `json:"proxyConfigmapBundle"` +} + +func newAdditionalTrustBundle(userCaBundle, proxyConfigmapName, proxyConfigmapBundle string) (*AdditionalTrustBundle, error) { + if proxyConfigmapName == common.ClusterAdditionalTrustBundleName && userCaBundle != proxyConfigmapBundle { + return nil, fmt.Errorf("proxyConfigmapName is %s, but userCaBundle and proxyConfigmapBundle differ", common.ClusterAdditionalTrustBundleName) + } + + return &AdditionalTrustBundle{ + UserCaBundle: userCaBundle, + ProxyConfigmapName: proxyConfigmapName, + ProxyConfigmapBundle: proxyConfigmapBundle, + }, nil +} + +func (r *UpgradeClusterConfigGather) GetAdditionalTrustBundle(ctx context.Context) (*AdditionalTrustBundle, error) { + clusterAdditionalTrustBundle, err := utils.GetAdditionalTrustBundleFromConfigmap(ctx, r.Client, common.ClusterAdditionalTrustBundleName) + if err != nil { + return nil, fmt.Errorf("failed to get additional trust bundle from configmap: %w", err) + } + + proxy := v1.Proxy{} + if err := r.Client.Get(ctx, types.NamespacedName{Name: common.OpenshiftProxyCRName}, &proxy); err != nil { + return nil, fmt.Errorf("failed to get proxy: %w", err) + + } + + proxyCaBundle := "" + switch proxy.Spec.TrustedCA.Name { + case "": + // No proxy CA bundle + case common.ClusterAdditionalTrustBundleName: + // Proxy is using the cluster's CA bundle + proxyCaBundle = clusterAdditionalTrustBundle + default: + // Proxy is using a different CA bundle, retrieve it from the named configmap + proxyCaBundle, err = utils.GetAdditionalTrustBundleFromConfigmap(ctx, r.Client, proxy.Spec.TrustedCA.Name) + if err != nil { + return nil, fmt.Errorf("failed to get additional trust bundle from configmap: %w", err) + } + + if proxyCaBundle == "" { + // This is a very weird but probably valid OCP configuration that we prefer to not support in LCA + return nil, fmt.Errorf("proxy trustedCA configmap %s/%s exists but is empty", common.OpenshiftConfigNamespace, proxy.Spec.TrustedCA.Name) + } + } + + additionalTrustBundle, err := newAdditionalTrustBundle(clusterAdditionalTrustBundle, proxy.Spec.TrustedCA.Name, proxyCaBundle) + if err != nil { + return nil, fmt.Errorf("failed to create additional trust bundle: %w", err) + } + + return additionalTrustBundle, err +} + func (r *UpgradeClusterConfigGather) GetProxy(ctx context.Context) (*seedreconfig.Proxy, *seedreconfig.Proxy, error) { proxy := v1.Proxy{} - if err := r.Client.Get(ctx, types.NamespacedName{Name: proxyName}, &proxy); err != nil { + if err := r.Client.Get(ctx, types.NamespacedName{Name: common.OpenshiftProxyCRName}, &proxy); err != nil { return nil, nil, fmt.Errorf("failed to get proxy: %w", err) } @@ -187,6 +242,7 @@ func SeedReconfigurationFromClusterInfo( proxy, statusProxy *seedreconfig.Proxy, installConfig string, + additionalTrustBundle *AdditionalTrustBundle, ) *seedreconfig.SeedReconfiguration { return &seedreconfig.SeedReconfiguration{ APIVersion: seedreconfig.SeedReconfigurationVersion, @@ -204,6 +260,11 @@ func SeedReconfigurationFromClusterInfo( Proxy: proxy, StatusProxy: statusProxy, InstallConfig: installConfig, + AdditionalTrustBundle: seedreconfig.AdditionalTrustBundle{ + UserCaBundle: additionalTrustBundle.UserCaBundle, + ProxyConfigmapName: additionalTrustBundle.ProxyConfigmapName, + ProxyConfigmapBundle: additionalTrustBundle.ProxyConfigmapBundle, + }, } } @@ -245,6 +306,11 @@ func (r *UpgradeClusterConfigGather) fetchClusterInfo(ctx context.Context, clust return err } + additionalTrustBundle, err := r.GetAdditionalTrustBundle(ctx) + if err != nil { + return err + } + installConfig, err := r.GetInstallConfig(ctx) if err != nil { return err @@ -258,6 +324,7 @@ func (r *UpgradeClusterConfigGather) fetchClusterInfo(ctx context.Context, clust proxy, statusProxy, installConfig, + additionalTrustBundle, ) filePath := filepath.Join(clusterConfigPath, common.SeedReconfigurationFileName) @@ -317,14 +384,6 @@ func (r *UpgradeClusterConfigGather) typeMetaForObject(o runtime.Object) (*metav }, nil } -func (r *UpgradeClusterConfigGather) cleanObjectMetadata(o client.Object) metav1.ObjectMeta { - return metav1.ObjectMeta{ - Name: o.GetName(), - Namespace: o.GetNamespace(), - Labels: o.GetLabels(), - } -} - func (r *UpgradeClusterConfigGather) getIDMSs(ctx context.Context) (v1.ImageDigestMirrorSetList, error) { idmsList := v1.ImageDigestMirrorSetList{} currentIdms := v1.ImageDigestMirrorSetList{} @@ -400,41 +459,6 @@ func (r *UpgradeClusterConfigGather) fetchICSPs(ctx context.Context, manifestsDi return nil } -func (r *UpgradeClusterConfigGather) fetchCABundle(ctx context.Context, manifestsDir, clusterConfigPath string) error { - r.Log.Info("Fetching user ca bundle") - caBundle := &corev1.ConfigMap{} - err := r.Client.Get(ctx, types.NamespacedName{Name: caBundleCMName, - Namespace: common.OpenshiftConfigNamespace}, caBundle) - if err != nil && errors.IsNotFound(err) { - return nil - } - if err != nil { - return fmt.Errorf("failed to get ca bundle cm, err: %w", err) - } - - typeMeta, err := r.typeMetaForObject(caBundle) - if err != nil { - return err - } - caBundle.TypeMeta = *typeMeta - caBundle.ObjectMeta = r.cleanObjectMetadata(caBundle) - - if err := utils.MarshalToFile(caBundle, filepath.Join(manifestsDir, caBundleFileName)); err != nil { - return fmt.Errorf("failed to write user ca bundle to %s, err: %w", - filepath.Join(manifestsDir, caBundleFileName), err) - } - - // we should copy ca-bundle from snoa as without doing it we will fail to pull images - // workaround for https://issues.redhat.com/browse/OCPBUGS-24035 - caBundleFilePath := filepath.Join(hostPath, common.CABundleFilePath) - r.Log.Info("Copying", "file", caBundleFilePath) - if err := utils.CopyFileIfExists(caBundleFilePath, filepath.Join(clusterConfigPath, filepath.Base(caBundleFilePath))); err != nil { - return fmt.Errorf("failed to copy ca-bundle file %s to %s, err %w", caBundleFilePath, clusterConfigPath, err) - } - - return nil -} - // gather network files and copy them func (r *UpgradeClusterConfigGather) fetchNetworkConfig(ostreeDir string) error { r.Log.Info("Fetching node network files") diff --git a/internal/clusterconfig/clusterconfig_test.go b/internal/clusterconfig/clusterconfig_test.go index b770922ca..d46413d1d 100644 --- a/internal/clusterconfig/clusterconfig_test.go +++ b/internal/clusterconfig/clusterconfig_test.go @@ -393,7 +393,7 @@ func TestClusterConfig(t *testing.T) { Name: "2", }, Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{{Source: "icspData2"}}}}}, - caBundleCM: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: caBundleCMName, + caBundleCM: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: common.ClusterAdditionalTrustBundleName, Namespace: common.OpenshiftConfigNamespace}, Data: map[string]string{"test": "data"}}, expectedErr: false, validateFunc: func(t *testing.T, tempDir string, err error, ucc UpgradeClusterConfigGather) { @@ -427,17 +427,6 @@ func TestClusterConfig(t *testing.T) { assert.Contains(t, resultSourcesAsString, "icspData") assert.Contains(t, resultSourcesAsString, "icspData2") } - - // validate caBundle - caBundle := &corev1.ConfigMap{} - if err := utils.ReadYamlOrJSONFile(filepath.Join(manifestsDir, caBundleFileName), caBundle); err != nil { - t.Errorf("unexpected error: %v", err) - } - assert.Equal(t, caBundleCMName, caBundle.Name) - assert.Equal(t, caBundle.Data, map[string]string{"test": "data"}) - - _, err = os.Stat(filepath.Join(clusterConfigPath, filepath.Base(common.CABundleFilePath))) - assert.Nil(t, err) }, }, } @@ -447,19 +436,6 @@ func TestClusterConfig(t *testing.T) { t.Run(testCase.testCaseName, func(t *testing.T) { hostPath = clusterConfigDir - if testCase.caBundleCM != nil { - dir := filepath.Join(clusterConfigDir, filepath.Dir(common.CABundleFilePath)) - if err := os.MkdirAll(dir, 0o700); err != nil { - t.Errorf("unexpected error: %v", err) - } - newPath := filepath.Join(dir, filepath.Base(common.CABundleFilePath)) - f, err := os.Create(newPath) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - _ = f.Close() - } - sshKeyDir := filepath.Join(clusterConfigDir, filepath.Dir(sshKeyFile)) if err := os.MkdirAll(sshKeyDir, 0o700); err != nil { t.Errorf("unexpected error: %v", err) diff --git a/internal/common/consts.go b/internal/common/consts.go index 53c026dee..d5d15ca5e 100644 --- a/internal/common/consts.go +++ b/internal/common/consts.go @@ -62,7 +62,6 @@ const ( EtcdContainerName = "recert_etcd" LvmConfigDir = "lvm-configuration" LvmDevicesPath = "/etc/lvm/devices/system.devices" - CABundleFilePath = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" LCAConfigDir = "/var/lib/lca" IBUAutoRollbackConfigFile = LCAConfigDir + "/autorollback_config.json" @@ -110,6 +109,9 @@ const ( NMConnectionFolder = "/etc/NetworkManager/system-connections" NetworkDir = "network-configuration" + + CaBundleDataKey = "ca-bundle.crt" + ClusterAdditionalTrustBundleName = "user-ca-bundle" ) // Annotation names and values related to extra manifest diff --git a/internal/recert/recert.go b/internal/recert/recert.go index 1a676a3bf..4cfed1af9 100644 --- a/internal/recert/recert.go +++ b/internal/recert/recert.go @@ -19,18 +19,28 @@ const ( SummaryFile = "/var/tmp/recert-summary.yaml" ) -var staticDirs = []string{"/kubelet", "/kubernetes", "/machine-config-daemon"} +var ( + // we don't want pki to go through recertification, only cluster customization (additional trust bundle appears in /pki) + cryptoDirs = []string{"/kubelet", "/kubernetes", "/machine-config-daemon"} + clusterCustomizationDirs = []string{"/kubelet", "/kubernetes", "/machine-config-daemon", "/pki"} + + cryptoFiles = []string{"/host-etc/mcs-machine-config-content.json"} + clusterCustomizationFiles = []string{"/host-etc/mcs-machine-config-content.json", "/host-etc/mco/proxy.env"} +) type RecertConfig struct { - DryRun bool `json:"dry_run,omitempty"` - ExtendExpiration bool `json:"extend_expiration,omitempty"` - ForceExpire bool `json:"force_expire,omitempty"` - EtcdEndpoint string `json:"etcd_endpoint,omitempty"` - ClusterRename string `json:"cluster_rename,omitempty"` - Hostname string `json:"hostname,omitempty"` - IP string `json:"ip,omitempty"` - Proxy string `json:"proxy,omitempty"` - InstallConfig string `json:"install_config,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + ExtendExpiration bool `json:"extend_expiration,omitempty"` + ForceExpire bool `json:"force_expire,omitempty"` + EtcdEndpoint string `json:"etcd_endpoint,omitempty"` + ClusterRename string `json:"cluster_rename,omitempty"` + Hostname string `json:"hostname,omitempty"` + IP string `json:"ip,omitempty"` + Proxy string `json:"proxy,omitempty"` + InstallConfig string `json:"install_config,omitempty"` + UserCaBundle string `json:"user_ca_bundle,omitempty"` + ProxyTrustedCaBundle string `json:"proxy_trusted_ca_bundle,omitempty"` + // We intentionally don't omitEmpty this field because an empty string here // means "delete the kubeadmin password secret" while a complete omission // of the field means "don't touch the secret". We never want the latter, @@ -38,14 +48,16 @@ type RecertConfig struct { KubeadminPasswordHash string `json:"kubeadmin_password_hash"` // WARNING: You probably don't want use `SummaryFile`! This will leak // private keys and tokens! - SummaryFile string `json:"summary_file,omitempty"` - SummaryFileClean string `json:"summary_file_clean,omitempty"` - StaticDirs []string `json:"static_dirs,omitempty"` - StaticFiles []string `json:"static_files,omitempty"` - CNSanReplaceRules []string `json:"cn_san_replace_rules,omitempty"` - UseKeyRules []string `json:"use_key_rules,omitempty"` - UseCertRules []string `json:"use_cert_rules,omitempty"` - PullSecret string `json:"pull_secret,omitempty"` + SummaryFile string `json:"summary_file,omitempty"` + SummaryFileClean string `json:"summary_file_clean,omitempty"` + CryptoDirs []string `json:"crypto_dirs,omitempty"` + CryptoFiles []string `json:"crypto_files,omitempty"` + ClusterCustomizationDirs []string `json:"cluster_customization_dirs,omitempty"` + ClusterCustomizationFiles []string `json:"cluster_customization_files,omitempty"` + CNSanReplaceRules []string `json:"cn_san_replace_rules,omitempty"` + UseKeyRules []string `json:"use_key_rules,omitempty"` + UseCertRules []string `json:"use_cert_rules,omitempty"` + PullSecret string `json:"pull_secret,omitempty"` } func FormatRecertProxyFromSeedReconfigProxy(proxy, statusProxy *seedreconfig.Proxy) string { @@ -59,6 +71,33 @@ func FormatRecertProxyFromSeedReconfigProxy(proxy, statusProxy *seedreconfig.Pro ) } +func SetRecertTrustedCaBundleFromSeedReconfigAdditionaTrustBundle(recertConfig *RecertConfig, additionalTrustBundle *seedreconfig.AdditionalTrustBundle) error { + if additionalTrustBundle == nil { + return nil + } + + if additionalTrustBundle.UserCaBundle != "" { + recertConfig.UserCaBundle = additionalTrustBundle.UserCaBundle + } + + if additionalTrustBundle.ProxyConfigmapName != "" { + if additionalTrustBundle.ProxyConfigmapBundle == "" { + return fmt.Errorf("proxy configmap bundle is empty while proxy configmap name is set") + } + + switch additionalTrustBundle.ProxyConfigmapName { + case common.ClusterAdditionalTrustBundleName: + recertConfig.ProxyTrustedCaBundle = fmt.Sprintf("%s:", common.ClusterAdditionalTrustBundleName) + case "": + recertConfig.ProxyTrustedCaBundle = "" + default: + recertConfig.ProxyTrustedCaBundle = fmt.Sprintf("%s:%s", additionalTrustBundle.ProxyConfigmapName, additionalTrustBundle.ProxyConfigmapBundle) + } + } + + return nil +} + // CreateRecertConfigFile function to create recert config file // those params will be provided to an installation script after reboot // that will run recert command with them @@ -80,6 +119,8 @@ func CreateRecertConfigFile(seedReconfig *seedreconfig.SeedReconfiguration, seed config.Proxy = FormatRecertProxyFromSeedReconfigProxy(seedReconfig.Proxy, seedReconfig.StatusProxy) + config.ProxyTrustedCaBundle = seedReconfig.ProxyTrustedCaBundle + config.InstallConfig = seedReconfig.InstallConfig config.SummaryFileClean = SummaryFile @@ -134,7 +175,7 @@ func CreateRecertConfigFileForSeedCreation(path string, withPassword bool) error } if err := utils.MarshalToFile(config, path); err != nil { - return fmt.Errorf("failed create recert config file for sed creatation in %s: %w", path, err) + return fmt.Errorf("failed create recert config file for seed creation in %s: %w", path, err) } return nil @@ -172,20 +213,19 @@ func CreateRecertConfigFileForSeedRestoration(path, originalPasswordHash string) config.KubeadminPasswordHash = originalPasswordHash if err := utils.MarshalToFile(config, path); err != nil { - return fmt.Errorf("failed to marshall recert config file for seed restoration: %w", err) + return fmt.Errorf("failed to marshal recert config file for seed restoration: %w", err) } return nil } func createBasicEmptyRecertConfig() RecertConfig { return RecertConfig{ - DryRun: false, - EtcdEndpoint: common.EtcdDefaultEndpoint, - StaticDirs: staticDirs, - StaticFiles: []string{ - "/host-etc/mcs-machine-config-content.json", - "/host-etc/mco/proxy.env", - }, + DryRun: false, + EtcdEndpoint: common.EtcdDefaultEndpoint, + CryptoDirs: cryptoDirs, + CryptoFiles: cryptoFiles, + ClusterCustomizationDirs: clusterCustomizationDirs, + ClusterCustomizationFiles: clusterCustomizationFiles, } } diff --git a/lca-cli/postpivot/postpivot.go b/lca-cli/postpivot/postpivot.go index 65514e941..9af8a2b46 100644 --- a/lca-cli/postpivot/postpivot.go +++ b/lca-cli/postpivot/postpivot.go @@ -121,10 +121,6 @@ func (p *PostPivot) PostPivotConfiguration(ctx context.Context) error { return fmt.Errorf("failed to run once recert for post pivot: %w", err) } - if err := p.copyClusterConfigFiles(); err != nil { - return fmt.Errorf("failed copy cluster config files: %w", err) - } - client, err := utils.CreateKubeClient(p.scheme, p.kubeconfig) if err != nil { return fmt.Errorf("failed to create k8s client, err: %w", err) @@ -408,14 +404,6 @@ func (p *PostPivot) restoreOadpDataProtectionApplication(ctx context.Context, cl return nil } -func (p *PostPivot) copyClusterConfigFiles() error { - if err := utils.CopyFileIfExists(path.Join(p.workingDir, common.ClusterConfigDir, path.Base(common.CABundleFilePath)), - common.CABundleFilePath); err != nil { - return fmt.Errorf("failed to copy cluster config file in %s: %w", common.ClusterConfigDir, err) - } - return nil -} - func (p *PostPivot) deleteAllOldMirrorResources(ctx context.Context, client runtimeclient.Client) error { p.log.Info("Deleting ImageContentSourcePolicy and ImageDigestMirrorSet if they exist") icsp := &operatorv1alpha1.ImageContentSourcePolicy{} diff --git a/lca-cli/seedclusterinfo/seedclusterinfo.go b/lca-cli/seedclusterinfo/seedclusterinfo.go index 097fda0f9..2b3565e98 100644 --- a/lca-cli/seedclusterinfo/seedclusterinfo.go +++ b/lca-cli/seedclusterinfo/seedclusterinfo.go @@ -68,9 +68,24 @@ type SeedClusterInfo struct { // with a proxy from seeds that don't have one. Similarly, installing a // cluster without a proxy from a seed with a proxy is also not supported. HasProxy bool `json:"has_proxy"` + + AdditionalTrustBundle *AdditionalTrustBundle `json:"additionalTrustBundle"` +} + +type AdditionalTrustBundle struct { + // Whether the "user-ca-bundle" configmap in the "openshift-config" + // namespace has a value or not + HasUserCaBundle bool `json:"hasUserCaBundle"` + + // The Proxy CR trustedCA configmap name + ProxyConfigmapName string `json:"proxyConfigmapName"` } -func NewFromClusterInfo(clusterInfo *utils.ClusterInfo, seedImagePullSpec string, hasProxy bool) *SeedClusterInfo { +func NewFromClusterInfo(clusterInfo *utils.ClusterInfo, + seedImagePullSpec string, + hasProxy bool, + additionalTrustBundle *AdditionalTrustBundle, +) *SeedClusterInfo { return &SeedClusterInfo{ SeedClusterOCPVersion: clusterInfo.OCPVersion, BaseDomain: clusterInfo.BaseDomain, @@ -81,6 +96,7 @@ func NewFromClusterInfo(clusterInfo *utils.ClusterInfo, seedImagePullSpec string MirrorRegistryConfigured: clusterInfo.MirrorRegistryConfigured, RecertImagePullSpec: seedImagePullSpec, HasProxy: hasProxy, + AdditionalTrustBundle: additionalTrustBundle, } } diff --git a/lca-cli/seedcreator/seedcreator.go b/lca-cli/seedcreator/seedcreator.go index 23464e76d..b79b5ca94 100644 --- a/lca-cli/seedcreator/seedcreator.go +++ b/lca-cli/seedcreator/seedcreator.go @@ -14,14 +14,17 @@ import ( "time" ignconfig "github.com/coreos/ignition/v2/config" + ocp_config_v1 "github.com/openshift/api/config/v1" mcv1 "github.com/openshift/api/machineconfiguration/v1" operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" cp "github.com/otiai10/copy" "github.com/samber/lo" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" runtime "sigs.k8s.io/controller-runtime/pkg/client" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/openshift-kni/lifecycle-agent/internal/common" "github.com/openshift-kni/lifecycle-agent/lca-cli/ops" @@ -193,6 +196,20 @@ func (s *SeedCreator) handleServices() error { }) } +func GetSeedAdditionalTrustBundleState(ctx context.Context, client runtimeclient.Client) (*seedclusterinfo.AdditionalTrustBundle, error) { + HasUserCaBundle, ProxyConfigmapName, err := utils.GetClusterAdditionalTrustBundleState(ctx, client) + if err != nil { + return nil, fmt.Errorf("failed to get cluster additional trust bundle state: %w", err) + } + + result := seedclusterinfo.AdditionalTrustBundle{ + HasUserCaBundle: HasUserCaBundle, + ProxyConfigmapName: ProxyConfigmapName, + } + + return &result, nil +} + func (s *SeedCreator) gatherClusterInfo(ctx context.Context) error { s.log.Info("Saving seed cluster configuration") clusterInfo, err := utils.GetClusterInfo(ctx, s.client) @@ -205,7 +222,17 @@ func (s *SeedCreator) gatherClusterInfo(ctx context.Context) error { return fmt.Errorf("failed to get proxy information: %w", err) } - seedClusterInfo := seedclusterinfo.NewFromClusterInfo(clusterInfo, s.recertContainerImage, hasProxy) + seedAdditionalTrustBundle, err := GetSeedAdditionalTrustBundleState(ctx, s.client) + if err != nil { + return fmt.Errorf("failed to get proxy information: %w", err) + } + + seedClusterInfo := seedclusterinfo.NewFromClusterInfo( + clusterInfo, + s.recertContainerImage, + hasProxy, + seedAdditionalTrustBundle, + ) if err := os.MkdirAll(common.SeedDataDir, os.ModePerm); err != nil { return fmt.Errorf("error creating SeedDataDir %s: %w", common.SeedDataDir, err) diff --git a/utils/client_helper.go b/utils/client_helper.go index 02314b7f5..800b2c2cd 100644 --- a/utils/client_helper.go +++ b/utils/client_helper.go @@ -13,9 +13,11 @@ import ( "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -256,6 +258,65 @@ func HasProxy(ctx context.Context, client runtimeclient.Client) (bool, error) { return true, nil } +func GetAdditionalTrustBundleFromConfigmap(ctx context.Context, client client.Client, configmapName string) (string, error) { + userCaBundleConfigmap := corev1.ConfigMap{} + if err := client.Get(ctx, types.NamespacedName{Name: common.ClusterAdditionalTrustBundleName, + Namespace: common.OpenshiftConfigNamespace}, &userCaBundleConfigmap); err != nil { + if errors.IsNotFound(err) { + return "", nil + } + + return "", fmt.Errorf("failed to get %s/%s configmap: %w", common.OpenshiftConfigNamespace, common.ClusterAdditionalTrustBundleName, err) + } + + if userCaBundleConfigmap.Data == nil { + return "", nil + } + + if userCaBundleConfigmap.Data[common.CaBundleDataKey] == "" { + return "", nil + } + + return userCaBundleConfigmap.Data[common.CaBundleDataKey], nil +} + +func GetClusterAdditionalTrustBundleState(ctx context.Context, client client.Client) (bool, string, error) { + clusterAdditionalTrustBundle, err := GetAdditionalTrustBundleFromConfigmap(ctx, client, common.ClusterAdditionalTrustBundleName) + if err != nil { + return false, "", fmt.Errorf("failed to get additional trust bundle from configmap: %w", err) + } + + hasUserCaBundle := clusterAdditionalTrustBundle != "" + + proxy := ocp_config_v1.Proxy{} + if err := client.Get(ctx, types.NamespacedName{Name: common.OpenshiftProxyCRName}, &proxy); err != nil { + return false, "", fmt.Errorf("failed to get proxy: %w", err) + } + + proxyCaBundle := "" + switch proxy.Spec.TrustedCA.Name { + case common.ClusterAdditionalTrustBundleName: + proxyCaBundle = clusterAdditionalTrustBundle + default: + proxyCaBundle, err = GetAdditionalTrustBundleFromConfigmap(ctx, client, proxy.Spec.TrustedCA.Name) + if err != nil { + return false, "", fmt.Errorf("failed to get additional trust bundle from configmap: %w", err) + } + + if proxyCaBundle == "" { + // This is a very weird but probably valid OCP configuration that we prefer to not support in LCA + return false, "", fmt.Errorf("proxy trustedCA configmap %s/%s exists but is empty", common.OpenshiftConfigNamespace, proxy.Spec.TrustedCA.Name) + } + } + + proxyConfigmapName := "" + if proxyCaBundle != "" { + proxyConfigmapName := proxy.Spec.TrustedCA.Name + } + + return hasUserCaBundle, proxyConfigmapName, nil +} + func ShouldOverrideSeedRegistry(ctx context.Context, client runtimeclient.Client, mirrorRegistryConfigured bool, releaseRegistry string) (bool, error) { mirroredRegistries, err := GetMirrorRegistrySourceRegistries(ctx, client) if err != nil {