From 043d7d3e11c64eccf102fb389a1f3c90d6125036 Mon Sep 17 00:00:00 2001 From: Omer Tuchfeld Date: Mon, 4 Mar 2024 00:19:13 +0100 Subject: [PATCH] Use the new recert additional trust bundle options # Background / Context Recert recently added ([1], [2]) some options that allow changing the cluster's trust bundle (it's recommended you read the PRs for more background about this). # Issue / Requirement / Reason for change The lifecycle-agent doesn't make use of the new options added to recert # Solution / Feature Overview Change the lifecycle-agent to use the new options added to recert # Implementation Details Multiple new fields have been added. - `AdditionalTrustBundle` in `SeedReconfiguration`. This represents the trust bundle to be used for seed-reconfiguration. This contains the user-ca-bundle contents, the proxy configmap name, and the proxy configmap contents. - `AdditionalTrustBundle` in `SeedClusterInfo`. This represents the state of the trust bundle in the seed cluster. This is simply booleans indicating the presence or lack there-of of the user-ca-bundle and the proxy configmap name (only if it actually has contents, a configmap with no contents is considered invalid OCP configuration). This is useful for when we want to verify that the seed is compatible with our desired `SeedReconfiguration`. - `RecertConfig` will now use the new `CryptoDirs` and `CryptoFiles` fields to specify the directories and files that should be considered part of the cluster's crypto material. Along with the `ClusterCustomizationDirs` and `ClusterCustomizationFiles` fields that specify the directories and files involved in cluster customization. Since these no longer overlap when it comes to customizing the trust bundle, we must use these new fields instead of the old common `StaticDirs` and `StaticFiles` fields. [1] https://github.com/rh-ecosystem-edge/recert/pull/110 [2] https://github.com/rh-ecosystem-edge/recert/pull/140 --- api/seedreconfig/seedreconfig.go | 14 ++ controllers/prep_handlers.go | 42 ++++++ internal/clusterconfig/clusterconfig.go | 130 +++++++++++-------- internal/clusterconfig/clusterconfig_test.go | 26 +--- internal/common/consts.go | 4 +- internal/recert/recert.go | 92 +++++++++---- lca-cli/ops/ops.go | 1 + lca-cli/postpivot/postpivot.go | 12 -- lca-cli/seedclusterinfo/seedclusterinfo.go | 19 ++- lca-cli/seedcreator/seedcreator.go | 30 ++++- utils/client_helper.go | 63 +++++++++ 11 files changed, 312 insertions(+), 121 deletions(-) diff --git a/api/seedreconfig/seedreconfig.go b/api/seedreconfig/seedreconfig.go index 899136f78..7b476c0fc 100644 --- a/api/seedreconfig/seedreconfig.go +++ b/api/seedreconfig/seedreconfig.go @@ -132,6 +132,8 @@ type SeedReconfiguration struct { // upgraded cluster. In IBI, a fake install-config should be generated by // the IBIO. This parameter is required when the proxy parameters are set. InstallConfig string `json:"install_config,omitempty"` + + AdditionalTrustBundle AdditionalTrustBundle `json:"additionalTrustBundle,omitempty"` } type KubeConfigCryptoRetention struct { @@ -174,3 +176,15 @@ type Proxy struct { // +optional NoProxy string `json:"noProxy,omitempty"` } + +type AdditionalTrustBundle struct { + // The contents of the "user-ca-bundle" configmap in the "openshift-config" namepace + UserCaBundle string `json:"userCaBundle"` + + // The Proxy CR trustedCA configmap nam + ProxyConfigmapName string `json:"proxyConfigmapName"` + + // The contents of the ProxyConfigmapName configmap. Must equal + // UserCaBundle if ProxyConfigmapName is "user-ca-bundle" + ProxyConfigmapBundle string `json:"proxyConfigmapBundle"` +} diff --git a/controllers/prep_handlers.go b/controllers/prep_handlers.go index ab68c54f4..bb542bef4 100644 --- a/controllers/prep_handlers.go +++ b/controllers/prep_handlers.go @@ -124,6 +124,23 @@ func GetSeedImage(c client.Client, ctx context.Context, ibu *ibuv1.ImageBasedUpg return fmt.Errorf("checking seed image compatibility: %w", err) } + hasUserCaBundle, proxyConfigmapName, err := lcautils.GetClusterAdditionalTrustBundleState(ctx, c) + if err != nil { + return fmt.Errorf("failed to get cluster additional trust bundle state: %w", err) + } + + //nolint:staticcheck //lint:ignore SA9003 // else branch intentionally left empty + if seedInfo.AdditionalTrustBundle != nil { + if err := checkSeedImageAdditionalTrustBundleCompatibility(*seedInfo.AdditionalTrustBundle, hasUserCaBundle, proxyConfigmapName); err != nil { + return fmt.Errorf("checking seed image additional trust bundle compatibility: %w", err) + } + } else { + // For the sake of backwards compatibility, we allow older seed images + // that don't have information about the additional trust bundle. This + // means that upgrade will fail at the recert stage if there's a + // mismatch between the seed and the seed reconfiguration data. + } + return nil } @@ -214,6 +231,31 @@ func checkSeedImageProxyCompatibility(seedHasProxy, hasProxy bool) error { return nil } +// checkSeedImageAdditionalTrustBundleCompatibility checks for proxy +// configuration compatibility of the seed image vs the current cluster. If the +// seed image has a proxy and the cluster being upgraded doesn't, we cannot +// proceed as recert does not support proxy rename under those conditions. +// Similarly, we cannot proceed if the cluster being upgraded has a proxy but +// the seed image doesn't. +func checkSeedImageAdditionalTrustBundleCompatibility(seedAdditionalTrustBundle seedclusterinfo.AdditionalTrustBundle, hasUserCaBundle bool, proxyConfigmapName string) error { + if seedAdditionalTrustBundle.HasUserCaBundle && !hasUserCaBundle { + return fmt.Errorf("seed image has an %s/%s configmap but the cluster being upgraded does not, this combination is not supported", + common.OpenshiftConfigNamespace, common.ClusterAdditionalTrustBundleName) + } + + if !seedAdditionalTrustBundle.HasUserCaBundle && hasUserCaBundle { + return fmt.Errorf("seed image does not have an %s/%s configmap but the cluster being upgraded does, this combination is not supported", + common.OpenshiftConfigNamespace, common.ClusterAdditionalTrustBundleName) + } + + if seedAdditionalTrustBundle.ProxyConfigmapName != proxyConfigmapName { + return fmt.Errorf("seed image's Proxy trustedCA configmap name %q (oc get proxy -oyaml) mismatches cluster's name %q, this combination is not supported", + seedAdditionalTrustBundle.ProxyConfigmapName, proxyConfigmapName) + } + + return nil +} + // checkSeedImageFIPSCompatibility checks for FIPS configuration compatibility // of the seed image vs the current cluster. If the seed image has FIPS enabled // and the cluster being upgraded doesn't, we cannot proceed as recert does not diff --git a/internal/clusterconfig/clusterconfig.go b/internal/clusterconfig/clusterconfig.go index 1aac366ea..216926526 100644 --- a/internal/clusterconfig/clusterconfig.go +++ b/internal/clusterconfig/clusterconfig.go @@ -10,7 +10,6 @@ import ( v1 "github.com/openshift/api/config/v1" operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -34,16 +33,11 @@ import ( const ( manifestDir = "manifests" - proxyName = "cluster" - pullSecretName = "pull-secret" idmsFileName = "image-digest-mirror-set.json" icspsFileName = "image-content-source-policy-list.json" - caBundleCMName = "user-ca-bundle" - caBundleFileName = caBundleCMName + ".json" - // ssh authorized keys file created by mco from ssh machine configs sshKeyFile = "/home/core/.ssh/authorized_keys.d/ignition" ) @@ -85,9 +79,6 @@ func (r *UpgradeClusterConfigGather) FetchClusterConfig(ctx context.Context, ost if err := r.fetchClusterInfo(ctx, clusterConfigPath); err != nil { return err } - if err := r.fetchCABundle(ctx, manifestsDir, clusterConfigPath); err != nil { - return err - } if err := r.fetchICSPs(ctx, manifestsDir); err != nil { return err } @@ -145,9 +136,73 @@ func (r *UpgradeClusterConfigGather) GetKubeadminPasswordHash(ctx context.Contex return kubeadminPasswordHash, nil } +type AdditionalTrustBundle struct { + // The contents of the "user-ca-bundle" configmap in the "openshift-config" namepace + UserCaBundle string `json:"userCaBundle"` + + // The Proxy CR trustedCA configmap name + ProxyConfigmapName string `json:"proxyConfigmapName"` + + // The contents of the ProxyConfigmapName configmap. Must equal + // UserCaBundle if ProxyConfigmapName is "user-ca-bundle" + ProxyConfigmapBundle string `json:"proxyConfigmapBundle"` +} + +func newAdditionalTrustBundle(userCaBundle, proxyConfigmapName, proxyConfigmapBundle string) (*AdditionalTrustBundle, error) { + if proxyConfigmapName == common.ClusterAdditionalTrustBundleName && userCaBundle != proxyConfigmapBundle { + return nil, fmt.Errorf("proxyConfigmapName is %s, but userCaBundle and proxyConfigmapBundle differ", common.ClusterAdditionalTrustBundleName) + } + + return &AdditionalTrustBundle{ + UserCaBundle: userCaBundle, + ProxyConfigmapName: proxyConfigmapName, + ProxyConfigmapBundle: proxyConfigmapBundle, + }, nil +} + +func (r *UpgradeClusterConfigGather) GetAdditionalTrustBundle(ctx context.Context) (*AdditionalTrustBundle, error) { + clusterAdditionalTrustBundle, err := utils.GetAdditionalTrustBundleFromConfigmap(ctx, r.Client, common.ClusterAdditionalTrustBundleName) + if err != nil { + return nil, fmt.Errorf("failed to get additional trust bundle from configmap: %w", err) + } + + proxy := v1.Proxy{} + if err := r.Client.Get(ctx, types.NamespacedName{Name: common.OpenshiftProxyCRName}, &proxy); err != nil { + return nil, fmt.Errorf("failed to get proxy: %w", err) + + } + + proxyCaBundle := "" + switch proxy.Spec.TrustedCA.Name { + case "": + // No proxy CA bundle + case common.ClusterAdditionalTrustBundleName: + // Proxy is using the cluster's CA bundle + proxyCaBundle = clusterAdditionalTrustBundle + default: + // Proxy is using a different CA bundle, retrieve it from the named configmap + proxyCaBundle, err = utils.GetAdditionalTrustBundleFromConfigmap(ctx, r.Client, proxy.Spec.TrustedCA.Name) + if err != nil { + return nil, fmt.Errorf("failed to get additional trust bundle from configmap: %w", err) + } + + if proxyCaBundle == "" { + // This is a very weird but probably valid OCP configuration that we prefer to not support in LCA + return nil, fmt.Errorf("proxy trustedCA configmap %s/%s exists but is empty", common.OpenshiftConfigNamespace, proxy.Spec.TrustedCA.Name) + } + } + + additionalTrustBundle, err := newAdditionalTrustBundle(clusterAdditionalTrustBundle, proxy.Spec.TrustedCA.Name, proxyCaBundle) + if err != nil { + return nil, fmt.Errorf("failed to create additional trust bundle: %w", err) + } + + return additionalTrustBundle, err +} + func (r *UpgradeClusterConfigGather) GetProxy(ctx context.Context) (*seedreconfig.Proxy, *seedreconfig.Proxy, error) { proxy := v1.Proxy{} - if err := r.Client.Get(ctx, types.NamespacedName{Name: proxyName}, &proxy); err != nil { + if err := r.Client.Get(ctx, types.NamespacedName{Name: common.OpenshiftProxyCRName}, &proxy); err != nil { return nil, nil, fmt.Errorf("failed to get proxy: %w", err) } @@ -188,6 +243,7 @@ func SeedReconfigurationFromClusterInfo( proxy, statusProxy *seedreconfig.Proxy, installConfig string, + additionalTrustBundle *AdditionalTrustBundle, ) *seedreconfig.SeedReconfiguration { return &seedreconfig.SeedReconfiguration{ APIVersion: seedreconfig.SeedReconfigurationVersion, @@ -206,6 +262,11 @@ func SeedReconfigurationFromClusterInfo( StatusProxy: statusProxy, InstallConfig: installConfig, MachineNetwork: clusterInfo.MachineNetwork, + AdditionalTrustBundle: seedreconfig.AdditionalTrustBundle{ + UserCaBundle: additionalTrustBundle.UserCaBundle, + ProxyConfigmapName: additionalTrustBundle.ProxyConfigmapName, + ProxyConfigmapBundle: additionalTrustBundle.ProxyConfigmapBundle, + }, } } @@ -247,6 +308,11 @@ func (r *UpgradeClusterConfigGather) fetchClusterInfo(ctx context.Context, clust return err } + additionalTrustBundle, err := r.GetAdditionalTrustBundle(ctx) + if err != nil { + return err + } + installConfig, err := r.GetInstallConfig(ctx) if err != nil { return err @@ -260,6 +326,7 @@ func (r *UpgradeClusterConfigGather) fetchClusterInfo(ctx context.Context, clust proxy, statusProxy, installConfig, + additionalTrustBundle, ) filePath := filepath.Join(clusterConfigPath, common.SeedReconfigurationFileName) @@ -319,14 +386,6 @@ func (r *UpgradeClusterConfigGather) typeMetaForObject(o runtime.Object) (*metav }, nil } -func (r *UpgradeClusterConfigGather) cleanObjectMetadata(o client.Object) metav1.ObjectMeta { - return metav1.ObjectMeta{ - Name: o.GetName(), - Namespace: o.GetNamespace(), - Labels: o.GetLabels(), - } -} - func (r *UpgradeClusterConfigGather) getIDMSs(ctx context.Context) (v1.ImageDigestMirrorSetList, error) { idmsList := v1.ImageDigestMirrorSetList{} currentIdms := v1.ImageDigestMirrorSetList{} @@ -402,41 +461,6 @@ func (r *UpgradeClusterConfigGather) fetchICSPs(ctx context.Context, manifestsDi return nil } -func (r *UpgradeClusterConfigGather) fetchCABundle(ctx context.Context, manifestsDir, clusterConfigPath string) error { - r.Log.Info("Fetching user ca bundle") - caBundle := &corev1.ConfigMap{} - err := r.Client.Get(ctx, types.NamespacedName{Name: caBundleCMName, - Namespace: common.OpenshiftConfigNamespace}, caBundle) - if err != nil && errors.IsNotFound(err) { - return nil - } - if err != nil { - return fmt.Errorf("failed to get ca bundle cm, err: %w", err) - } - - typeMeta, err := r.typeMetaForObject(caBundle) - if err != nil { - return err - } - caBundle.TypeMeta = *typeMeta - caBundle.ObjectMeta = r.cleanObjectMetadata(caBundle) - - if err := utils.MarshalToFile(caBundle, filepath.Join(manifestsDir, caBundleFileName)); err != nil { - return fmt.Errorf("failed to write user ca bundle to %s, err: %w", - filepath.Join(manifestsDir, caBundleFileName), err) - } - - // we should copy ca-bundle from snoa as without doing it we will fail to pull images - // workaround for https://issues.redhat.com/browse/OCPBUGS-24035 - caBundleFilePath := filepath.Join(hostPath, common.CABundleFilePath) - r.Log.Info("Copying", "file", caBundleFilePath) - if err := utils.CopyFileIfExists(caBundleFilePath, filepath.Join(clusterConfigPath, filepath.Base(caBundleFilePath))); err != nil { - return fmt.Errorf("failed to copy ca-bundle file %s to %s, err %w", caBundleFilePath, clusterConfigPath, err) - } - - return nil -} - // gather network files and copy them func (r *UpgradeClusterConfigGather) fetchNetworkConfig(ostreeDir string) error { r.Log.Info("Fetching node network files") diff --git a/internal/clusterconfig/clusterconfig_test.go b/internal/clusterconfig/clusterconfig_test.go index 5ad26f64f..e2b59e0e2 100644 --- a/internal/clusterconfig/clusterconfig_test.go +++ b/internal/clusterconfig/clusterconfig_test.go @@ -404,7 +404,7 @@ func TestClusterConfig(t *testing.T) { Name: "2", }, Spec: operatorv1alpha1.ImageContentSourcePolicySpec{ RepositoryDigestMirrors: []operatorv1alpha1.RepositoryDigestMirrors{{Source: "icspData2"}}}}}, - caBundleCM: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: caBundleCMName, + caBundleCM: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: common.ClusterAdditionalTrustBundleName, Namespace: common.OpenshiftConfigNamespace}, Data: map[string]string{"test": "data"}}, expectedErr: false, validateFunc: func(t *testing.T, tempDir string, err error, ucc UpgradeClusterConfigGather) { @@ -438,17 +438,6 @@ func TestClusterConfig(t *testing.T) { assert.Contains(t, resultSourcesAsString, "icspData") assert.Contains(t, resultSourcesAsString, "icspData2") } - - // validate caBundle - caBundle := &corev1.ConfigMap{} - if err := utils.ReadYamlOrJSONFile(filepath.Join(manifestsDir, caBundleFileName), caBundle); err != nil { - t.Errorf("unexpected error: %v", err) - } - assert.Equal(t, caBundleCMName, caBundle.Name) - assert.Equal(t, caBundle.Data, map[string]string{"test": "data"}) - - _, err = os.Stat(filepath.Join(clusterConfigPath, filepath.Base(common.CABundleFilePath))) - assert.Nil(t, err) }, }, } @@ -458,19 +447,6 @@ func TestClusterConfig(t *testing.T) { t.Run(testCase.testCaseName, func(t *testing.T) { hostPath = clusterConfigDir - if testCase.caBundleCM != nil { - dir := filepath.Join(clusterConfigDir, filepath.Dir(common.CABundleFilePath)) - if err := os.MkdirAll(dir, 0o700); err != nil { - t.Errorf("unexpected error: %v", err) - } - newPath := filepath.Join(dir, filepath.Base(common.CABundleFilePath)) - f, err := os.Create(newPath) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - _ = f.Close() - } - sshKeyDir := filepath.Join(clusterConfigDir, filepath.Dir(sshKeyFile)) if err := os.MkdirAll(sshKeyDir, 0o700); err != nil { t.Errorf("unexpected error: %v", err) diff --git a/internal/common/consts.go b/internal/common/consts.go index c78c3adef..428d9cbfd 100644 --- a/internal/common/consts.go +++ b/internal/common/consts.go @@ -62,7 +62,6 @@ const ( EtcdContainerName = "recert_etcd" LvmConfigDir = "lvm-configuration" LvmDevicesPath = "/etc/lvm/devices/system.devices" - CABundleFilePath = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" LCAConfigDir = "/var/lib/lca" IBUAutoRollbackConfigFile = LCAConfigDir + "/autorollback_config.json" @@ -110,6 +109,9 @@ const ( NMConnectionFolder = "/etc/NetworkManager/system-connections" NetworkDir = "network-configuration" + + CaBundleDataKey = "ca-bundle.crt" + ClusterAdditionalTrustBundleName = "user-ca-bundle" ) // Annotation names and values related to extra manifest diff --git a/internal/recert/recert.go b/internal/recert/recert.go index 1a676a3bf..39c126509 100644 --- a/internal/recert/recert.go +++ b/internal/recert/recert.go @@ -19,18 +19,28 @@ const ( SummaryFile = "/var/tmp/recert-summary.yaml" ) -var staticDirs = []string{"/kubelet", "/kubernetes", "/machine-config-daemon"} +var ( + // we don't want pki to go through recertification, only cluster customization (additional trust bundle appears in /pki) + cryptoDirs = []string{"/kubelet", "/kubernetes", "/machine-config-daemon"} + clusterCustomizationDirs = []string{"/kubelet", "/kubernetes", "/machine-config-daemon", "/pki"} + + cryptoFiles = []string{"/host-etc/mcs-machine-config-content.json"} + clusterCustomizationFiles = []string{"/host-etc/mcs-machine-config-content.json", "/host-etc/mco/proxy.env"} +) type RecertConfig struct { - DryRun bool `json:"dry_run,omitempty"` - ExtendExpiration bool `json:"extend_expiration,omitempty"` - ForceExpire bool `json:"force_expire,omitempty"` - EtcdEndpoint string `json:"etcd_endpoint,omitempty"` - ClusterRename string `json:"cluster_rename,omitempty"` - Hostname string `json:"hostname,omitempty"` - IP string `json:"ip,omitempty"` - Proxy string `json:"proxy,omitempty"` - InstallConfig string `json:"install_config,omitempty"` + DryRun bool `json:"dry_run,omitempty"` + ExtendExpiration bool `json:"extend_expiration,omitempty"` + ForceExpire bool `json:"force_expire,omitempty"` + EtcdEndpoint string `json:"etcd_endpoint,omitempty"` + ClusterRename string `json:"cluster_rename,omitempty"` + Hostname string `json:"hostname,omitempty"` + IP string `json:"ip,omitempty"` + Proxy string `json:"proxy,omitempty"` + InstallConfig string `json:"install_config,omitempty"` + UserCaBundle string `json:"user_ca_bundle,omitempty"` + ProxyTrustedCaBundle string `json:"proxy_trusted_ca_bundle,omitempty"` + // We intentionally don't omitEmpty this field because an empty string here // means "delete the kubeadmin password secret" while a complete omission // of the field means "don't touch the secret". We never want the latter, @@ -38,14 +48,16 @@ type RecertConfig struct { KubeadminPasswordHash string `json:"kubeadmin_password_hash"` // WARNING: You probably don't want use `SummaryFile`! This will leak // private keys and tokens! - SummaryFile string `json:"summary_file,omitempty"` - SummaryFileClean string `json:"summary_file_clean,omitempty"` - StaticDirs []string `json:"static_dirs,omitempty"` - StaticFiles []string `json:"static_files,omitempty"` - CNSanReplaceRules []string `json:"cn_san_replace_rules,omitempty"` - UseKeyRules []string `json:"use_key_rules,omitempty"` - UseCertRules []string `json:"use_cert_rules,omitempty"` - PullSecret string `json:"pull_secret,omitempty"` + SummaryFile string `json:"summary_file,omitempty"` + SummaryFileClean string `json:"summary_file_clean,omitempty"` + CryptoDirs []string `json:"crypto_dirs,omitempty"` + CryptoFiles []string `json:"crypto_files,omitempty"` + ClusterCustomizationDirs []string `json:"cluster_customization_dirs,omitempty"` + ClusterCustomizationFiles []string `json:"cluster_customization_files,omitempty"` + CNSanReplaceRules []string `json:"cn_san_replace_rules,omitempty"` + UseKeyRules []string `json:"use_key_rules,omitempty"` + UseCertRules []string `json:"use_cert_rules,omitempty"` + PullSecret string `json:"pull_secret,omitempty"` } func FormatRecertProxyFromSeedReconfigProxy(proxy, statusProxy *seedreconfig.Proxy) string { @@ -59,6 +71,29 @@ func FormatRecertProxyFromSeedReconfigProxy(proxy, statusProxy *seedreconfig.Pro ) } +func SetRecertTrustedCaBundleFromSeedReconfigAdditionaTrustBundle(recertConfig *RecertConfig, additionalTrustBundle seedreconfig.AdditionalTrustBundle) error { + if additionalTrustBundle.UserCaBundle != "" { + recertConfig.UserCaBundle = additionalTrustBundle.UserCaBundle + } + + if additionalTrustBundle.ProxyConfigmapName != "" { + if additionalTrustBundle.ProxyConfigmapBundle == "" { + return fmt.Errorf("proxy configmap bundle is empty while proxy configmap name is set") + } + + switch additionalTrustBundle.ProxyConfigmapName { + case common.ClusterAdditionalTrustBundleName: + recertConfig.ProxyTrustedCaBundle = fmt.Sprintf("%s:", common.ClusterAdditionalTrustBundleName) + case "": + recertConfig.ProxyTrustedCaBundle = "" + default: + recertConfig.ProxyTrustedCaBundle = fmt.Sprintf("%s:%s", additionalTrustBundle.ProxyConfigmapName, additionalTrustBundle.ProxyConfigmapBundle) + } + } + + return nil +} + // CreateRecertConfigFile function to create recert config file // those params will be provided to an installation script after reboot // that will run recert command with them @@ -80,6 +115,10 @@ func CreateRecertConfigFile(seedReconfig *seedreconfig.SeedReconfiguration, seed config.Proxy = FormatRecertProxyFromSeedReconfigProxy(seedReconfig.Proxy, seedReconfig.StatusProxy) + if err := SetRecertTrustedCaBundleFromSeedReconfigAdditionaTrustBundle(&config, seedReconfig.AdditionalTrustBundle); err != nil { + return fmt.Errorf("failed to set recert trusted ca bundle from seed reconfig additional trust bundle: %w", err) + } + config.InstallConfig = seedReconfig.InstallConfig config.SummaryFileClean = SummaryFile @@ -134,7 +173,7 @@ func CreateRecertConfigFileForSeedCreation(path string, withPassword bool) error } if err := utils.MarshalToFile(config, path); err != nil { - return fmt.Errorf("failed create recert config file for sed creatation in %s: %w", path, err) + return fmt.Errorf("failed create recert config file for seed creation in %s: %w", path, err) } return nil @@ -172,20 +211,19 @@ func CreateRecertConfigFileForSeedRestoration(path, originalPasswordHash string) config.KubeadminPasswordHash = originalPasswordHash if err := utils.MarshalToFile(config, path); err != nil { - return fmt.Errorf("failed to marshall recert config file for seed restoration: %w", err) + return fmt.Errorf("failed to marshal recert config file for seed restoration: %w", err) } return nil } func createBasicEmptyRecertConfig() RecertConfig { return RecertConfig{ - DryRun: false, - EtcdEndpoint: common.EtcdDefaultEndpoint, - StaticDirs: staticDirs, - StaticFiles: []string{ - "/host-etc/mcs-machine-config-content.json", - "/host-etc/mco/proxy.env", - }, + DryRun: false, + EtcdEndpoint: common.EtcdDefaultEndpoint, + CryptoDirs: cryptoDirs, + CryptoFiles: cryptoFiles, + ClusterCustomizationDirs: clusterCustomizationDirs, + ClusterCustomizationFiles: clusterCustomizationFiles, } } diff --git a/lca-cli/ops/ops.go b/lca-cli/ops/ops.go index 219aeb1d6..1f178c414 100644 --- a/lca-cli/ops/ops.go +++ b/lca-cli/ops/ops.go @@ -251,6 +251,7 @@ func (o *ops) RunRecert(recertContainerImage, authFile, recertConfigFile string, "-v", "/var/lib/kubelet:/kubelet", "-v", "/var/tmp:/var/tmp", "-v", "/etc/machine-config-daemon:/machine-config-daemon", + "-v", "/etc/pki:/pki", "-e", fmt.Sprintf("RECERT_CONFIG=%s", recertConfigFile), ) if authFile != "" { diff --git a/lca-cli/postpivot/postpivot.go b/lca-cli/postpivot/postpivot.go index 050601deb..5db3a69f0 100644 --- a/lca-cli/postpivot/postpivot.go +++ b/lca-cli/postpivot/postpivot.go @@ -127,10 +127,6 @@ func (p *PostPivot) PostPivotConfiguration(ctx context.Context) error { return fmt.Errorf("failed to run once recert for post pivot: %w", err) } - if err := p.copyClusterConfigFiles(); err != nil { - return fmt.Errorf("failed copy cluster config files: %w", err) - } - client, err := utils.CreateKubeClient(p.scheme, p.kubeconfig) if err != nil { return fmt.Errorf("failed to create k8s client, err: %w", err) @@ -464,14 +460,6 @@ func (p *PostPivot) restoreOadpDataProtectionApplication(ctx context.Context, cl return nil } -func (p *PostPivot) copyClusterConfigFiles() error { - if err := utils.CopyFileIfExists(path.Join(p.workingDir, common.ClusterConfigDir, path.Base(common.CABundleFilePath)), - common.CABundleFilePath); err != nil { - return fmt.Errorf("failed to copy cluster config file in %s: %w", common.ClusterConfigDir, err) - } - return nil -} - func (p *PostPivot) deleteAllOldMirrorResources(ctx context.Context, client runtimeclient.Client) error { p.log.Info("Deleting ImageContentSourcePolicy and ImageDigestMirrorSet if they exist") icsp := &operatorv1alpha1.ImageContentSourcePolicy{} diff --git a/lca-cli/seedclusterinfo/seedclusterinfo.go b/lca-cli/seedclusterinfo/seedclusterinfo.go index 729145857..db7dbe933 100644 --- a/lca-cli/seedclusterinfo/seedclusterinfo.go +++ b/lca-cli/seedclusterinfo/seedclusterinfo.go @@ -82,9 +82,25 @@ type SeedClusterInfo struct { // from seeds that don't have one. Similarly, installing a cluster without // FIPS from a seed with FIPS is also not supported. HasFIPS bool `json:"has_fips"` + + AdditionalTrustBundle *AdditionalTrustBundle `json:"additionalTrustBundle"` +} + +type AdditionalTrustBundle struct { + // Whether the "user-ca-bundle" configmap in the "openshift-config" + // namespace has a value or not + HasUserCaBundle bool `json:"hasUserCaBundle"` + + // The Proxy CR trustedCA configmap name + ProxyConfigmapName string `json:"proxyConfigmapName"` } -func NewFromClusterInfo(clusterInfo *utils.ClusterInfo, seedImagePullSpec string, hasProxy, hasFIPS bool) *SeedClusterInfo { +func NewFromClusterInfo(clusterInfo *utils.ClusterInfo, + seedImagePullSpec string, + hasProxy, + hasFIPS bool, + additionalTrustBundle *AdditionalTrustBundle, +) *SeedClusterInfo { return &SeedClusterInfo{ SeedClusterOCPVersion: clusterInfo.OCPVersion, BaseDomain: clusterInfo.BaseDomain, @@ -96,6 +112,7 @@ func NewFromClusterInfo(clusterInfo *utils.ClusterInfo, seedImagePullSpec string RecertImagePullSpec: seedImagePullSpec, HasProxy: hasProxy, HasFIPS: hasFIPS, + AdditionalTrustBundle: additionalTrustBundle, } } diff --git a/lca-cli/seedcreator/seedcreator.go b/lca-cli/seedcreator/seedcreator.go index 4d96cfd34..d476cdfd5 100644 --- a/lca-cli/seedcreator/seedcreator.go +++ b/lca-cli/seedcreator/seedcreator.go @@ -22,6 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" runtime "sigs.k8s.io/controller-runtime/pkg/client" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/openshift-kni/lifecycle-agent/internal/common" "github.com/openshift-kni/lifecycle-agent/lca-cli/ops" @@ -193,6 +194,20 @@ func (s *SeedCreator) handleServices() error { }) } +func GetSeedAdditionalTrustBundleState(ctx context.Context, client runtimeclient.Client) (*seedclusterinfo.AdditionalTrustBundle, error) { + HasUserCaBundle, ProxyConfigmapName, err := utils.GetClusterAdditionalTrustBundleState(ctx, client) + if err != nil { + return nil, fmt.Errorf("failed to get cluster additional trust bundle state: %w", err) + } + + result := seedclusterinfo.AdditionalTrustBundle{ + HasUserCaBundle: HasUserCaBundle, + ProxyConfigmapName: ProxyConfigmapName, + } + + return &result, nil +} + func (s *SeedCreator) gatherClusterInfo(ctx context.Context) error { s.log.Info("Saving seed cluster configuration") clusterInfo, err := utils.GetClusterInfo(ctx, s.client) @@ -207,10 +222,21 @@ func (s *SeedCreator) gatherClusterInfo(ctx context.Context) error { hasFIPS, err := utils.HasFIPS(ctx, s.client) if err != nil { - return fmt.Errorf("failed to get proxy information: %w", err) + return fmt.Errorf("failed to get FIPS information: %w", err) + } + + seedAdditionalTrustBundle, err := GetSeedAdditionalTrustBundleState(ctx, s.client) + if err != nil { + return fmt.Errorf("failed to get additional trust bundle information: %w", err) } - seedClusterInfo := seedclusterinfo.NewFromClusterInfo(clusterInfo, s.recertContainerImage, hasProxy, hasFIPS) + seedClusterInfo := seedclusterinfo.NewFromClusterInfo( + clusterInfo, + s.recertContainerImage, + hasProxy, + hasFIPS, + seedAdditionalTrustBundle, + ) if err := os.MkdirAll(common.SeedDataDir, os.ModePerm); err != nil { return fmt.Errorf("error creating SeedDataDir %s: %w", common.SeedDataDir, err) diff --git a/utils/client_helper.go b/utils/client_helper.go index 497f74b00..3626d6224 100644 --- a/utils/client_helper.go +++ b/utils/client_helper.go @@ -18,9 +18,11 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -334,6 +336,67 @@ func HasFIPS(ctx context.Context, client runtimeclient.Client) (bool, error) { return machineConfig.Spec.FIPS, nil } +func GetAdditionalTrustBundleFromConfigmap(ctx context.Context, client client.Client, configmapName string) (string, error) { + userCaBundleConfigmap := corev1.ConfigMap{} + if err := client.Get(ctx, types.NamespacedName{Name: common.ClusterAdditionalTrustBundleName, + Namespace: common.OpenshiftConfigNamespace}, &userCaBundleConfigmap); err != nil { + if errors.IsNotFound(err) { + return "", nil + } + + return "", fmt.Errorf("failed to get %s/%s configmap: %w", common.OpenshiftConfigNamespace, common.ClusterAdditionalTrustBundleName, err) + } + + if userCaBundleConfigmap.Data == nil { + return "", nil + } + + if userCaBundleConfigmap.Data[common.CaBundleDataKey] == "" { + return "", nil + } + + return userCaBundleConfigmap.Data[common.CaBundleDataKey], nil +} + +func GetClusterAdditionalTrustBundleState(ctx context.Context, client client.Client) (bool, string, error) { + clusterAdditionalTrustBundle, err := GetAdditionalTrustBundleFromConfigmap(ctx, client, common.ClusterAdditionalTrustBundleName) + if err != nil { + return false, "", fmt.Errorf("failed to get additional trust bundle from configmap: %w", err) + } + + hasUserCaBundle := clusterAdditionalTrustBundle != "" + + proxy := ocp_config_v1.Proxy{} + if err := client.Get(ctx, types.NamespacedName{Name: common.OpenshiftProxyCRName}, &proxy); err != nil { + return false, "", fmt.Errorf("failed to get proxy: %w", err) + } + + proxyCaBundle := "" + switch proxy.Spec.TrustedCA.Name { + case common.ClusterAdditionalTrustBundleName: + proxyCaBundle = clusterAdditionalTrustBundle + case "": + // No proxy trustedCA configmap is set, do nothing + default: + proxyCaBundle, err = GetAdditionalTrustBundleFromConfigmap(ctx, client, proxy.Spec.TrustedCA.Name) + if err != nil { + return false, "", fmt.Errorf("failed to get additional trust bundle from configmap: %w", err) + } + + if proxyCaBundle == "" { + // This is a very weird but probably valid OCP configuration that we prefer to not support in LCA + return false, "", fmt.Errorf("proxy trustedCA configmap %s/%s exists but is empty", common.OpenshiftConfigNamespace, proxy.Spec.TrustedCA.Name) + } + } + + proxyConfigmapName := "" + if proxyCaBundle != "" { + proxyConfigmapName = proxy.Spec.TrustedCA.Name + } + + return hasUserCaBundle, proxyConfigmapName, nil +} + func ShouldOverrideSeedRegistry(ctx context.Context, client runtimeclient.Client, mirrorRegistryConfigured bool, releaseRegistry string) (bool, error) { mirroredRegistries, err := GetMirrorRegistrySourceRegistries(ctx, client) if err != nil {