MGMT-16873: Change pull secret as part of recert reconfiguration

run_seed.sh

@@ -108,6 +108,7 @@ summary_file: summary.yaml
 summary_file_clean: summary_redacted.yaml
 extend_expiration: true
 force_expire: false
+pull_secret: "{\"auths\":{\"empty_registry\":{\"username\":\"empty\",\"password\":\"empty\",\"auth\":\"ZW1wdHk6ZW1wdHk=\",\"email\":\"\"}}}"
 threads: 1
 ') cargo run --release
 else
@@ -128,6 +129,7 @@ else
 		--kubeadmin-password-hash '$2a$10$20Q4iRLy7cWZkjn/D07bF.RZQZonKwstyRGH0qiYbYRkx5Pe4Ztyi' \
 		--summary-file summary.yaml \
 		--summary-file-clean summary_redacted.yaml \
+		--pull-secret '{"auths":{"empty_registry":{"username":"empty","password":"empty","auth":"ZW1wdHk6ZW1wdHk=","email":""}}}' \
 		--extend-expiration
 	# --regenerate-server-ssh-keys backup/etc/ssh/ \
 fi

src/config.rs

@@ -68,6 +68,7 @@ pub(crate) struct RecertConfig {
     pub(crate) hostname: Option<String>,
     pub(crate) ip: Option<String>,
     pub(crate) kubeadmin_password_hash: Option<String>,
+    pub(crate) pull_secret: Option<String>,
     pub(crate) threads: Option<usize>,
     pub(crate) regenerate_server_ssh_keys: Option<ConfigPath>,
     pub(crate) summary_file: Option<ConfigPath>,
@@ -246,6 +247,11 @@ impl RecertConfig {
             None => None,
         };
 
+        let pull_secret = match value.remove("pull_secret") {
+            Some(value) => Some(value.as_str().context("pull_secret must be a string")?.to_string()),
+            None => None,
+        };
+
         let set_kubeadmin_password_hash = match value.remove("kubeadmin_password_hash") {
             Some(value) => Some(value.as_str().context("set_kubeadmin_password_hash must be a string")?.to_string()),
             None => None,
@@ -314,6 +320,7 @@ impl RecertConfig {
             hostname,
             ip,
             kubeadmin_password_hash: set_kubeadmin_password_hash,
+            pull_secret,
             threads,
             regenerate_server_ssh_keys,
             summary_file,
@@ -358,6 +365,7 @@ impl RecertConfig {
             hostname: cli.hostname,
             ip: cli.ip,
             kubeadmin_password_hash: cli.kubeadmin_password_hash,
+            pull_secret: cli.pull_secret,
             threads: cli.threads,
             regenerate_server_ssh_keys: cli.regenerate_server_ssh_keys.map(ConfigPath::from),
             summary_file: cli.summary_file.map(ConfigPath::from),

src/config/cli.rs

@@ -58,6 +58,11 @@ pub(crate) struct Cli {
     #[clap(long)]
     pub(crate) kubeadmin_password_hash: Option<String>,
 
+    /// If given, the cluster resources that include the pull secret will be modified to use this
+    /// one instead.
+    #[clap(long)]
+    pub(crate) pull_secret: Option<String>,
+
     /// A list of CNs and the private keys to use for their certs. By default, new keys will be
     /// generated for all regenerated certificates, this option allows you to use existing keys
     /// instead. Must come in pairs of CN and private key file path, separated by a space. For

src/ocp_postprocess.rs

@@ -19,6 +19,7 @@ pub(crate) mod cluster_domain_rename;
 mod fnv;
 pub(crate) mod hostname_rename;
 pub(crate) mod ip_rename;
+pub(crate) mod pull_secret_rename;
 
 /// Perform some OCP-related post-processing to make some OCP operators happy
 pub(crate) async fn ocp_postprocess(
@@ -27,6 +28,7 @@ pub(crate) async fn ocp_postprocess(
     hostname: &Option<String>,
     ip: &Option<String>,
     kubeadmin_password_hash: &Option<String>,
+    pull_secret: &Option<String>,
     static_dirs: &Vec<ConfigPath>,
     static_files: &Vec<ConfigPath>,
 ) -> Result<()> {
@@ -71,6 +73,13 @@ pub(crate) async fn ocp_postprocess(
             .context("setting kubeadmin password hash")?;
     }
 
+    if let Some(pull_secret) = pull_secret {
+        log::info!("setting new pull_secret");
+        pull_secret_rename(in_memory_etcd_client, pull_secret, static_dirs, static_files)
+            .await
+            .context("renaming pull_secret")?;
+    }
+
     fix_deployment_dep_annotations(
         in_memory_etcd_client,
         K8sResourceLocation::new(Some("openshift-apiserver"), "Deployment", "apiserver", "v1"),
@@ -454,3 +463,18 @@ pub(crate) async fn ip_rename(
 
     Ok(())
 }
+
+pub(crate) async fn pull_secret_rename(
+    in_memory_etcd_client: &Arc<InMemoryK8sEtcd>,
+    pull_secret: &str,
+    static_dirs: &[ConfigPath],
+    static_files: &[ConfigPath],
+) -> Result<()> {
+    let etcd_client = in_memory_etcd_client;
+
+    pull_secret_rename::rename_all(etcd_client, pull_secret, static_dirs, static_files)
+        .await
+        .context("renaming all")?;
+
+    Ok(())
+}

src/ocp_postprocess/pull_secret_rename.rs

@@ -0,0 +1,63 @@
+use crate::{config::ConfigPath, k8s_etcd::InMemoryK8sEtcd};
+use anyhow::{Context, Result};
+use std::{path::Path, sync::Arc};
+
+mod etcd_rename;
+mod filesystem_rename;
+mod utils;
+
+pub(crate) async fn rename_all(
+    etcd_client: &Arc<InMemoryK8sEtcd>,
+    pull_secret: &str,
+    static_dirs: &[ConfigPath],
+    static_files: &[ConfigPath],
+) -> Result<(), anyhow::Error> {
+    fix_etcd_resources(etcd_client, pull_secret)
+        .await
+        .context("renaming etcd resources")?;
+
+    fix_filesystem_resources(pull_secret, static_dirs, static_files)
+        .await
+        .context("renaming filesystem resources")?;
+
+    Ok(())
+}
+
+async fn fix_filesystem_resources(pull_secret: &str, static_dirs: &[ConfigPath], static_files: &[ConfigPath]) -> Result<(), anyhow::Error> {
+    for dir in static_dirs {
+        fix_dir_resources(pull_secret, dir).await?;
+    }
+    for file in static_files {
+        fix_file_resources(pull_secret, file).await?;
+    }
+
+    Ok(())
+}
+
+async fn fix_dir_resources(pull_secret: &str, dir: &Path) -> Result<(), anyhow::Error> {
+    filesystem_rename::fix_filesystem_currentconfig(pull_secret, dir)
+        .await
+        .context("renaming currentconfig")?;
+
+    filesystem_rename::fix_filesystem_pull_secret(pull_secret, dir)
+        .await
+        .context("renaming config.json")?;
+    Ok(())
+}
+
+async fn fix_file_resources(pull_secret: &str, file: &Path) -> Result<(), anyhow::Error> {
+    filesystem_rename::fix_filesystem_mcs_machine_config_content(pull_secret, file)
+        .await
+        .context("fix filesystem mcs machine config content")?;
+    Ok(())
+}
+
+async fn fix_etcd_resources(etcd_client: &Arc<InMemoryK8sEtcd>, pull_secret: &str) -> Result<()> {
+    etcd_rename::fix_machineconfigs(etcd_client, pull_secret)
+        .await
+        .context("fixing machine configs")?;
+    etcd_rename::fix_pull_secret_secret(etcd_client, pull_secret)
+        .await
+        .context("fixing secret")?;
+    Ok(())
+}

src/ocp_postprocess/pull_secret_rename/etcd_rename.rs

@@ -0,0 +1,72 @@
+use super::utils::override_machineconfig_source;
+use crate::{
+    cluster_crypto::locations::K8sResourceLocation,
+    k8s_etcd::{get_etcd_json, put_etcd_yaml, InMemoryK8sEtcd},
+};
+use anyhow::{Context, Result};
+use futures_util::future::join_all;
+use serde_json::Value;
+use std::sync::Arc;
+
+pub(crate) async fn fix_machineconfigs(etcd_client: &Arc<InMemoryK8sEtcd>, pull_secret: &str) -> Result<()> {
+    join_all(
+        etcd_client
+            .list_keys("machineconfiguration.openshift.io/machineconfigs")
+            .await?
+            .into_iter()
+            .map(|key| async move {
+                let etcd_result = etcd_client
+                    .get(key.clone())
+                    .await
+                    .with_context(|| format!("getting key {:?}", key))?
+                    .context("key disappeared")?;
+                let value: Value = serde_yaml::from_slice(etcd_result.value.as_slice())
+                    .with_context(|| format!("deserializing value of key {:?}", key,))?;
+                let k8s_resource_location = K8sResourceLocation::try_from(&value)?;
+
+                let mut machineconfig = get_etcd_json(etcd_client, &k8s_resource_location)
+                    .await?
+                    .context("no machineconfig")?;
+
+                override_machineconfig_source(&mut machineconfig, pull_secret, "/var/lib/kubelet/config.json")
+                    .context("fixing machineconfig")?;
+
+                put_etcd_yaml(etcd_client, &k8s_resource_location, machineconfig).await?;
+
+                Ok(())
+            }),
+    )
+    .await
+    .into_iter()
+    .collect::<Result<Vec<_>>>()?;
+
+    Ok(())
+}
+
+pub(crate) async fn fix_pull_secret_secret(etcd_client: &Arc<InMemoryK8sEtcd>, pull_secret: &str) -> Result<()> {
+    let k8s_resource_location = K8sResourceLocation::new(Some("openshift-config"), "Secret", "pull-secret", "v1");
+
+    log::info!("setting pull secret secret");
+    let mut secret = get_etcd_json(etcd_client, &k8s_resource_location)
+        .await?
+        .context(format!("couldn't find {}", k8s_resource_location))?;
+
+    let data = secret
+        .pointer_mut("/data")
+        .context("no .data")?
+        .as_object_mut()
+        .context("data not an object")?;
+
+    data.insert(
+        ".dockerconfigjson".to_string(),
+        serde_json::Value::Array(
+            pull_secret
+                .as_bytes()
+                .iter()
+                .map(|byte| serde_json::Value::Number(serde_json::Number::from(*byte)))
+                .collect(),
+        ),
+    );
+    put_etcd_yaml(etcd_client, &k8s_resource_location, secret).await?;
+    Ok(())
+}

src/ocp_postprocess/pull_secret_rename/filesystem_rename.rs

@@ -0,0 +1,89 @@
+use super::utils::override_machineconfig_source;
+use crate::file_utils::{self, commit_file, read_file_to_string};
+use anyhow::{self, Context, Result};
+use futures_util::future::join_all;
+use serde_json::Value;
+use std::path::Path;
+
+pub(crate) async fn fix_filesystem_mcs_machine_config_content(pull_secret: &str, file_path: &Path) -> Result<()> {
+    if let Some(file_name) = file_path.file_name() {
+        if let Some(file_name) = file_name.to_str() {
+            if file_name == "mcs-machine-config-content.json" {
+                let contents = read_file_to_string(file_path)
+                    .await
+                    .context("reading machine config currentconfig")?;
+
+                let mut config: Value = serde_json::from_str(&contents).context("parsing currentconfig")?;
+
+                override_machineconfig_source(&mut config, pull_secret, "/var/lib/kubelet/config.json")?;
+
+                commit_file(file_path, serde_json::to_string(&config).context("serializing currentconfig")?)
+                    .await
+                    .context("writing currentconfig to disk")?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+pub(crate) async fn fix_filesystem_currentconfig(pull_secret: &str, dir: &Path) -> Result<()> {
+    join_all(file_utils::globvec(dir, "**/currentconfig")?.into_iter().map(|file_path| {
+        let config_path = file_path.clone();
+        let pull_secret = pull_secret.to_string();
+        tokio::spawn(async move {
+            async move {
+                let contents = read_file_to_string(&file_path).await.context("reading pull secret data")?;
+                let mut config: Value = serde_json::from_str(&contents).context("parsing currentconfig")?;
+
+                override_machineconfig_source(&mut config, &pull_secret, "/var/lib/kubelet/config.json")?;
+
+                commit_file(file_path, serde_json::to_string(&config).context("serializing currentconfig")?)
+                    .await
+                    .context("writing currentconfig to disk")?;
+
+                anyhow::Ok(())
+            }
+            .await
+            .context(format!("fixing currentconfig {:?}", config_path))
+        })
+    }))
+    .await
+    .into_iter()
+    .collect::<core::result::Result<Vec<_>, _>>()?
+    .into_iter()
+    .collect::<Result<Vec<_>>>()?;
+
+    Ok(())
+}
+
+pub(crate) async fn fix_filesystem_pull_secret(pull_secret: &str, dir: &Path) -> Result<()> {
+    if let Some(dir_name) = dir.file_name() {
+        if let Some(dir_name) = dir_name.to_str() {
+            // TODO: add verification that config.json as actually pull_secret
+            if dir_name == "kubelet" {
+                log::info!("setting pull secret in config.json");
+                join_all(file_utils::globvec(dir, "**/config.json")?.into_iter().map(|file_path| {
+                    let config_path = file_path.clone();
+                    let pull_secret = pull_secret.to_string();
+                    tokio::spawn(async move {
+                        async move {
+                            commit_file(file_path, &pull_secret).await.context("writing config.json to disk")?;
+
+                            anyhow::Ok(())
+                        }
+                        .await
+                        .context(format!("fixing config.json {:?}", config_path))
+                    })
+                }))
+                .await
+                .into_iter()
+                .collect::<core::result::Result<Vec<_>, _>>()?
+                .into_iter()
+                .collect::<Result<Vec<_>>>()?;
+            }
+        }
+    }
+
+    Ok(())
+}

src/ocp_postprocess/pull_secret_rename/utils.rs

@@ -0,0 +1,38 @@
+use anyhow::{Context, Result};
+use serde_json::Value;
+
+use crate::file_utils;
+
+pub(crate) fn override_machineconfig_source(machineconfig: &mut Value, new_source: &str, path: &str) -> Result<()> {
+    let pointer_mut = machineconfig.pointer_mut("/spec/config/storage/files");
+    if pointer_mut.is_none() {
+        // Not all machineconfigs have files to look at and that's ok
+        return Ok(());
+    };
+
+    let find_map = pointer_mut
+        .context("no /spec/config/storage/files")?
+        .as_array_mut()
+        .context("files not an array")?
+        .iter_mut()
+        .find_map(|file| (file.pointer("/path")? == &path).then_some(file));
+
+    if find_map.is_none() {
+        // Not all machineconfigs have the file we're looking for and that's ok
+        return Ok(());
+    };
+
+    let file_contents = find_map
+        .context(format!("no {} file in machineconfig", &path))?
+        .pointer_mut("/contents")
+        .context("no .contents")?
+        .as_object_mut()
+        .context("annotations not an object")?;
+
+    file_contents.insert(
+        "source".to_string(),
+        serde_json::Value::String(file_utils::dataurl_encode(&new_source)),
+    );
+
+    Ok(())
+}