Skip to content

Commit

Permalink
Proxy rename support
Browse files Browse the repository at this point in the history
MGMT-16796

Added new proxy option that sets the cluster proxy to the desired
configuration.

Proxy editing with recert is required because its inside machineconfigs,
we need to avoid a reboot
  • Loading branch information
omertuc committed Mar 4, 2024
1 parent 089d1a3 commit 5623444
Show file tree
Hide file tree
Showing 10 changed files with 559 additions and 4 deletions.
4 changes: 4 additions & 0 deletions run_seed.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ cluster_customization_dirs:
- backup/etc/pki/ca-trust
cluster_customization_files:
- backup/etc/mcs-machine-config-content.json
- backup/etc/mco/proxy.env
cn_san_replace_rules:
- api-int.seed.redhat.com:api-int.new-name.foo.com
- api.seed.redhat.com:api.new-name.foo.com
Expand Down Expand Up @@ -110,6 +111,7 @@ use_cert_rules:
cluster_rename: new-name:foo.com:some-random-infra-id
hostname: test.hostname
ip: 192.168.126.99
proxy: http://registry.kni-qe-0.lab.eng.rdu2.redhat.com:3128|http://registry.kni-qe-0.lab.eng.rdu2.redhat.com:3130|.cluster.local,.kni-qe-2.lab.eng.rdu2.redhat.com,.svc,127.0.0.1,2620:52:0:11c::/64,2620:52:0:11c::1,2620:52:0:11c::10,2620:52:0:11c::11,2620:52:0:199::/64,api-int.kni-qe-2.lab.eng.rdu2.redhat.com,fd01::/48,fd02::/112,localhost
kubeadmin_password_hash: "$2a$10$20Q4iRLy7cWZkjn/D07bF.RZQZonKwstyRGH0qiYbYRkx5Pe4Ztyi"
additional_trust_bundle: |
# Foo
Expand Down Expand Up @@ -182,6 +184,7 @@ else
--cluster-customization-dir backup/etc/machine-config-daemon \
--cluster-customization-dir backup/etc/pki/ca-trust \
--cluster-customization-file backup/etc/mcs-machine-config-content.json \
--cluster-customization-file backup/etc/mco/proxy.env \
\
--cn-san-replace api-int.seed.redhat.com:api-int.new-name.foo.com \
--cn-san-replace api.seed.redhat.com:api.new-name.foo.com \
Expand All @@ -192,6 +195,7 @@ else
--cluster-rename new-name:foo.com:some-random-infra-id \
--hostname test.hostname \
--ip 192.168.126.99 \
--proxy 'http://registry.kni-qe-0.lab.eng.rdu2.redhat.com:3128|http://registry.kni-qe-0.lab.eng.rdu2.redhat.com:3130|.cluster.local,.kni-qe-2.lab.eng.rdu2.redhat.com,.svc,127.0.0.1,2620:52:0:11c::/64,2620:52:0:11c::1,2620:52:0:11c::10,2620:52:0:11c::11,2620:52:0:199::/64,api-int.kni-qe-2.lab.eng.rdu2.redhat.com,fd01::/48,fd02::/112,localhost' \
--kubeadmin-password-hash '$2a$10$20Q4iRLy7cWZkjn/D07bF.RZQZonKwstyRGH0qiYbYRkx5Pe4Ztyi' \
--additional-trust-bundle ./hack/dummy_trust_bundle.pem \
--pull-secret '{"auths":{"empty_registry":{"username":"empty","password":"empty","auth":"ZW1wdHk6ZW1wdHk=","email":""}}}' \
Expand Down
9 changes: 8 additions & 1 deletion src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use self::{cli::Cli, path::ConfigPath};
use crate::{
cluster_crypto::REDACT_SECRETS,
cnsanreplace::{CnSanReplace, CnSanReplaceRules},
ocp_postprocess::cluster_domain_rename::params::ClusterNamesRename,
ocp_postprocess::{cluster_domain_rename::params::ClusterNamesRename, proxy_rename::args::Proxy},
use_cert::{UseCert, UseCertRules},
use_key::{UseKey, UseKeyRules},
};
Expand Down Expand Up @@ -208,6 +208,7 @@ impl RecertConfig {
cluster_rename: cli.cluster_rename,
hostname: cli.hostname,
ip: cli.ip,
proxy: cli.proxy,
kubeadmin_password_hash: cli.kubeadmin_password_hash,
pull_secret: cli.pull_secret,
additional_trust_bundle: cli.additional_trust_bundle,
Expand Down Expand Up @@ -335,6 +336,12 @@ fn parse_cluster_customization_config(
Some(value) => Some(value.as_str().context("pull_secret must be a string")?.to_string()),
None => None,
};
let proxy = match value.get("proxy") {
Some(value) => {
Some(Proxy::parse(value.as_str().context("proxy must be a string")?).context(format!("proxy {}", value.as_str().unwrap()))?)
}
None => None,
};
let set_kubeadmin_password_hash = match value.remove("kubeadmin_password_hash") {
Some(value) => Some(value.as_str().context("set_kubeadmin_password_hash must be a string")?.to_string()),
None => None,
Expand Down
9 changes: 8 additions & 1 deletion src/config/cli.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
use crate::{
cnsanreplace::CnSanReplace, ocp_postprocess::cluster_domain_rename::params::ClusterNamesRename, use_cert::UseCert, use_key::UseKey,
cnsanreplace::CnSanReplace,
ocp_postprocess::{cluster_domain_rename::params::ClusterNamesRename, proxy_rename::args::Proxy},
use_cert::UseCert,
use_key::UseKey,
};
use clap::Parser;
use clio::ClioPath;
Expand Down Expand Up @@ -67,6 +70,10 @@ pub(crate) struct Cli {
#[clap(long)]
pub(crate) ip: Option<String>,

/// If given, the cluster's HTTP proxy configuration will be modified to use this one instead.
#[clap(long, value_parser = Proxy::parse)]
pub(crate) proxy: Option<Proxy>,

/// Modify the OCP kubeadmin password secret hash. If given but empty, the kubeadmin password
/// secret will be deleted (thus disabling password login). If given and non-empty, the secret
/// will be updated with the given password hash, unless no existing kubeadmin secret resource
Expand Down
25 changes: 24 additions & 1 deletion src/ocp_postprocess.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use self::cluster_domain_rename::params::ClusterNamesRename;
use self::{cluster_domain_rename::params::ClusterNamesRename, proxy_rename::args::Proxy};
use crate::{
cluster_crypto::locations::K8sResourceLocation,
config::{path::ConfigPath, ClusterCustomizations},
Expand All @@ -21,9 +21,11 @@ mod fnv;
mod go_base32;
pub(crate) mod hostname_rename;
pub(crate) mod ip_rename;
pub(crate) mod proxy_rename;
pub(crate) mod pull_secret_rename;

/// Perform some OCP-related post-processing to make some OCP operators happy
#[allow(clippy::too_many_arguments)]
pub(crate) async fn ocp_postprocess(
in_memory_etcd_client: &Arc<InMemoryK8sEtcd>,
cluster_customizations: &ClusterCustomizations,
Expand Down Expand Up @@ -93,6 +95,12 @@ async fn run_cluster_customizations(
.context("setting kubeadmin password hash")?;
}

if let Some(proxy) = proxy {
proxy_rename(in_memory_etcd_client, proxy, static_dirs, static_files)
.await
.context("renaming proxy")?;
}

if let Some(pull_secret) = &cluster_customizations.pull_secret {
log::info!("setting new pull_secret");
pull_secret_rename(in_memory_etcd_client, pull_secret, dirs, files)
Expand Down Expand Up @@ -505,3 +513,18 @@ pub(crate) async fn additional_trust_bundle_rename(

Ok(())
}

pub(crate) async fn proxy_rename(
in_memory_etcd_client: &Arc<InMemoryK8sEtcd>,
proxy: &Proxy,
static_dirs: &[ConfigPath],
static_files: &[ConfigPath],
) -> Result<()> {
let etcd_client = in_memory_etcd_client;

proxy_rename::rename_all(etcd_client, proxy, static_dirs, static_files)
.await
.context("renaming all")?;

Ok(())
}
74 changes: 74 additions & 0 deletions src/ocp_postprocess/proxy_rename.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
pub(crate) mod args;

use crate::{config::ConfigPath, k8s_etcd::InMemoryK8sEtcd};
use anyhow::{Context, Result};
use std::{path::Path, sync::Arc};

use self::args::Proxy;

mod etcd_rename;
mod filesystem_rename;
mod utils;

pub(crate) async fn rename_all(
etcd_client: &Arc<InMemoryK8sEtcd>,
proxy: &Proxy,
static_dirs: &[ConfigPath],
static_files: &[ConfigPath],
) -> Result<(), anyhow::Error> {
fix_etcd_resources(etcd_client, proxy).await.context("renaming etcd resources")?;

fix_filesystem_resources(proxy, static_dirs, static_files)
.await
.context("renaming filesystem resources")?;

Ok(())
}

async fn fix_filesystem_resources(proxy: &Proxy, static_dirs: &[ConfigPath], static_files: &[ConfigPath]) -> Result<()> {
for dir in static_dirs {
fix_dir_resources(proxy, dir).await?;
}

for file in static_files {
fix_file_resources(proxy, file).await?;
}

Ok(())
}

async fn fix_dir_resources(proxy: &Proxy, dir: &Path) -> Result<()> {
filesystem_rename::rename_proxy_env_dir(proxy, dir)
.await
.context("fixing etcd static pods")?;

filesystem_rename::fix_filesystem_currentconfig(proxy, dir)
.await
.context("renaming currentconfig")?;

filesystem_rename::fix_pods_yaml(proxy, dir).await.context("renaming pod yaml")?;

Ok(())
}

async fn fix_file_resources(proxy: &Proxy, file: &Path) -> Result<()> {
filesystem_rename::rename_proxy_env_file(proxy, file)
.await
.context("fixing etcd static pods")?;

Ok(())
}

async fn fix_etcd_resources(etcd_client: &Arc<InMemoryK8sEtcd>, proxy: &Proxy) -> Result<()> {
etcd_rename::fix_machineconfigs(etcd_client, proxy)
.await
.context("fixing machineconfigs")?;

etcd_rename::fix_proxy(etcd_client, proxy).await.context("fixing proxy")?;

etcd_rename::fix_controllerconfigs(etcd_client, proxy)
.await
.context("fixing machineconfigs")?;

Ok(())
}
30 changes: 30 additions & 0 deletions src/ocp_postprocess/proxy_rename/args.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
use anyhow::{ensure, Result};

#[derive(Clone, serde::Serialize)]
pub(crate) struct Proxy {
pub(crate) http_proxy: String,
pub(crate) https_proxy: String,
pub(crate) no_proxy: String,
}

impl Proxy {
pub(crate) fn parse(value: &str) -> Result<Self> {
let parts = value.split('|').collect::<Vec<_>>();

ensure!(
parts.len() == 3,
"expected three parts separated by '|' in proxy argument, i.e. '<http_proxy>|<cluster-base-domain>|<infra-id>', found {}",
parts.len()
);

let http_proxy = parts[0].to_string();
let https_proxy = parts[1].to_string();
let no_proxy = parts[2].to_string();

Ok(Self {
http_proxy,
https_proxy,
no_proxy,
})
}
}
131 changes: 131 additions & 0 deletions src/ocp_postprocess/proxy_rename/etcd_rename.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
use super::{args::Proxy, utils::fix_machineconfig};
use crate::{
cluster_crypto::locations::K8sResourceLocation,
k8s_etcd::{get_etcd_json, put_etcd_yaml, InMemoryK8sEtcd},
};
use anyhow::{Context, Result};
use futures_util::future::join_all;
use serde_json::Value;
use std::sync::Arc;

pub(crate) async fn fix_machineconfigs(etcd_client: &Arc<InMemoryK8sEtcd>, proxy: &Proxy) -> Result<()> {
join_all(
etcd_client
.list_keys("machineconfiguration.openshift.io/machineconfigs")
.await?
.into_iter()
.map(|key| async move {
let etcd_result = etcd_client
.get(key.clone())
.await
.with_context(|| format!("getting key {:?}", key))?
.context("key disappeared")?;
let value: Value = serde_yaml::from_slice(etcd_result.value.as_slice())
.with_context(|| format!("deserializing value of key {:?}", key,))?;
let k8s_resource_location = K8sResourceLocation::try_from(&value)?;

let mut machineconfig = get_etcd_json(etcd_client, &k8s_resource_location)
.await?
.context("no machineconfig")?;

fix_machineconfig(&mut machineconfig, proxy).context("fixing machineconfig")?;

put_etcd_yaml(etcd_client, &k8s_resource_location, machineconfig).await?;

Ok(())
}),
)
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;

Ok(())
}

pub(crate) async fn fix_proxy(etcd_client: &InMemoryK8sEtcd, proxy: &Proxy) -> Result<()> {
join_all(
etcd_client
.list_keys("config.openshift.io/proxies/cluster")
.await?
.into_iter()
.map(|key| async move {
let etcd_result = etcd_client
.get(key.clone())
.await
.with_context(|| format!("getting key {:?}", key))?
.context("key disappeared")?;

let value: Value = serde_yaml::from_slice(etcd_result.value.as_slice())
.with_context(|| format!("deserializing value of key {:?}", key,))?;

let k8s_resource_location = K8sResourceLocation::try_from(&value)?;

let mut cluster_proxy = get_etcd_json(etcd_client, &k8s_resource_location)
.await?
.context("no machineconfig")?;

let spec = cluster_proxy
.pointer_mut("/spec")
.context("no /spec")?
.as_object_mut()
.context("spec not an object")?;

spec.insert("httpProxy".to_string(), Value::String(proxy.http_proxy.clone()));
spec.insert("httpsProxy".to_string(), Value::String(proxy.https_proxy.clone()));
spec.insert("noProxy".to_string(), Value::String(proxy.no_proxy.clone()));

put_etcd_yaml(etcd_client, &k8s_resource_location, cluster_proxy).await?;

Ok(())
}),
)
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;

Ok(())
}

pub(crate) async fn fix_controllerconfigs(etcd_client: &InMemoryK8sEtcd, proxy: &Proxy) -> Result<()> {
join_all(
etcd_client
.list_keys("machineconfiguration.openshift.io/controllerconfigs/machine-config-controller")
.await?
.into_iter()
.map(|key| async move {
let etcd_result = etcd_client
.get(key.clone())
.await
.with_context(|| format!("getting key {:?}", key))?
.context("key disappeared")?;

let value: Value = serde_yaml::from_slice(etcd_result.value.as_slice())
.with_context(|| format!("deserializing value of key {:?}", key,))?;

let k8s_resource_location = K8sResourceLocation::try_from(&value)?;

let mut cluster_proxy = get_etcd_json(etcd_client, &k8s_resource_location)
.await?
.context("no machineconfig")?;

let spec_proxy = cluster_proxy
.pointer_mut("/spec/proxy")
.context("no /spec/proxy")?
.as_object_mut()
.context("/spec/proxy not an object")?;

spec_proxy.insert("httpProxy".to_string(), Value::String(proxy.http_proxy.clone()));
spec_proxy.insert("httpsProxy".to_string(), Value::String(proxy.https_proxy.clone()));
spec_proxy.insert("noProxy".to_string(), Value::String(proxy.no_proxy.clone()));

put_etcd_yaml(etcd_client, &k8s_resource_location, cluster_proxy).await?;

Ok(())
}),
)
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;

Ok(())
}
Loading

0 comments on commit 5623444

Please sign in to comment.