diff --git a/Cargo.lock b/Cargo.lock index 5678b5e..9f30e4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,6 +175,75 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "crossbeam" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f25d8400f4a7a5778f0e4e52384a48cbd9b5c495d110786187fc750075277a2" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if", + "lazy_static", +] + [[package]] name = "crossterm" version = "0.23.2" @@ -710,6 +779,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "jwalk" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "172752e853a067cbce46427de8470ddf308af7fd8ceaf9b682ef31a5021b6bb9" +dependencies = [ + "crossbeam", + "rayon", +] + [[package]] name = "k8s-openapi" version = "0.14.0" @@ -802,14 +881,17 @@ dependencies = [ "comfy-table", "csv", "env_logger", + "jwalk", "k8s-openapi", "kube", "log", "openssl", + "rayon", "reqwest", "serde", "serde_json", "tokio", + "yaml-rust", ] [[package]] @@ -907,6 +989,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -1213,6 +1304,30 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd249e82c21598a9a426a4e00dd7adc1d640b22445ec8545feef801d1a74c221" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.13" diff --git a/Cargo.toml b/Cargo.toml index 5bb8edc..54f261d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] serde = {version = "1.0", features = ["derive"] } serde_json = "1.0" +yaml-rust = "0.4.5" kube = {version="0.71.0", features = ["runtime","derive"]} k8s-openapi = {version = "0.14.0",features=["v1_22"]} tokio = { version = "1.17.0", features = ["full"] } @@ -18,4 +19,6 @@ reqwest = { version = "0.11", features = ["json"] } clap = { version = "3.1.8", features = ["derive"] } comfy-table = "5.0.1" openssl = { version = "0.10", features = ["vendored"] } -csv="1.1.6" \ No newline at end of file +csv="1.1.6" +jwalk = "0.6.0" +rayon ="1.5.1" diff --git a/src/cluster.rs b/src/cluster.rs index 0680f2c..c5d96f9 100644 --- a/src/cluster.rs +++ b/src/cluster.rs @@ -1,4 +1,4 @@ -use crate::utils::{ClusterOP, Deprecated, JsonDetails, TableDetails}; +use crate::utils::{ClusterOP, JsonDetails, TableDetails}; use anyhow::Result; use kube::{ api::{Api, DynamicObject, ResourceExt}, @@ -7,24 +7,18 @@ use kube::{ Client, }; use log::info; +use serde_json::Value; use std::sync::Arc; use tokio::task::spawn; -pub(crate) async fn get_cluster_resources(version: &str) -> Result { +pub(crate) async fn get_cluster_resources(version: &str, val: Vec) -> Result { let client = Client::try_default().await?; - //let current_config = kube::config::Config::infer().await?; let current_config = kube::config::Kubeconfig::read().unwrap(); info!( "Connected to cluster {:?}", current_config.current_context.unwrap() ); info!("Target apiversions v{}", version); - - let val = Deprecated::get_apiversion(format!("v{}", version).as_str()) - .await? - .as_array() - .unwrap() - .to_owned(); Ok(val .into_iter() .map(|resource| { @@ -82,7 +76,6 @@ pub(crate) async fn get_cluster_resources(version: &str) -> Result { } } } - Ok(temp_table) }) }) diff --git a/src/csv.rs b/src/csv.rs deleted file mode 100644 index e69de29..0000000 diff --git a/src/file.rs b/src/file.rs new file mode 100644 index 0000000..5c05fbe --- /dev/null +++ b/src/file.rs @@ -0,0 +1,106 @@ +use crate::utils::TableDetails; +use jwalk::{Parallelism, WalkDir}; +use rayon::iter::ParallelBridge; +use rayon::prelude::ParallelIterator; +use serde_json::Value; +use std::fs::File; +use std::io::Read; +use std::sync::mpsc::{channel, Sender}; +use yaml_rust::YamlLoader; + +pub(crate) fn search_files(t: Vec, loc: String) -> Vec { + let (sender, receiver) = channel(); + WalkDir::new(loc) + .parallelism(Parallelism::RayonNewPool(0)) + .into_iter() + .par_bridge() + .try_for_each_with( + sender, + |sed: &mut Sender<(String, String, String, String, String)>, op| { + let dir_entry = op.ok().unwrap(); + if dir_entry.file_type().is_file() { + let path = dir_entry.path(); + if let Some(yaml_file) = path.extension() { + if yaml_file.eq("yaml") { + let mut file = File::open(&path).expect("Unable to open file"); + let mut contents = String::new(); + file.read_to_string(&mut contents) + .expect("Unable to read file"); + let docs = YamlLoader::load_from_str(&contents).unwrap(); + // TODO: use combinators + for doc in docs { + if let Some(mut api_version) = doc["apiVersion"].as_str() { + for z in t.iter() { + if z["kind"] + .as_str() + .unwrap() + .eq(doc["kind"].as_str().unwrap()) + { + let mut supported_api_version = format!( + "{}/{}", + z["group"].as_str().unwrap(), + z["version"].as_str().unwrap() + ); + + let p = path + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); + if z["removed"].as_str().unwrap().eq("true") { + supported_api_version = "REMOVED".to_string(); + api_version = "REMOVED"; + + if let Err(e) = sed.send(( + doc["kind"].as_str().unwrap().to_string(), + supported_api_version, + api_version.to_string(), + doc["metadata"]["name"] + .as_str() + .unwrap() + .to_string(), + p, + )) { + return Err(e); + } + } else if supported_api_version.ne(api_version) { + if let Err(e) = sed.send(( + doc["kind"].as_str().unwrap().to_string(), + supported_api_version, + api_version.to_string(), + doc["metadata"]["name"] + .as_str() + .unwrap() + .to_string(), + p, + )) { + return Err(e); + } + } + } + } + } + } + } + } + } + Ok(()) + }, + ) + .expect("expected no send errors"); + let res: Vec<_> = receiver.iter().collect(); + let mut temp_table: Vec = vec![]; + for (kind, supported_api_version, deprecated_api_version, name, path) in res { + let p = path; + let t = TableDetails { + kind, + namespace: p.to_owned(), + name, + supported_api_version, + deprecated_api_version, + }; + temp_table.push(t); + } + temp_table +} diff --git a/src/main.rs b/src/main.rs index 37637bd..9dc49ef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,13 @@ -use utils::{DeprecatedResult, Output}; +use comfy_table::Table; +use file::search_files; +use utils::{generate_csv_header, Deprecated, Output, Scrape}; mod cluster; +mod file; mod utils; use crate::cluster::get_cluster_resources; -use crate::utils::{init_logger, ClusterOP}; +use crate::utils::{generate_table_header, init_logger, ClusterOP, VecTableDetails}; use clap::Parser; +use log::info; #[derive(Parser)] #[clap(author, version, about, long_about = None)] @@ -15,10 +19,23 @@ struct Sunset { output: Output, #[clap(long, short)] kubeconfig: Option, + /// Scrape the cluster for deprecated apis, + #[clap(long, short)] + file: Option, #[clap(short, long, parse(from_occurrences))] debug: usize, } +impl Sunset { + // if there is a mention of -d in the args, it will be scraping the directory else default will be cluster + fn check_scrape_type(&self) -> Scrape { + match &self.file { + Some(d) => Scrape::Dir(d.to_string()), + None => Scrape::Cluster, + } + } +} + #[tokio::main] async fn main() -> anyhow::Result<()> { let cli = Sunset::parse(); @@ -38,18 +55,66 @@ async fn main() -> anyhow::Result<()> { init_logger(); - let join_handle: ClusterOP = get_cluster_resources(version).await?; - let d = DeprecatedResult::new(join_handle); + let val = Deprecated::get_apiversion(format!("v{}", version).as_str()) + .await? + .as_array() + .unwrap() + .to_owned(); - match cli.output { - Output::Csv => { - d.generate_csv().await?; - } - Output::Junit => { - println!("Junit"); + match cli.check_scrape_type() { + Scrape::Cluster => { + let join_handle: ClusterOP = get_cluster_resources(version, val).await?; + match cli.output { + Output::Csv => { + let mut wtr = csv::Writer::from_path("./deprecated-list.csv")?; + generate_csv_header(&mut wtr, "Filename")?; + for task in join_handle { + let x: VecTableDetails = utils::VecTableDetails(task.await?.unwrap()); + x.generate_csv(&mut wtr)?; + } + wtr.flush()?; + info!( + "deprecated-list.csv written at location {}", + std::env::current_dir()?.as_os_str().to_str().unwrap() + ); + } + Output::Junit => { + println!("Junit"); + } + Output::Table => { + let mut t = Table::new(); + let t = generate_table_header(&mut t, "Namespace"); + for task in join_handle { + let x: VecTableDetails = utils::VecTableDetails(task.await?.unwrap()); + x.generate_table(t)?; + } + println!("{t}"); + } + } } - Output::Table => { - d.generate_table().await?; + Scrape::Dir(loc) => { + let x: VecTableDetails = utils::VecTableDetails(search_files(val, loc)); + match cli.output { + Output::Csv => { + let mut wtr = csv::Writer::from_path("./deprecated-list.csv")?; + generate_csv_header(&mut wtr, "Filename")?; + x.generate_csv(&mut wtr)?; + wtr.flush()?; + info!( + "deprecated-list.csv written at location {}", + std::env::current_dir()?.as_os_str().to_str().unwrap() + ); + } + Output::Junit => { + println!("Junit"); + } + Output::Table => { + let mut t = Table::new(); + let t = generate_table_header(&mut t, "filename"); + x.generate_table(t)?; + println!("{t}"); + } + } } } Ok(()) diff --git a/src/utils.rs b/src/utils.rs index 6640217..72fd5a5 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,84 +1,24 @@ use anyhow::Result; use clap::ArgEnum; -use comfy_table::Table; +use comfy_table::{ContentArrangement, Table}; +use csv::Writer; use env_logger::{Builder, Env}; -use log::info; + use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::io::Write; +use std::{fs::File, io::Write}; use tokio::task::JoinHandle; pub(crate) type ClusterOP = Vec>>>; -pub(crate) struct DeprecatedResult { - pub(crate) output: ClusterOP, -} - -impl DeprecatedResult { - pub(crate) fn new(d: ClusterOP) -> Self { - Self { output: d } - } - - pub(crate) async fn generate_table(self) -> Result<()> { - let mut table = Table::new(); - table.set_header(vec![ - "Kind", - "Namespace", - "Name", - "DeprecatedApiVersion", - "SupportedApiVersion", - ]); - for task in self.output { - let result = task.await?.unwrap(); - for r in result { - table.add_row(vec![ - r.kind, - r.namespace, - r.name, - r.deprecated_api_version, - r.supported_api_version, - ]); - } - } - println!("{table}"); - Ok(()) - } - - pub(crate) async fn generate_csv(self) -> Result<()> { - let mut wtr = csv::Writer::from_path("./deprecated-list.csv")?; - wtr.write_record([ - "Kind", - "Namespace", - "Name", - "DeprecatedApiVersion", - "SupportedApiVersion", - ])?; - for task in self.output { - let result = task.await?.unwrap(); - for r in result { - wtr.write_record([ - r.kind, - r.namespace, - r.name, - r.deprecated_api_version, - r.supported_api_version, - ])?; - } - } - wtr.flush()?; - info!( - "deprecated-list.csv written at location {}", - std::env::current_dir()?.as_os_str().to_str().unwrap() - ); - Ok(()) - } -} #[derive(Serialize, Deserialize, Default, Debug)] pub(crate) struct JsonDetails { #[serde(rename = "apiVersion")] pub(crate) api_version: String, } +pub(crate) struct VecTableDetails(pub(crate) Vec); + #[derive(Default)] pub(crate) struct TableDetails { pub(crate) kind: String, @@ -88,6 +28,55 @@ pub(crate) struct TableDetails { pub(crate) supported_api_version: String, } +impl VecTableDetails { + pub(crate) fn generate_table(self, t: &mut Table) -> Result<()> { + for r in self.0 { + t.add_row(vec![ + r.kind, + r.namespace, + r.name, + r.deprecated_api_version, + r.supported_api_version, + ]); + } + Ok(()) + } + pub(crate) fn generate_csv(self, wtr: &mut Writer) -> Result<()> { + for r in self.0 { + wtr.write_record([ + r.kind, + r.namespace, + r.name, + r.deprecated_api_version, + r.supported_api_version, + ])?; + } + Ok(()) + } +} + +pub(crate) fn generate_table_header<'a>(t: &'a mut Table, column_replace: &str) -> &'a mut Table { + t.set_header(vec![ + "Kind", + column_replace, + "Name", + "DeprecatedApiVersion", + "SupportedApiVersion", + ]) + .set_content_arrangement(ContentArrangement::Dynamic) +} + +pub(crate) fn generate_csv_header(wtr: &mut Writer, column_replace: &str) -> Result<()> { + wtr.write_record([ + "Kind", + column_replace, + "Name", + "DeprecatedApiVersion", + "SupportedApiVersion", + ])?; + Ok(()) +} + #[derive(Serialize, Deserialize, Default, Debug)] pub(crate) struct Deprecated { pub(crate) apis: serde_json::Value, @@ -127,3 +116,9 @@ pub(crate) enum Output { Junit, Csv, } + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum Scrape { + Cluster, + Dir(String), +}