From 4c86a981d667951fb854abcec09bf610092b7d14 Mon Sep 17 00:00:00 2001 From: WeetHet Date: Tue, 18 Feb 2025 08:36:20 +0200 Subject: [PATCH 1/2] Revert "devenv: disable the generate command" This reverts commit 649aad54979c88574918a83c3ba72b7043e1319e. --- devenv/src/devenv.rs | 116 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 108 insertions(+), 8 deletions(-) diff --git a/devenv/src/devenv.rs b/devenv/src/devenv.rs index bd50e9766..818aaa842 100644 --- a/devenv/src/devenv.rs +++ b/devenv/src/devenv.rs @@ -203,16 +203,116 @@ impl Devenv { pub async fn generate( &mut self, - _description: Option, - _host: &str, - _exclude: Vec, - _disable_telemetry: bool, + description: Option, + host: &str, + exclude: Vec, + disable_telemetry: bool, ) -> Result<()> { - bail!(indoc::formatdoc! {" - Generating devenv.nix has been temporarily removed. + let client = reqwest::Client::new(); + let mut request = client + .post(host) + .query(&[("disable_telemetry", disable_telemetry)]) + .header(reqwest::header::USER_AGENT, crate_version!()); + + let (asyncwriter, asyncreader) = tokio::io::duplex(256 * 1024); + let streamreader = tokio_util::io::ReaderStream::new(asyncreader); + + let (body_sender, body) = match description { + Some(desc) => { + request = request.query(&[("q", desc)]); + (None, None) + } + None => { + let git_output = std::process::Command::new("git") + .args(["ls-files", "-z"]) + .output() + .map_err(|_| { + miette::miette!("Failed to get list of files from git ls-files") + })?; + + let files = String::from_utf8_lossy(&git_output.stdout) + .split('\0') + .filter(|s| !s.is_empty()) + .filter(|s| !binaryornot::is_binary(s).unwrap_or(false)) + .map(PathBuf::from) + .collect::>(); + + if files.is_empty() { + warn!("No files found. Are you in a git repository?"); + return Ok(()); + } + + if let Some(stderr) = String::from_utf8(git_output.stderr).ok() { + if !stderr.is_empty() { + warn!("{}", &stderr); + } + } + + let body = reqwest::Body::wrap_stream(streamreader); + + request = request + .body(body) + .header(reqwest::header::CONTENT_TYPE, "application/x-tar"); + + (Some(tokio_tar::Builder::new(asyncwriter)), Some(files)) + } + }; + + info!("Generating devenv.nix and devenv.yaml, this should take about a minute ..."); + + let response_future = request.send(); + + let tar_task = async { + if let (Some(mut builder), Some(files)) = (body_sender, body) { + for path in files { + if path.is_file() && !exclude.iter().any(|exclude| path.starts_with(exclude)) { + builder.append_path(&path).await?; + } + } + builder.finish().await?; + } + Ok::<(), std::io::Error>(()) + }; + + let (response, _) = tokio::join!(response_future, tar_task); + + let response = response.into_diagnostic()?; + let status = response.status(); + if !status.is_success() { + let error_text = &response + .text() + .await + .unwrap_or_else(|_| "No error details available".to_string()); + bail!( + "Failed to generate (HTTP {}): {}", + &status.as_u16(), + match serde_json::from_str::(error_text) { + Ok(json) => json["message"] + .as_str() + .map(String::from) + .unwrap_or_else(|| error_text.clone()), + Err(_) => error_text.clone(), + } + ); + } - For more information, see: https://github.com/cachix/devenv/issues/1733 - "}) + let response_json: GenerateResponse = response.json().await.expect("Failed to parse JSON."); + + confirm_overwrite(Path::new("devenv.nix"), response_json.devenv_nix)?; + confirm_overwrite(Path::new("devenv.yaml"), response_json.devenv_yaml)?; + + info!( + "{}", + indoc::formatdoc!(" + Generated devenv.nix and devenv.yaml 🎉 + + Treat these as templates and open an issue at https://github.com/cachix/devenv/issues if you think we can do better! + + Start by running: + + $ devenv shell + ")); + Ok(()) } pub fn inputs_add(&mut self, name: &str, url: &str, follows: &[String]) -> Result<()> { From 6b618b352aea02f578e62649fb8d93694100a23a Mon Sep 17 00:00:00 2001 From: WeetHet Date: Tue, 18 Feb 2025 09:20:43 +0200 Subject: [PATCH 2/2] Move devenv generate to a separate binary --- Cargo.lock | 27 ++++- Cargo.toml | 5 + devenv-generate/Cargo.toml | 24 ++++ devenv-generate/src/main.rs | 236 ++++++++++++++++++++++++++++++++++++ devenv/Cargo.toml | 8 +- devenv/src/cnix.rs | 4 +- devenv/src/devenv.rs | 122 +------------------ devenv/src/main.rs | 38 +++--- package.nix | 1 + 9 files changed, 316 insertions(+), 149 deletions(-) create mode 100644 devenv-generate/Cargo.toml create mode 100644 devenv-generate/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 882f4c978..a51c5a64a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -611,6 +611,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "devenv-generate" +version = "1.4.1" +dependencies = [ + "binaryornot", + "clap", + "console", + "devenv", + "dialoguer", + "indoc", + "miette", + "once_cell", + "reqwest", + "serde", + "serde_json", + "similar", + "tokio", + "tokio-tar", + "tokio-util", + "tracing", +] + [[package]] name = "devenv-run-tests" version = "0.1.0" @@ -2459,9 +2481,9 @@ dependencies = [ [[package]] name = "similar" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "slab" @@ -3000,6 +3022,7 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", diff --git a/Cargo.toml b/Cargo.toml index dec0998db..6f6c7aa84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "2" members = [ "devenv", + "devenv-generate", "devenv-eval-cache", "devenv-run-tests", "devenv-tasks", @@ -24,6 +25,7 @@ nix-conf-parser = { path = "nix-conf-parser" } xtask = { path = "xtask" } ansiterm = "0.12.2" +binaryornot = "1.0.0" blake3 = "1.5.4" clap = { version = "4.5.1", features = ["derive", "cargo", "env"] } cli-table = "0.4.7" @@ -38,6 +40,7 @@ indoc = "2.0.4" lazy_static = "1.5.0" miette = { version = "7.1.0", features = ["fancy"] } nix = { version = "0.28.0", features = ["signal"] } +once_cell = "1.20.2" petgraph = "0.6.5" pretty_assertions = { version = "1.4.0", features = ["unstable"] } regex = "1.10.3" @@ -53,6 +56,7 @@ serde = { version = "1.0.197", features = ["derive"] } serde_json = "1.0.114" serde_repr = "0.1.19" serde_yaml = "0.9.32" +similar = "2.6.0" sha2 = "0.10.8" sqlx = { version = "0.8.2", features = ["time", "sqlite", "runtime-tokio"] } tempdir = "0.3.7" @@ -70,6 +74,7 @@ tokio = { version = "1.39.3", features = [ "sync", "time", ] } +tokio-util = { version = "0.7.12", features = ["io"] } which = "6.0.0" whoami = "1.5.1" xdg = "2.5.2" diff --git a/devenv-generate/Cargo.toml b/devenv-generate/Cargo.toml new file mode 100644 index 000000000..b602474cf --- /dev/null +++ b/devenv-generate/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "devenv-generate" +version = "1.4.1" +edition.workspace = true +license.workspace = true + +[dependencies] +devenv.workspace = true + +clap = { workspace = true, features = ["derive"] } +console.workspace = true +dialoguer.workspace = true +indoc.workspace = true +miette.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio = { workspace = true, features = ["full"] } +tokio-tar.workspace = true +tracing.workspace = true +tokio-util.workspace = true +similar.workspace = true +binaryornot.workspace = true +once_cell.workspace = true diff --git a/devenv-generate/src/main.rs b/devenv-generate/src/main.rs new file mode 100644 index 000000000..eb43b083f --- /dev/null +++ b/devenv-generate/src/main.rs @@ -0,0 +1,236 @@ +use clap::{crate_version, Parser}; +use devenv::{ + default_system, + log::{self, LogFormat}, +}; +use miette::{bail, IntoDiagnostic, Result}; +use similar::{ChangeTag, TextDiff}; +use std::path::{Path, PathBuf}; +use tracing::{info, warn}; + +#[derive(Parser, Debug)] +#[command( + name = "devenv-generate", + about = "Generate devenv.yaml and devenv.nix using AI" +)] +struct Cli { + #[arg(num_args=0.., trailing_var_arg = true)] + description: Vec, + + #[clap(long, default_value = "https://devenv.new")] + host: String, + + #[arg( + long, + help = "Paths to exclude during generation.", + value_name = "PATH" + )] + exclude: Vec, + + // https://consoledonottrack.com/ + #[clap(long, env = "DO_NOT_TRACK", action = clap::ArgAction::SetTrue)] + disable_telemetry: bool, + + #[arg( + short = 'V', + long, + global = true, + help = "Print version information", + long_help = "Print version information and exit" + )] + pub version: bool, + + #[arg(short, long, global = true, default_value_t = default_system())] + pub system: String, + + #[arg(short, long, global = true, help = "Enable additional debug logs.")] + verbose: bool, + + #[arg( + short, + long, + global = true, + conflicts_with = "verbose", + help = "Silence all logs" + )] + pub quiet: bool, + + #[arg( + long, + global = true, + help = "Configure the output format of the logs.", + default_value_t, + value_enum + )] + pub log_format: LogFormat, +} + +#[derive(serde::Deserialize)] +struct GenerateResponse { + devenv_nix: String, + devenv_yaml: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + + if cli.version { + println!("devenv {} ({})", crate_version!(), cli.system); + return Ok(()); + } + + let level = if cli.verbose { + log::Level::Debug + } else if cli.quiet { + log::Level::Silent + } else { + log::Level::default() + }; + + log::init_tracing(level, cli.log_format); + + let description = if !cli.description.is_empty() { + Some(cli.description.join(" ")) + } else { + None + }; + + let client = reqwest::Client::new(); + let mut request = client + .post(&cli.host) + .query(&[("disable_telemetry", cli.disable_telemetry)]) + .header(reqwest::header::USER_AGENT, crate_version!()); + + let (asyncwriter, asyncreader) = tokio::io::duplex(256 * 1024); + let streamreader = tokio_util::io::ReaderStream::new(asyncreader); + + let (body_sender, body) = match description { + Some(desc) => { + request = request.query(&[("q", desc)]); + (None, None) + } + None => { + let git_output = std::process::Command::new("git") + .args(["ls-files", "-z"]) + .output() + .map_err(|_| miette::miette!("Failed to get list of files from git ls-files"))?; + + let files = String::from_utf8_lossy(&git_output.stdout) + .split('\0') + .filter(|s| !s.is_empty()) + .filter(|s| !binaryornot::is_binary(s).unwrap_or(false)) + .map(PathBuf::from) + .collect::>(); + + if files.is_empty() { + warn!("No files found. Are you in a git repository?"); + return Ok(()); + } + + if let Ok(stderr) = String::from_utf8(git_output.stderr) { + if !stderr.is_empty() { + warn!("{}", &stderr); + } + } + + let body = reqwest::Body::wrap_stream(streamreader); + + request = request + .body(body) + .header(reqwest::header::CONTENT_TYPE, "application/x-tar"); + + (Some(tokio_tar::Builder::new(asyncwriter)), Some(files)) + } + }; + + info!("Generating devenv.nix and devenv.yaml, this should take about a minute ..."); + + let response_future = request.send(); + + let tar_task = async { + if let (Some(mut builder), Some(files)) = (body_sender, body) { + for path in files { + if path.is_file() && !cli.exclude.iter().any(|exclude| path.starts_with(exclude)) { + builder.append_path(&path).await?; + } + } + builder.finish().await?; + } + Ok::<(), std::io::Error>(()) + }; + + let (response, _) = tokio::join!(response_future, tar_task); + + let response = response.into_diagnostic()?; + let status = response.status(); + if !status.is_success() { + let error_text = &response + .text() + .await + .unwrap_or_else(|_| "No error details available".to_string()); + bail!( + "Failed to generate (HTTP {}): {}", + &status.as_u16(), + match serde_json::from_str::(error_text) { + Ok(json) => json["message"] + .as_str() + .map(String::from) + .unwrap_or_else(|| error_text.clone()), + Err(_) => error_text.clone(), + } + ); + } + + let response_json: GenerateResponse = response.json().await.expect("Failed to parse JSON."); + + confirm_overwrite(Path::new("devenv.nix"), response_json.devenv_nix)?; + confirm_overwrite(Path::new("devenv.yaml"), response_json.devenv_yaml)?; + + info!( + "{}", + indoc::formatdoc!(" + Generated devenv.nix and devenv.yaml 🎉 + + Treat these as templates and open an issue at https://github.com/cachix/devenv/issues if you think we can do better! + + Start by running: + + $ devenv shell + ")); + Ok(()) +} + +fn confirm_overwrite(file: &Path, contents: String) -> Result<()> { + if std::fs::metadata(file).is_ok() { + // first output the old version and propose new changes + let before = std::fs::read_to_string(file).expect("Failed to read file"); + + let diff = TextDiff::from_lines(&before, &contents); + + println!("\nChanges that will be made to {}:", file.to_string_lossy()); + for change in diff.iter_all_changes() { + let sign = match change.tag() { + ChangeTag::Delete => "\x1b[31m-\x1b[0m", + ChangeTag::Insert => "\x1b[32m+\x1b[0m", + ChangeTag::Equal => " ", + }; + print!("{}{}", sign, change); + } + + let confirm = dialoguer::Confirm::new() + .with_prompt(format!( + "{} already exists. Do you want to overwrite it?", + file.to_string_lossy() + )) + .interact() + .into_diagnostic()?; + + if confirm { + std::fs::write(file, contents).into_diagnostic()?; + } + } else { + std::fs::write(file, contents).into_diagnostic()?; + } + Ok(()) +} diff --git a/devenv/Cargo.toml b/devenv/Cargo.toml index c43078c48..35603453c 100644 --- a/devenv/Cargo.toml +++ b/devenv/Cargo.toml @@ -45,7 +45,7 @@ which.workspace = true whoami.workspace = true xdg.workspace = true tokio-tar.workspace = true -tokio-util = { version = "0.7.12", features = ["io"] } -similar = "2.6.0" -binaryornot = "1.0.0" -once_cell = "1.20.2" +tokio-util.workspace = true +similar.workspace = true +binaryornot.workspace = true +once_cell.workspace = true diff --git a/devenv/src/cnix.rs b/devenv/src/cnix.rs index eccb252c9..08d397049 100644 --- a/devenv/src/cnix.rs +++ b/devenv/src/cnix.rs @@ -182,7 +182,7 @@ impl<'a> Nix<'a> { pub fn repl(&self) -> Result<()> { let mut cmd = self.prepare_command("nix", &["repl", "."], &self.options)?; - cmd.exec(); + let _ = cmd.exec(); Ok(()) } @@ -425,7 +425,7 @@ impl<'a> Nix<'a> { && cmd.get_program().to_string_lossy().ends_with("bin/nix") { info!("Starting Nix debugger ..."); - cmd.arg("--debugger").exec(); + let _ = cmd.arg("--debugger").exec(); } if options.bail_on_error { diff --git a/devenv/src/devenv.rs b/devenv/src/devenv.rs index 818aaa842..6cd284831 100644 --- a/devenv/src/devenv.rs +++ b/devenv/src/devenv.rs @@ -194,127 +194,13 @@ impl Devenv { }; // run direnv allow - std::process::Command::new(direnv) + let _ = std::process::Command::new(direnv) .arg("allow") .current_dir(&target) .exec(); Ok(()) } - pub async fn generate( - &mut self, - description: Option, - host: &str, - exclude: Vec, - disable_telemetry: bool, - ) -> Result<()> { - let client = reqwest::Client::new(); - let mut request = client - .post(host) - .query(&[("disable_telemetry", disable_telemetry)]) - .header(reqwest::header::USER_AGENT, crate_version!()); - - let (asyncwriter, asyncreader) = tokio::io::duplex(256 * 1024); - let streamreader = tokio_util::io::ReaderStream::new(asyncreader); - - let (body_sender, body) = match description { - Some(desc) => { - request = request.query(&[("q", desc)]); - (None, None) - } - None => { - let git_output = std::process::Command::new("git") - .args(["ls-files", "-z"]) - .output() - .map_err(|_| { - miette::miette!("Failed to get list of files from git ls-files") - })?; - - let files = String::from_utf8_lossy(&git_output.stdout) - .split('\0') - .filter(|s| !s.is_empty()) - .filter(|s| !binaryornot::is_binary(s).unwrap_or(false)) - .map(PathBuf::from) - .collect::>(); - - if files.is_empty() { - warn!("No files found. Are you in a git repository?"); - return Ok(()); - } - - if let Some(stderr) = String::from_utf8(git_output.stderr).ok() { - if !stderr.is_empty() { - warn!("{}", &stderr); - } - } - - let body = reqwest::Body::wrap_stream(streamreader); - - request = request - .body(body) - .header(reqwest::header::CONTENT_TYPE, "application/x-tar"); - - (Some(tokio_tar::Builder::new(asyncwriter)), Some(files)) - } - }; - - info!("Generating devenv.nix and devenv.yaml, this should take about a minute ..."); - - let response_future = request.send(); - - let tar_task = async { - if let (Some(mut builder), Some(files)) = (body_sender, body) { - for path in files { - if path.is_file() && !exclude.iter().any(|exclude| path.starts_with(exclude)) { - builder.append_path(&path).await?; - } - } - builder.finish().await?; - } - Ok::<(), std::io::Error>(()) - }; - - let (response, _) = tokio::join!(response_future, tar_task); - - let response = response.into_diagnostic()?; - let status = response.status(); - if !status.is_success() { - let error_text = &response - .text() - .await - .unwrap_or_else(|_| "No error details available".to_string()); - bail!( - "Failed to generate (HTTP {}): {}", - &status.as_u16(), - match serde_json::from_str::(error_text) { - Ok(json) => json["message"] - .as_str() - .map(String::from) - .unwrap_or_else(|| error_text.clone()), - Err(_) => error_text.clone(), - } - ); - } - - let response_json: GenerateResponse = response.json().await.expect("Failed to parse JSON."); - - confirm_overwrite(Path::new("devenv.nix"), response_json.devenv_nix)?; - confirm_overwrite(Path::new("devenv.yaml"), response_json.devenv_yaml)?; - - info!( - "{}", - indoc::formatdoc!(" - Generated devenv.nix and devenv.yaml 🎉 - - Treat these as templates and open an issue at https://github.com/cachix/devenv/issues if you think we can do better! - - Start by running: - - $ devenv shell - ")); - Ok(()) - } - pub fn inputs_add(&mut self, name: &str, url: &str, follows: &[String]) -> Result<()> { self.config.add_input(name, url, follows); self.config.write(); @@ -1042,12 +928,6 @@ fn confirm_overwrite(file: &Path, contents: String) -> Result<()> { Ok(()) } -#[derive(Deserialize)] -struct GenerateResponse { - devenv_nix: String, - devenv_yaml: String, -} - pub struct DevEnv { output: Vec, gc_root: PathBuf, diff --git a/devenv/src/main.rs b/devenv/src/main.rs index bfbd12df4..0698488b6 100644 --- a/devenv/src/main.rs +++ b/devenv/src/main.rs @@ -1,3 +1,5 @@ +use std::{os::unix::process::CommandExt, process::Command}; + use clap::crate_version; use devenv::{ cli::{Cli, Commands, ContainerCommand, InputsCommand, ProcessesCommand, TasksCommand}, @@ -22,7 +24,7 @@ async fn main() -> Result<()> { let command = match cli.command { None | Some(Commands::Version) => return print_version(), Some(Commands::Direnvrc) => { - print!("{}", devenv::DIRENVRC.to_string()); + print!("{}", *devenv::DIRENVRC); return Ok(()); } Some(cmd) => cmd, @@ -132,25 +134,21 @@ async fn main() -> Result<()> { Ok(()) } Commands::Init { target } => devenv.init(&target), - Commands::Generate { - description, - host, - exclude, - disable_telemetry, - } => { - devenv - .generate( - if description.is_empty() { - None - } else { - Some(description.join(" ")) - }, - &host, - exclude, - disable_telemetry, - ) - .await - } + Commands::Generate { .. } => match which::which("devenv-generate") { + Ok(devenv_generate) => { + let error = Command::new(devenv_generate) + .args(std::env::args().skip(1).filter(|arg| arg != "generate")) + .exec(); + miette::bail!("failed to execute devenv-generate {error}"); + } + Err(_) => { + miette::bail!(indoc::formatdoc! {" + devenv-generate was not found in PATH + + It was moved to a separate binary due to https://github.com/cachix/devenv/issues/1733 + "}) + } + }, Commands::Search { name } => devenv.search(&name).await, Commands::Gc {} => devenv.gc(), Commands::Info {} => devenv.info().await, diff --git a/package.nix b/package.nix index ba4ce37ed..4b0b14183 100644 --- a/package.nix +++ b/package.nix @@ -24,6 +24,7 @@ rustPlatform.buildRustPackage { ".*Cargo\.toml" ".*Cargo\.lock" ".*devenv(/.*)?" + ".*devenv-generate(/.*)?" ".*devenv-eval-cache(/.*)?" ".*devenv-run-tests(/.*)?" ".*devenv-tasks(/.*)?"