diff --git a/Cargo.toml b/Cargo.toml index 28050fe..eb094c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,3 +47,6 @@ reqwest = { version = "0.12.9", default-features = false, features = [ "macos-system-configuration", ] } tempfile = "3.13.0" + +[dev-dependencies] +rstest = "0.23.0" diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..c27ffb5 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,241 @@ +use anyhow::Result; +use rattler::install::{link_package, InstallDriver, InstallOptions}; +use rattler_conda_types::{prefix_record::PathsEntry, PackageRecord, PrefixRecord, RepoDataRecord}; +use rattler_package_streaming::fs::extract; +use std::{ + fmt::{Display, Formatter}, + path::{Path, PathBuf}, +}; + +use std::collections::HashSet; + +use anyhow::Context; +use rattler_conda_types::{package::ArchiveType, Platform}; +use rattler_index::{package_record_from_conda, package_record_from_tar_bz2}; +use reqwest::Url; + +pub struct PackageRecordVec(pub Vec); + +impl Display for PackageRecordVec { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "[{}]", + self.0 + .iter() + .map(|p| format!("{}", p)) + .collect::>() + .join(", ") + ) + } +} + +pub async fn pixi_inject(target_prefix: PathBuf, packages: Vec) -> Result<()> { + if packages.is_empty() { + return Err(anyhow::anyhow!("No packages were provided.")); + } + + let installed_packages = PrefixRecord::collect_from_prefix(&target_prefix)?; + let installed_package_records = installed_packages + .iter() + .map(|e| e.repodata_record.clone().package_record) + .collect::>(); + + let injected_packages = packages + .iter() + .map(|p| { + let record = package_record_from_archive(p)?; + anyhow::Ok((p.clone(), record)) + }) + .collect::>>()?; + + tracing::debug!( + "Installed packages: {}", + PackageRecordVec( + installed_packages + .iter() + .map(|p| p.repodata_record.package_record.clone()) + .collect::>() + ) + ); + tracing::debug!( + "Injected packages: {}", + PackageRecordVec( + injected_packages + .iter() + .map(|p| p.1.clone()) + .collect::>() + ) + ); + + let not_matching_platform = injected_packages + .iter() + .map(|p| &p.1) + .filter(|p| { + p.subdir != Platform::NoArch.to_string() && p.subdir != Platform::current().to_string() + }) + .collect::>(); + if !not_matching_platform.is_empty() { + return Err(anyhow::anyhow!( + "Packages with platform not matching the current platform ({}) were found: {}", + Platform::current().to_string(), + not_matching_platform + .into_iter() + .map(|p| format!("{} ({})", p, p.subdir.clone())) + .collect::>() + .join(", ") + )); + } + + tracing::debug!("Validating package compatibility with prefix."); + let all_records = installed_package_records + .iter() + .chain(injected_packages.iter().map(|p| &p.1)) + .collect(); + PackageRecord::validate(all_records)?; + tracing::debug!("All packages are compatible with the prefix."); + + // check whether the package is already installed + let injected_package_names = injected_packages + .iter() + .map(|p| p.1.name.as_normalized()) + .collect::>(); + let installed_package_names = installed_packages + .iter() + .map(|p| p.repodata_record.package_record.name.as_normalized()) + .collect::>(); + if !injected_package_names.is_disjoint(&installed_package_names) { + return Err(anyhow::anyhow!( + "Some of the packages are already installed: {}", + injected_package_names + .intersection(&injected_package_names) + .map(|p| p.to_string()) + .collect::>() + .join(", ") + )); + } + + eprintln!( + "⏳ Extracting and installing {} package{} to {}...", + packages.len(), + if packages.len() == 1 { "" } else { "s" }, + target_prefix.display() + ); + + let driver = InstallDriver::default(); + let options = InstallOptions::default(); + + for (path, package_record) in injected_packages.iter() { + let repodata_record = RepoDataRecord { + package_record: package_record.clone(), + file_name: path + .to_str() + .context("Could not create file name from path")? + .to_string(), + url: Url::from_file_path(path.canonicalize()?) + .map_err(|_| anyhow::anyhow!("Could not convert path to URL"))?, + channel: "".to_string(), + }; + install_package_to_environment_from_archive( + target_prefix.as_path(), + path.clone(), + repodata_record, + &driver, + &options, + ) + .await?; + tracing::debug!("Installed package: {}", path.display()); + } + + eprintln!("✅ Finished installing packages to prefix."); + + tracing::debug!("Finished running pixi-inject"); + Ok(()) +} + +fn package_record_from_archive(file: &Path) -> Result { + let archive_type = ArchiveType::split_str(file.to_string_lossy().as_ref()) + .context("Could not create ArchiveType")? + .1; + match archive_type { + ArchiveType::TarBz2 => package_record_from_tar_bz2(file), + ArchiveType::Conda => package_record_from_conda(file), + } + .map_err(|e| anyhow::anyhow!("Could not read package record from archive: {}", e)) +} + +/// Install a package into the environment and write a `conda-meta` file that +/// contains information about how the file was linked. +async fn install_package_to_environment_from_archive( + target_prefix: &Path, + package_path: PathBuf, + repodata_record: RepoDataRecord, + install_driver: &InstallDriver, + install_options: &InstallOptions, +) -> anyhow::Result<()> { + // Link the contents of the package into our environment. This returns all the + // paths that were linked. + let paths = link_package_from_archive( + &package_path, + target_prefix, + install_driver, + install_options.clone(), + ) + .await?; + + // Construct a PrefixRecord for the package + let prefix_record = PrefixRecord { + repodata_record, + package_tarball_full_path: None, + extracted_package_dir: None, + files: paths + .iter() + .map(|entry| entry.relative_path.clone()) + .collect(), + paths_data: paths.into(), + requested_spec: None, + link: None, + }; + + // Create the conda-meta directory if it doesnt exist yet. + let target_prefix = target_prefix.to_path_buf(); + let result = tokio::task::spawn_blocking(move || { + let conda_meta_path = target_prefix.join("conda-meta"); + std::fs::create_dir_all(&conda_meta_path)?; + + // Write the conda-meta information + let pkg_meta_path = conda_meta_path.join(prefix_record.file_name()); + prefix_record.write_to_path(pkg_meta_path, true) + }) + .await; + match result { + Ok(result) => Ok(result?), + Err(err) => { + if let Ok(panic) = err.try_into_panic() { + std::panic::resume_unwind(panic); + } + // The operation has been cancelled, so we can also just ignore everything. + Ok(()) + } + } +} + +// https://github.com/conda/rattler/pull/937 +async fn link_package_from_archive( + package_path: &Path, + target_dir: &Path, + driver: &InstallDriver, + options: InstallOptions, +) -> Result> { + let temp_dir = tempfile::tempdir()?; + + tracing::debug!( + "extracting {} to temporary directory {}", + package_path.display(), + temp_dir.path().display() + ); + extract(package_path, temp_dir.path())?; + link_package(temp_dir.path(), target_dir, driver, options) + .await + .map_err(|e| anyhow::anyhow!("Could not create temporary directory: {}", e)) +} diff --git a/src/main.rs b/src/main.rs index f5d40a3..4ba6a40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,21 +1,9 @@ -use std::{ - collections::HashSet, - fmt::{Display, Formatter}, - path::{Path, PathBuf}, -}; +use std::path::PathBuf; use clap::Parser; use clap_verbosity_flag::Verbosity; -use anyhow::{Context, Result}; -use rattler::install::{link_package, InstallDriver, InstallOptions}; -use rattler_conda_types::{ - package::ArchiveType, prefix_record::PathsEntry, PackageRecord, Platform, PrefixRecord, - RepoDataRecord, -}; -use rattler_index::{package_record_from_conda, package_record_from_tar_bz2}; -use rattler_package_streaming::fs::extract; -use reqwest::Url; +use anyhow::Result; use tracing_log::AsTrace; /* -------------------------------------------- CLI -------------------------------------------- */ @@ -36,22 +24,6 @@ struct Cli { /* -------------------------------------------- MAIN ------------------------------------------- */ -struct PackageRecordVec(Vec); - -impl Display for PackageRecordVec { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "[{}]", - self.0 - .iter() - .map(|p| format!("{}", p)) - .collect::>() - .join(", ") - ) - } -} - /// The main entrypoint for the pixi-inject CLI. #[tokio::main] async fn main() -> Result<()> { @@ -67,211 +39,5 @@ async fn main() -> Result<()> { let target_prefix = cli.prefix; let packages = cli.package; - if packages.is_empty() { - return Err(anyhow::anyhow!("No packages were provided.")); - } - - let installed_packages = PrefixRecord::collect_from_prefix(&target_prefix)?; - let installed_package_records = installed_packages - .iter() - .map(|e| e.repodata_record.clone().package_record) - .collect::>(); - - let injected_packages = packages - .iter() - .map(|p| { - let record = package_record_from_archive(p)?; - anyhow::Ok((p.clone(), record)) - }) - .collect::>>()?; - - tracing::debug!( - "Installed packages: {}", - PackageRecordVec( - installed_packages - .iter() - .map(|p| p.repodata_record.package_record.clone()) - .collect::>() - ) - ); - tracing::debug!( - "Injected packages: {}", - PackageRecordVec( - injected_packages - .iter() - .map(|p| p.1.clone()) - .collect::>() - ) - ); - - let not_matching_platform = injected_packages - .iter() - .map(|p| &p.1) - .filter(|p| { - p.subdir != Platform::NoArch.to_string() && p.subdir != Platform::current().to_string() - }) - .collect::>(); - if !not_matching_platform.is_empty() { - return Err(anyhow::anyhow!( - "Packages with platform not matching the current platform ({}) were found: {}", - Platform::current().to_string(), - not_matching_platform - .into_iter() - .map(|p| format!("{} ({})", p, p.subdir.clone())) - .collect::>() - .join(", ") - )); - } - - tracing::debug!("Validating package compatibility with prefix."); - let all_records = installed_package_records - .iter() - .chain(injected_packages.iter().map(|p| &p.1)) - .collect(); - PackageRecord::validate(all_records)?; - tracing::debug!("All packages are compatible with the prefix."); - - // check whether the package is already installed - let injected_package_names = injected_packages - .iter() - .map(|p| p.1.name.as_normalized()) - .collect::>(); - let installed_package_names = installed_packages - .iter() - .map(|p| p.repodata_record.package_record.name.as_normalized()) - .collect::>(); - if !injected_package_names.is_disjoint(&installed_package_names) { - return Err(anyhow::anyhow!( - "Some of the packages are already installed: {}", - injected_package_names - .intersection(&injected_package_names) - .map(|p| p.to_string()) - .collect::>() - .join(", ") - )); - } - - eprintln!( - "⏳ Extracting and installing {} package{} to {}...", - packages.len(), - if packages.len() == 1 { "" } else { "s" }, - target_prefix.display() - ); - - let driver = InstallDriver::default(); - let options = InstallOptions::default(); - - for (path, package_record) in injected_packages.iter() { - let repodata_record = RepoDataRecord { - package_record: package_record.clone(), - file_name: path - .to_str() - .context("Could not create file name from path")? - .to_string(), - url: Url::from_file_path(path.canonicalize()?) - .map_err(|_| anyhow::anyhow!("Could not convert path to URL"))?, - channel: "".to_string(), - }; - install_package_to_environment_from_archive( - target_prefix.as_path(), - path.clone(), - repodata_record, - &driver, - &options, - ) - .await?; - tracing::debug!("Installed package: {}", path.display()); - } - - eprintln!("✅ Finished installing packages to prefix."); - - tracing::debug!("Finished running pixi-inject"); - Ok(()) -} - -fn package_record_from_archive(file: &Path) -> Result { - let archive_type = ArchiveType::split_str(file.to_string_lossy().as_ref()) - .context("Could not create ArchiveType")? - .1; - match archive_type { - ArchiveType::TarBz2 => package_record_from_tar_bz2(file), - ArchiveType::Conda => package_record_from_conda(file), - } - .map_err(|e| anyhow::anyhow!("Could not read package record from archive: {}", e)) -} - -/// Install a package into the environment and write a `conda-meta` file that -/// contains information about how the file was linked. -async fn install_package_to_environment_from_archive( - target_prefix: &Path, - package_path: PathBuf, - repodata_record: RepoDataRecord, - install_driver: &InstallDriver, - install_options: &InstallOptions, -) -> anyhow::Result<()> { - // Link the contents of the package into our environment. This returns all the - // paths that were linked. - let paths = link_package_from_archive( - &package_path, - target_prefix, - install_driver, - install_options.clone(), - ) - .await?; - - // Construct a PrefixRecord for the package - let prefix_record = PrefixRecord { - repodata_record, - package_tarball_full_path: None, - extracted_package_dir: None, - files: paths - .iter() - .map(|entry| entry.relative_path.clone()) - .collect(), - paths_data: paths.into(), - requested_spec: None, - link: None, - }; - - // Create the conda-meta directory if it doesnt exist yet. - let target_prefix = target_prefix.to_path_buf(); - let result = tokio::task::spawn_blocking(move || { - let conda_meta_path = target_prefix.join("conda-meta"); - std::fs::create_dir_all(&conda_meta_path)?; - - // Write the conda-meta information - let pkg_meta_path = conda_meta_path.join(prefix_record.file_name()); - prefix_record.write_to_path(pkg_meta_path, true) - }) - .await; - match result { - Ok(result) => Ok(result?), - Err(err) => { - if let Ok(panic) = err.try_into_panic() { - std::panic::resume_unwind(panic); - } - // The operation has been cancelled, so we can also just ignore everything. - Ok(()) - } - } -} - -// https://github.com/conda/rattler/pull/937 -pub async fn link_package_from_archive( - package_path: &Path, - target_dir: &Path, - driver: &InstallDriver, - options: InstallOptions, -) -> Result> { - let temp_dir = tempfile::tempdir()?; - - tracing::debug!( - "extracting {} to temporary directory {}", - package_path.display(), - temp_dir.path().display() - ); - extract(package_path, temp_dir.path())?; - link_package(temp_dir.path(), target_dir, driver, options) - .await - .map_err(|e| anyhow::anyhow!("Could not create temporary directory: {}", e)) + pixi_inject::pixi_inject(target_prefix, packages).await } diff --git a/tests/integration_test.rs b/tests/integration_test.rs new file mode 100644 index 0000000..66b8d4e --- /dev/null +++ b/tests/integration_test.rs @@ -0,0 +1,121 @@ +use std::{path::PathBuf, process::Command}; + +use rattler_conda_types::Platform; +use rstest::*; +use tempfile::tempdir; + +struct Options { + prefix: PathBuf, + package: Vec, + _output_dir: tempfile::TempDir, +} + +#[fixture] +fn options(#[default("simple-example")] test_case: String) -> Options { + let output_dir = tempdir().unwrap(); + + // copy pixi.toml and pixi.lock to temporary location + let pixi_toml = output_dir.path().join("pixi.toml"); + let pixi_lock = output_dir.path().join("pixi.lock"); + std::fs::copy( + format!("tests/resources/{}/pixi.toml", test_case), + &pixi_toml, + ) + .unwrap(); + std::fs::copy( + format!("tests/resources/{}/pixi.lock", test_case), + &pixi_lock, + ) + .unwrap(); + + let pixi_install = Command::new("pixi") + .arg("install") + .arg("--manifest-path") + .arg(pixi_toml) + .output() + .unwrap(); + assert!(pixi_install.status.success()); + + let prefix = output_dir.path().join(".pixi").join("envs").join("default"); + + let package = match Platform::current() { + Platform::Linux64 => "linux-64-pydantic-core-2.26.0-py313h920b4c0_0.conda", + Platform::LinuxAarch64 => "linux-aarch64-pydantic-core-2.26.0-py313h8aa417a_0.conda", + Platform::OsxArm64 => "osx-arm64-pydantic-core-2.26.0-py313hdde674f_0.conda", + Platform::Osx64 => "osx-64-pydantic-core-2.26.0-py313h3c055b9_0.conda", + Platform::Win64 => "win-64-pydantic-core-2.26.0-py313hf3b5b86_0.conda", + _ => panic!("Unsupported platform"), + }; + let package = PathBuf::from(format!("tests/resources/packages/{}", package)); + assert!(package.exists()); + Options { + prefix, + package: vec![package], + _output_dir: output_dir, + } +} + +#[fixture] +fn required_fs_objects() -> Vec<&'static str> { + let pydantic_core_conda_meta = match Platform::current() { + Platform::Linux64 => "conda-meta/pydantic-core-2.26.0-py313h920b4c0_0.json", + Platform::LinuxAarch64 => "conda-meta/pydantic-core-2.26.0-py313h8aa417a_0.json", + Platform::OsxArm64 => "conda-meta/pydantic-core-2.26.0-py313hdde674f_0.json", + Platform::Osx64 => "conda-meta/pydantic-core-2.26.0-py313h3c055b9_0.json", + Platform::Win64 => "conda-meta/pydantic-core-2.26.0-py313hf3b5b86_0.json", + _ => panic!("Unsupported platform"), + }; + + vec![ + #[cfg(not(unix))] + "Lib\\site-packages\\pydantic_core", + #[cfg(unix)] + "lib/python3.13/site-packages/pydantic_core", + pydantic_core_conda_meta, + ] +} + +#[rstest] +#[tokio::test] +async fn test_simple_example(options: Options, required_fs_objects: Vec<&'static str>) { + pixi_inject::pixi_inject(options.prefix.clone(), options.package) + .await + .unwrap(); + + for fs_object in required_fs_objects { + assert!(options.prefix.join(fs_object).exists()); + } +} + +#[rstest] +#[tokio::test] +async fn test_install_twice(options: Options) { + pixi_inject::pixi_inject(options.prefix.clone(), options.package.clone()) + .await + .unwrap(); + + let result = pixi_inject::pixi_inject(options.prefix.clone(), options.package).await; + assert!(result.is_err()); + assert!(result + .err() + .unwrap() + .to_string() + .contains("Some of the packages are already installed: pydantic-core")) +} + +#[rstest] +#[case("already-installed-different-version".to_string())] +#[case("already-installed".to_string())] +#[tokio::test] +async fn test_already_installed( + #[case] _test_case: String, + #[with(_test_case.clone())] options: Options, +) { + let result = pixi_inject::pixi_inject(options.prefix.clone(), options.package).await; + assert!(result.is_err()); + assert!(result + .err() + .unwrap() + .to_string() + .contains("Some of the packages are already installed: pydantic-core")) +} diff --git a/examples/already-installed-different-version/pixi.lock b/tests/resources/already-installed-different-version/pixi.lock similarity index 100% rename from examples/already-installed-different-version/pixi.lock rename to tests/resources/already-installed-different-version/pixi.lock diff --git a/examples/already-installed-different-version/pixi.toml b/tests/resources/already-installed-different-version/pixi.toml similarity index 100% rename from examples/already-installed-different-version/pixi.toml rename to tests/resources/already-installed-different-version/pixi.toml diff --git a/examples/already-installed/pixi.lock b/tests/resources/already-installed/pixi.lock similarity index 100% rename from examples/already-installed/pixi.lock rename to tests/resources/already-installed/pixi.lock diff --git a/examples/already-installed/pixi.toml b/tests/resources/already-installed/pixi.toml similarity index 100% rename from examples/already-installed/pixi.toml rename to tests/resources/already-installed/pixi.toml diff --git a/examples/packages/linux-64-pydantic-core-2.26.0-py313h920b4c0_0.conda b/tests/resources/packages/linux-64-pydantic-core-2.26.0-py313h920b4c0_0.conda similarity index 100% rename from examples/packages/linux-64-pydantic-core-2.26.0-py313h920b4c0_0.conda rename to tests/resources/packages/linux-64-pydantic-core-2.26.0-py313h920b4c0_0.conda diff --git a/tests/resources/packages/linux-aarch64-pydantic-core-2.26.0-py313h8aa417a_0.conda b/tests/resources/packages/linux-aarch64-pydantic-core-2.26.0-py313h8aa417a_0.conda new file mode 100644 index 0000000..cbd3be2 Binary files /dev/null and b/tests/resources/packages/linux-aarch64-pydantic-core-2.26.0-py313h8aa417a_0.conda differ diff --git a/examples/packages/osx-64-pydantic-core-2.26.0-py313h3c055b9_0.conda b/tests/resources/packages/osx-64-pydantic-core-2.26.0-py313h3c055b9_0.conda similarity index 100% rename from examples/packages/osx-64-pydantic-core-2.26.0-py313h3c055b9_0.conda rename to tests/resources/packages/osx-64-pydantic-core-2.26.0-py313h3c055b9_0.conda diff --git a/examples/packages/osx-arm64-pydantic-core-2.26.0-py313hdde674f_0.conda b/tests/resources/packages/osx-arm64-pydantic-core-2.26.0-py313hdde674f_0.conda similarity index 100% rename from examples/packages/osx-arm64-pydantic-core-2.26.0-py313hdde674f_0.conda rename to tests/resources/packages/osx-arm64-pydantic-core-2.26.0-py313hdde674f_0.conda diff --git a/examples/packages/win-64-pydantic-core-2.26.0-py313hf3b5b86_0.conda b/tests/resources/packages/win-64-pydantic-core-2.26.0-py313hf3b5b86_0.conda similarity index 100% rename from examples/packages/win-64-pydantic-core-2.26.0-py313hf3b5b86_0.conda rename to tests/resources/packages/win-64-pydantic-core-2.26.0-py313hf3b5b86_0.conda diff --git a/examples/simple-example/.gitignore b/tests/resources/simple-example/.gitignore similarity index 100% rename from examples/simple-example/.gitignore rename to tests/resources/simple-example/.gitignore diff --git a/examples/simple-example/pixi.lock b/tests/resources/simple-example/pixi.lock similarity index 100% rename from examples/simple-example/pixi.lock rename to tests/resources/simple-example/pixi.lock diff --git a/examples/simple-example/pixi.toml b/tests/resources/simple-example/pixi.toml similarity index 100% rename from examples/simple-example/pixi.toml rename to tests/resources/simple-example/pixi.toml