diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3fa37b7..1a882bd 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,3 +5,9 @@ # Changes + +--- + +If updating documentation: + +- [ ] Updated documentation in https://github.com/prefix-dev/pixi/blob/main/docs/advanced/production_deployment.md as well diff --git a/Cargo.lock b/Cargo.lock index e1520a0..aeb8330 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" [[package]] name = "arbitrary" @@ -535,9 +535,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "92b7b18d71fad5313a1e320fa9897994228ce274b60faa4d694fe0ea89cd9e6d" dependencies = [ "clap_builder", "clap_derive", @@ -551,13 +551,14 @@ checksum = "2678fade3b77aa3a8ff3aae87e9c008d3fb00473a41c71fbf74e91c8c7b37e84" dependencies = [ "clap", "log", + "tracing-core", ] [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "a35db2071778a7344791a4fb4f95308b5673d219dee3ae348b86642574ecc90c" dependencies = [ "anstream", "anstyle", @@ -567,9 +568,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" dependencies = [ "heck", "proc-macro2", @@ -1184,10 +1185,22 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", +] + [[package]] name = "gimli" version = "0.31.1" @@ -1668,9 +1681,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.9" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" dependencies = [ "console", "number_prefix", @@ -1691,13 +1704,14 @@ dependencies = [ [[package]] name = "insta" -version = "1.41.1" +version = "1.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9ffc4d4892617c50a928c52b2961cb5174b6fc6ebf252b2fac9d21955c48b8" +checksum = "71c1b125e30d93896b365e156c33dadfffab45ee8400afcbba4752f59de08a86" dependencies = [ "console", - "lazy_static", "linked-hash-map", + "once_cell", + "pin-project", "similar", ] @@ -1817,9 +1831,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libdbus-sys" @@ -1956,7 +1970,7 @@ checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ "hermit-abi 0.3.9", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -2128,9 +2142,9 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl" -version = "0.10.68" +version = "0.10.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -2160,9 +2174,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.104" +version = "0.9.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" dependencies = [ "cc", "libc", @@ -2324,6 +2338,26 @@ dependencies = [ "unicase", ] +[[package]] +name = "pin-project" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.15" @@ -2349,7 +2383,7 @@ dependencies = [ [[package]] name = "pixi-pack" -version = "0.3.1" +version = "0.3.3" dependencies = [ "anyhow", "async-std", @@ -2380,7 +2414,6 @@ dependencies = [ "tokio-stream", "tokio-tar", "tracing", - "tracing-log", "tracing-subscriber", "url", "walkdir", @@ -2486,7 +2519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", - "getrandom", + "getrandom 0.2.15", "rand", "ring", "rustc-hash", @@ -2549,14 +2582,14 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] name = "rattler" -version = "0.28.4" +version = "0.28.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360e9b22e6e3a43e0f801050aa7ad457861fa34df1914132c53266ca5a4ab7c2" +checksum = "fdd552c29726ad42d6023060dc3f42c6bad97af00b479b4c1fb77d3c3e1f6db4" dependencies = [ "anyhow", "digest", @@ -2592,9 +2625,9 @@ dependencies = [ [[package]] name = "rattler_cache" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d884a3183c7f986b158260394824a7ddcd3ca64e4664e58ce45ea3322b4d46d" +checksum = "2523385739abca920f8b9ac5c292b0835f003ee4fa3d5da89b44fa2e4be68961" dependencies = [ "anyhow", "dashmap", @@ -2673,9 +2706,9 @@ dependencies = [ [[package]] name = "rattler_index" -version = "0.19.37" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba9d13d0cb24a6755778c4652078b6f3926d3917c23c4bcf094fd4b6471a6b" +checksum = "188b27919eaf192845768c671515253bfc12ccc46151158b998047736e8b1542" dependencies = [ "fs-err 3.0.0", "rattler_conda_types", @@ -2733,7 +2766,7 @@ dependencies = [ "chrono", "dirs", "fslock", - "getrandom", + "getrandom 0.2.15", "google-cloud-auth", "http", "itertools", @@ -2831,7 +2864,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", "thiserror 1.0.68", ] @@ -3002,7 +3035,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", @@ -3228,9 +3261,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" dependencies = [ "serde_derive", ] @@ -3258,9 +3291,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" dependencies = [ "proc-macro2", "quote", @@ -3269,9 +3302,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.134" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" dependencies = [ "indexmap 2.6.0", "itoa", @@ -3404,7 +3437,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1df0290e9bfe79ddd5ff8798ca887cd107b75353d2957efe9777296e17f26b5" dependencies = [ - "getrandom", + "getrandom 0.2.15", "halfbrown", "ref-cast", "serde", @@ -3608,12 +3641,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.14.0" +version = "3.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "22e5a0acb1f3f55f65cc4a866c361b2fb2a0ff6366785ae6fbb5f85df07ba230" dependencies = [ "cfg-if", "fastrand", + "getrandom 0.3.1", "once_cell", "rustix", "windows-sys 0.59.0", @@ -3727,9 +3761,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -3744,9 +3778,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", @@ -4037,7 +4071,7 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ - "getrandom", + "getrandom 0.2.15", "rand", ] @@ -4111,6 +4145,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.95" @@ -4492,6 +4535,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index bbc62f3..c9b67b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pixi-pack" description = "A command line tool to pack and unpack conda environments for easy sharing" -version = "0.3.1" +version = "0.3.3" edition = "2021" [features] @@ -22,15 +22,15 @@ rustls-tls = [ ] [dependencies] -anyhow = "1.0.95" -clap = { version = "4.5.23", features = ["derive", "string"] } -clap-verbosity-flag = "3.0.2" +anyhow = "1.0.96" +clap = { version = "4.5.30", features = ["derive", "string"] } +clap-verbosity-flag = { version = "3.0.2", features = ["tracing"] } futures = "0.3.31" -indicatif = "0.17.9" -rattler = { version = "0.28.4", default-features = false } +indicatif = "0.17.11" +rattler = { version = "0.28.7", default-features = false } rattler_digest = "1.0.4" rattler_conda_types = "0.29.5" -rattler_index = "0.19.37" +rattler_index = "0.20.2" rattler_lock = "0.22.34" rattler_networking = { version = "0.21.8", default-features = false } rattler_package_streaming = { version = "0.22.18", default-features = false } @@ -40,21 +40,20 @@ reqwest = { version = "0.12.12", default-features = false, features = [ "macos-system-configuration", ] } reqwest-middleware = "0.4.0" -serde = { version = "1.0.215", features = ["derive"] } -serde_json = "1.0.134" +serde = { version = "1.0.218", features = ["derive"] } +serde_json = "1.0.139" serde_yaml = "0.9.34" tokio-tar = "0.3.1" -tokio = { version = "1.42.0", features = ["rt-multi-thread"] } +tokio = { version = "1.43.0", features = ["rt-multi-thread"] } tokio-stream = { version = "0.1.17", features = ["fs"] } tracing = "0.1.41" tracing-subscriber = { version = "0.3.19", features = [ "default", "env-filter", ] } -tracing-log = "0.2.0" url = "2.5.4" fxhash = "0.2.1" -tempfile = "3.14.0" +tempfile = "3.17.1" walkdir = "2.5.0" base64 = "0.22.1" @@ -62,4 +61,4 @@ base64 = "0.22.1" async-std = "1.13.0" rstest = "0.24.0" sha2 = "0.10.8" -insta = "1.41.1" +insta = "1.42.1" diff --git a/README.md b/README.md index 010b249..4ba0810 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,23 @@ pixi-pack pack --inject local-package-1.0.0-hbefa133_0.conda --manifest-pack pix This can be particularly useful if you build the project itself and want to include the built package in the environment but still want to use `pixi.lock` from the project. Before creating the pack, `pixi-pack` will ensure that the injected packages' dependencies and constraints are compatible with the packages in the environment. +### Cache downloaded packages + +You can cache downloaded packages to speed up subsequent pack operations by using the `--use-cache` flag: + +```bash +pixi-pack pack --use-cache ~/.pixi-pack/cache +``` + +This will store all downloaded packages in the specified directory and reuse them in future pack operations. The cache follows the same structure as conda channels, organizing packages by platform subdirectories (e.g., linux-64, win-64, etc.). + +Using a cache is particularly useful when: + +- Creating multiple packs with overlapping dependencies +- Working with large packages that take time to download +- Operating in environments with limited bandwidth +- Running CI/CD pipelines where package caching can significantly improve build times + ### Unpacking without `pixi-pack` If you don't have `pixi-pack` available on your target system, you can still install the environment if you have `conda` or `micromamba` available. @@ -181,3 +198,11 @@ conda env create -p ./env --file environment.yml > [!NOTE] > The `environment.yml` and `repodata.json` files are only for this use case, `pixi-pack unpack` does not use them. + +> [!NOTE] +> Both `conda` and `mamba` are always installing pip as a side effect when they install python, see [`conda`'s documentation](https://docs.conda.io/projects/conda/en/25.1.x/user-guide/configuration/settings.html#add-pip-as-python-dependency-add-pip-as-python-dependency). +> This is not different from how `pixi` works and can lead to solver errors when using `pixi-pack`'s compatibility mode since `pixi-pack` doesn't include `pip` by default. +> You can fix this issue in two ways: +> +> - Add `pip` to your `pixi.lock` file using `pixi add pip`. +> - Configuring `conda` (or `mamba`) to not install `pip` by default by running `conda config --set add_pip_as_python_dependency false` (or by adding `add_pip_as_python_dependency: False` to your `~/.condarc`) diff --git a/src/main.rs b/src/main.rs index 2dcb5d4..fc54c96 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,6 @@ use pixi_pack::{ PIXI_PACK_VERSION, }; use rattler_shell::shell::ShellEnum; -use tracing_log::AsTrace; /* -------------------------------------------- CLI -------------------------------------------- */ @@ -46,14 +45,18 @@ enum Commands { #[arg(long)] // TODO: Read from environment variable? auth_file: Option, - /// The path to 'pixi.toml' or 'pyproject.toml' - #[arg(default_value = cwd().join("pixi.toml").into_os_string())] + /// The path to `pixi.toml`, `pyproject.toml`, or the project directory + #[arg(default_value = cwd().into_os_string())] manifest_path: PathBuf, /// Output file to write the pack to (will be an archive) #[arg(short, long)] output_file: Option, + /// Use a cache directory for downloaded packages + #[arg(long)] + use_cache: Option, + /// Inject an additional conda package into the final prefix #[arg(short, long, num_args(0..))] inject: Vec, @@ -67,7 +70,6 @@ enum Commands { #[arg(long, default_value = "false")] create_executable: bool, }, - /// Unpack a pixi environment Unpack { /// Where to unpack the environment. @@ -111,7 +113,7 @@ async fn main() -> Result<()> { let cli = Cli::parse(); tracing_subscriber::FmtSubscriber::builder() - .with_max_level(cli.verbose.log_level_filter().as_trace()) + .with_max_level(cli.verbose) .init(); tracing::debug!("Starting pixi-pack CLI"); @@ -126,6 +128,7 @@ async fn main() -> Result<()> { inject, ignore_pypi_errors, create_executable, + use_cache, } => { let output_file = output_file.unwrap_or_else(|| default_output_file(platform, create_executable)); @@ -144,6 +147,7 @@ async fn main() -> Result<()> { injected_packages: inject, ignore_pypi_errors, create_executable, + cache_dir: use_cache, }; tracing::debug!("Running pack command with options: {:?}", options); pack(options).await? diff --git a/src/pack.rs b/src/pack.rs index 31433e6..a14ed88 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -39,26 +39,41 @@ pub struct PackOptions { pub output_file: PathBuf, pub manifest_path: PathBuf, pub metadata: PixiPackMetadata, + pub cache_dir: Option, pub injected_packages: Vec, pub ignore_pypi_errors: bool, pub create_executable: bool, } +fn load_lockfile(manifest_path: &Path) -> Result { + if !manifest_path.exists() { + anyhow::bail!( + "manifest path does not exist at {}", + manifest_path.display() + ); + } -/// Pack a pixi environment. -pub async fn pack(options: PackOptions) -> Result<()> { - let lockfile_path = options - .manifest_path - .parent() - .ok_or(anyhow!("could not get parent directory"))? - .join("pixi.lock"); + let manifest_path = if !manifest_path.is_dir() { + manifest_path + .parent() + .ok_or(anyhow!("could not get parent directory"))? + } else { + manifest_path + }; + + let lockfile_path = manifest_path.join("pixi.lock"); - let lockfile = LockFile::from_path(&lockfile_path).map_err(|e| { + LockFile::from_path(&lockfile_path).map_err(|e| { anyhow!( "could not read lockfile at {}: {}", lockfile_path.display(), e ) - })?; + }) +} + +/// Pack a pixi environment. +pub async fn pack(options: PackOptions) -> Result<()> { + let lockfile = load_lockfile(&options.manifest_path)?; let client = reqwest_client_from_auth_storage(options.auth_file) .map_err(|e| anyhow!("could not create reqwest client from auth storage: {e}"))?; @@ -113,7 +128,7 @@ pub async fn pack(options: PackOptions) -> Result<()> { stream::iter(conda_packages_from_lockfile.iter()) .map(Ok) .try_for_each_concurrent(50, |package| async { - download_package(&client, package, &channel_dir).await?; + download_package(&client, package, &channel_dir, options.cache_dir.as_deref()).await?; bar.pb.inc(1); Ok(()) }) @@ -239,6 +254,7 @@ async fn download_package( client: &ClientWithMiddleware, package: &CondaBinaryData, output_dir: &Path, + cache_dir: Option<&Path>, ) -> Result<()> { let output_dir = output_dir.join(&package.package_record.subdir); create_dir_all(&output_dir) @@ -246,7 +262,21 @@ async fn download_package( .map_err(|e| anyhow!("could not create download directory: {}", e))?; let file_name = &package.file_name; - let mut dest = File::create(output_dir.join(file_name)).await?; + let output_path = output_dir.join(file_name); + + // Check cache first if enabled + if let Some(cache_dir) = cache_dir { + let cache_path = cache_dir + .join(&package.package_record.subdir) + .join(file_name); + if cache_path.exists() { + tracing::debug!("Using cached package from {}", cache_path.display()); + fs::copy(&cache_path, &output_path).await?; + return Ok(()); + } + } + + let mut dest = File::create(&output_path).await?; tracing::debug!("Fetching package {}", package.location); let url = match &package.location { @@ -266,9 +296,16 @@ async fn download_package( dest.write_all(&chunk).await?; } + // Save to cache if enabled + if let Some(cache_dir) = cache_dir { + let cache_subdir = cache_dir.join(&package.package_record.subdir); + create_dir_all(&cache_subdir).await?; + let cache_path = cache_subdir.join(file_name); + fs::copy(&output_path, &cache_path).await?; + } + Ok(()) } - async fn archive_directory( input_dir: &Path, archive_target: &Path, diff --git a/src/unpack.rs b/src/unpack.rs index 572db7c..80c131b 100644 --- a/src/unpack.rs +++ b/src/unpack.rs @@ -195,8 +195,22 @@ async fn create_prefix(channel_dir: &Path, target_prefix: &Path, cache_dir: &Pat let cache_key = CacheKey::from(&package_record); let package_path = channel_dir.join(&package_record.subdir).join(&file_name); + let normalized_path = package_path.canonicalize().unwrap(); - let url = Url::parse(&format!("file:///{}", file_name)).unwrap(); + let url = Url::from_file_path(&normalized_path) + .map_err(|_| { + anyhow!( + "could not convert path to URL: {}", + normalized_path.display() + ) + }) + .unwrap(); + + tracing::debug!( + "Extracting package {} with URL {}", + package_record.name.as_normalized(), + url + ); let repodata_record = RepoDataRecord { package_record, diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 54f9010..3971b13 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,8 +1,10 @@ #![allow(clippy::too_many_arguments)] use sha2::{Digest, Sha256}; +use std::collections::HashMap; use std::{fs, io}; use std::{path::PathBuf, process::Command}; +use walkdir::WalkDir; use pixi_pack::{ unarchive, PackOptions, PixiPackMetadata, UnpackOptions, DEFAULT_PIXI_PACK_VERSION, @@ -61,6 +63,7 @@ fn options( injected_packages: vec![], ignore_pypi_errors, create_executable, + cache_dir: None, }, unpack_options: UnpackOptions { pack_file, @@ -71,7 +74,6 @@ fn options( output_dir, } } - #[fixture] fn required_fs_objects() -> Vec<&'static str> { let mut required_fs_objects = vec!["conda-meta/history", "include", "share"]; @@ -557,3 +559,86 @@ async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<& // Keep the temporary directory alive until the end of the test drop(temp_dir); } + +#[rstest] +#[tokio::test] +async fn test_manifest_path_dir(#[with(PathBuf::from("examples/simple-python"))] options: Options) { + let pack_options = options.pack_options; + let unpack_options = options.unpack_options; + let pack_file = unpack_options.pack_file.clone(); + + let pack_result = pixi_pack::pack(pack_options).await; + assert!(pack_result.is_ok(), "{:?}", pack_result); + assert!(pack_file.is_file()); +} +#[rstest] +#[tokio::test] +async fn test_package_caching( + #[with(PathBuf::from("examples/simple-python/pixi.toml"))] options: Options, +) { + let temp_cache = tempdir().expect("Couldn't create a temp cache dir"); + let cache_dir = temp_cache.path().to_path_buf(); + + // First pack with cache - should download packages + let mut pack_options = options.pack_options.clone(); + pack_options.cache_dir = Some(cache_dir.clone()); + let pack_result = pixi_pack::pack(pack_options).await; + assert!(pack_result.is_ok(), "{:?}", pack_result); + + // Get files and their modification times after first pack + let mut initial_cache_files = HashMap::new(); + for entry in WalkDir::new(&cache_dir) { + let entry = entry.unwrap(); + if entry.file_type().is_file() { + let path = entry.path().to_path_buf(); + let modified_time = fs::metadata(&path).unwrap().modified().unwrap(); + initial_cache_files.insert(path, modified_time); + } + } + assert!( + !initial_cache_files.is_empty(), + "Cache should contain downloaded files" + ); + + // Calculate first pack's SHA256, reusing test_reproducible_shasum + let first_sha256 = sha256_digest_bytes(&options.pack_options.output_file); + insta::assert_snapshot!( + format!("sha256-{}", options.pack_options.platform), + &first_sha256 + ); + + // Small delay to ensure any new writes would have different timestamps + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + + // Second pack with same cache - should use cached packages + let temp_dir2 = tempdir().expect("Couldn't create second temp dir"); + let mut pack_options2 = options.pack_options.clone(); + pack_options2.cache_dir = Some(cache_dir.clone()); + let output_file2 = temp_dir2.path().join("environment.tar"); + pack_options2.output_file = output_file2.clone(); + + let pack_result2 = pixi_pack::pack(pack_options2).await; + assert!(pack_result2.is_ok(), "{:?}", pack_result2); + + // Check that cache files weren't modified + for (path, initial_mtime) in initial_cache_files { + let current_mtime = fs::metadata(&path).unwrap().modified().unwrap(); + assert_eq!( + initial_mtime, + current_mtime, + "Cache file {} was modified when it should have been reused", + path.display() + ); + } + + // Verify second pack produces identical output + let second_sha256 = sha256_digest_bytes(&output_file2); + assert_eq!( + first_sha256, second_sha256, + "Pack outputs should be identical when using cache" + ); + + // Both output files should exist and be valid + assert!(options.pack_options.output_file.exists()); + assert!(output_file2.exists()); +} diff --git a/tests/snapshots/integration_test__sha256-linux-64-executable.snap b/tests/snapshots/integration_test__sha256-linux-64-executable.snap index 283b70d..de690fa 100644 --- a/tests/snapshots/integration_test__sha256-linux-64-executable.snap +++ b/tests/snapshots/integration_test__sha256-linux-64-executable.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 360 +assertion_line: 362 expression: "&sha256_digest" -snapshot_kind: text --- -5FDC3D26D96C49EC68F2EB651EBE2FB78C61331B7E1A9C2071C81FEC28AD982B +BF36CCED8B4FCB63EDAD3431BA47B47F9A916CE985B57BF9A56AC98E411F32A1 diff --git a/tests/snapshots/integration_test__sha256-linux-64.snap b/tests/snapshots/integration_test__sha256-linux-64.snap index 1962c22..14f2935 100644 --- a/tests/snapshots/integration_test__sha256-linux-64.snap +++ b/tests/snapshots/integration_test__sha256-linux-64.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 337 +assertion_line: 341 expression: "&sha256_digest" -snapshot_kind: text --- -3DA7D8E917CE10B76AB7D24157BE7AC686C14BD08A7B7604352E406DD4C7199A +435792A0115699D2558AD9FC8418301D3460F318045B6339B3F76AF5F4A9E9D1 diff --git a/tests/snapshots/integration_test__sha256-linux-aarch64-executable.snap b/tests/snapshots/integration_test__sha256-linux-aarch64-executable.snap index 357ae11..2b4ba74 100644 --- a/tests/snapshots/integration_test__sha256-linux-aarch64-executable.snap +++ b/tests/snapshots/integration_test__sha256-linux-aarch64-executable.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 360 +assertion_line: 362 expression: "&sha256_digest" -snapshot_kind: text --- -4716A5C284D649400A157EE028969E183CBA4A1B03EB0C2A3E12DF752518975D +41A62DBF7D7566A8C06B60B6C2C386B5E99E18E4B24884B265B5D6811C130346 diff --git a/tests/snapshots/integration_test__sha256-linux-aarch64.snap b/tests/snapshots/integration_test__sha256-linux-aarch64.snap index 6fef15b..f821da6 100644 --- a/tests/snapshots/integration_test__sha256-linux-aarch64.snap +++ b/tests/snapshots/integration_test__sha256-linux-aarch64.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 337 +assertion_line: 341 expression: "&sha256_digest" -snapshot_kind: text --- -C830AA6DB664106E23D6CB383A97AA06DF2CC7FE918C7507FE2DDB05AE0A7B01 +6DCC430611BBC3172CCFC3D3A28D36BF5DED2E6BA489C7062D4D4B730B232613 diff --git a/tests/snapshots/integration_test__sha256-linux-ppc64le.snap b/tests/snapshots/integration_test__sha256-linux-ppc64le.snap index d2b5eb0..427bb86 100644 --- a/tests/snapshots/integration_test__sha256-linux-ppc64le.snap +++ b/tests/snapshots/integration_test__sha256-linux-ppc64le.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 337 +assertion_line: 341 expression: "&sha256_digest" -snapshot_kind: text --- -42F0E1D7013D3E4EF999004E58A20E30428310056C5EA70CBB3D3BA12D908DFF +C11D7FC5BC136C378680B9C615DE13F3C4E81CA9AD61F57663CCC4B6334050E0 diff --git a/tests/snapshots/integration_test__sha256-osx-64-executable.snap b/tests/snapshots/integration_test__sha256-osx-64-executable.snap index bb47961..feebd2a 100644 --- a/tests/snapshots/integration_test__sha256-osx-64-executable.snap +++ b/tests/snapshots/integration_test__sha256-osx-64-executable.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 360 +assertion_line: 362 expression: "&sha256_digest" -snapshot_kind: text --- -54FB7BD4CB9E9B2E8B1AE3825B4B4BADC1D5CFEAB36700ECA1E95BC71A09FFD1 +330CD104213D7B57B197AFFF92786A0A20B11569CB93429DCCA09E7B9BB9003D diff --git a/tests/snapshots/integration_test__sha256-osx-64.snap b/tests/snapshots/integration_test__sha256-osx-64.snap index 06824af..73d0527 100644 --- a/tests/snapshots/integration_test__sha256-osx-64.snap +++ b/tests/snapshots/integration_test__sha256-osx-64.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 337 +assertion_line: 341 expression: "&sha256_digest" -snapshot_kind: text --- -5125E67F003D8375D23261DBDE5D7509D48269D6173EB8101D63E3E9740DE1B0 +5B560C3F7E62751A95AEAC4A8F36E84EB2A58496EBD8E2603C2399B49345B3B8 diff --git a/tests/snapshots/integration_test__sha256-osx-arm64-executable.snap b/tests/snapshots/integration_test__sha256-osx-arm64-executable.snap index 1f9f4ed..c56a071 100644 --- a/tests/snapshots/integration_test__sha256-osx-arm64-executable.snap +++ b/tests/snapshots/integration_test__sha256-osx-arm64-executable.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 360 +assertion_line: 362 expression: "&sha256_digest" -snapshot_kind: text --- -36294C223CDCBD5F7E8FAF29A60659D1B2F02072CB854FCA05B981B718BFDD74 +33F8E849ED5317B98204DB09C08309A46AB871E512231A62B66AFC8954D5221A diff --git a/tests/snapshots/integration_test__sha256-osx-arm64.snap b/tests/snapshots/integration_test__sha256-osx-arm64.snap index f3c4dcc..7d9a7d0 100644 --- a/tests/snapshots/integration_test__sha256-osx-arm64.snap +++ b/tests/snapshots/integration_test__sha256-osx-arm64.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 337 +assertion_line: 341 expression: "&sha256_digest" -snapshot_kind: text --- -758859377C320DA0B47E71A97513633C8D0B2C45B25D4D83A515F0CC6B48CCAE +9CD8C1C2DFE5C9D693649015E4F6065087DEBC4D66C743CBFD8093EE6ED27098 diff --git a/tests/snapshots/integration_test__sha256-win-64-executable.snap b/tests/snapshots/integration_test__sha256-win-64-executable.snap index 3bdd724..3988cd4 100644 --- a/tests/snapshots/integration_test__sha256-win-64-executable.snap +++ b/tests/snapshots/integration_test__sha256-win-64-executable.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 360 +assertion_line: 362 expression: "&sha256_digest" -snapshot_kind: text --- -8DE1CEF641517F10312C9ABD61356CA55804DF276160FA754A730AB14BCD956A +A954D258D72B2CDA0EAFE5359A21FF95D8F7F4301D6D1EABAF42C2778EB38281 diff --git a/tests/snapshots/integration_test__sha256-win-64.snap b/tests/snapshots/integration_test__sha256-win-64.snap index fba94cd..e86ee0e 100644 --- a/tests/snapshots/integration_test__sha256-win-64.snap +++ b/tests/snapshots/integration_test__sha256-win-64.snap @@ -1,7 +1,6 @@ --- source: tests/integration_test.rs -assertion_line: 337 +assertion_line: 341 expression: "&sha256_digest" -snapshot_kind: text --- -35D90AB1314AD52F6BC78BF8CBECBEC148C8FBF383C1C76E7F576629678506D6 +ABCE996BDB49879EEDF62B2C13FB6AD4CF553196C1267D259D8CF0C4E588E520