diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 8cf5468cf..8aab4dff9 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -61,26 +61,27 @@ jobs: fail-fast: false matrix: include: - - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-latest } - - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest, skip-tests: true } - - { name: "Linux-arm", target: arm-unknown-linux-musleabi, os: ubuntu-latest, use-cross: true, skip-tests: true } +# - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-latest } +# - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest, skip-tests: true } +# - { name: "Linux-arm", target: arm-unknown-linux-musleabi, os: ubuntu-latest, use-cross: true, skip-tests: true } - # - { name: "Linux-mips", target: mips-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } - # - { name: "Linux-mipsel", target: mipsel-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } - # - { name: "Linux-mips64", target: mips64-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } - # - { name: "Linux-mips64el", target: mips64el-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } +# # - { name: "Linux-mips", target: mips-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } +# # - { name: "Linux-mipsel", target: mipsel-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } +# # - { name: "Linux-mips64", target: mips64-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } +# # - { name: "Linux-mips64el", target: mips64el-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } -# - { name: "Linux-powerpc", target: powerpc-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { name: "Linux-powerpc64", target: powerpc64-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { name: "Linux-powerpc64le", target: powerpc64le-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } +# # - { name: "Linux-powerpc", target: powerpc-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } +# - { name: "Linux-powerpc64", target: powerpc64-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } +# - { name: "Linux-powerpc64le", target: powerpc64le-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { name: "Linux-s390x", target: s390x-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } +# - { name: "Linux-s390x", target: s390x-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } - - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest, skip-tests: true } +# - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } +# - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest, skip-tests: true } - - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } - - { name: "Windows-aarch64", target: aarch64-pc-windows-msvc, os: windows-latest, skip-tests: true } + # - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } + # - { name: "Windows-aarch64", target: aarch64-pc-windows-msvc, os: windows-latest, skip-tests: true } + - { name: "Webassembly", target: wasm32-unknown-unknown, os: ubuntu-latest, skip-tests: true } steps: - name: Checkout source code uses: actions/checkout@v4 @@ -96,7 +97,7 @@ jobs: cache: false - uses: taiki-e/setup-cross-toolchain-action@v1 - if: matrix.target != 'x86_64-unknown-linux-musl' + if: ${{ !(matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'wasm32-unknown-unknown') }} with: target: ${{ matrix.target }} @@ -123,6 +124,7 @@ jobs: echo "CARGO_BUILD_OPTIONS=${CARGO_BUILD_OPTIONS} --no-default-features --features rustls-tls" >> $GITHUB_OUTPUT - name: Build + if: matrix.target != 'wasm32-unknown-unknown' run: > cargo build --all-targets @@ -130,6 +132,15 @@ jobs: --target ${{ matrix.target }} ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} + - name: Build + if: matrix.target == 'wasm32-unknown-unknown' + run: | + cd crates/rattler_package_streaming + cargo build \ + --no-default-features \ + --features wasm \ + --target ${{ matrix.target }} + - name: Disable testing the tools crate if cross compiling id: test-options if: ${{ !matrix.skip-tests }} diff --git a/crates/rattler/Cargo.toml b/crates/rattler/Cargo.toml index 5abd5597b..2bf8a8a22 100644 --- a/crates/rattler/Cargo.toml +++ b/crates/rattler/Cargo.toml @@ -39,7 +39,7 @@ pin-project-lite = { workspace = true } rattler_conda_types = { path="../rattler_conda_types", version = "0.19.0", default-features = false } rattler_digest = { path="../rattler_digest", version = "0.19.0", default-features = false } rattler_networking = { path="../rattler_networking", version = "0.19.0", default-features = false } -rattler_package_streaming = { path="../rattler_package_streaming", version = "0.19.0", default-features = false, features = ["reqwest"] } +rattler_package_streaming = { path="../rattler_package_streaming", version = "0.19.0", default-features = false, features = ["reqwest", "zstdmt"] } reflink-copy = { workspace = true } regex = { workspace = true } reqwest = { workspace = true, features = ["stream", "json", "gzip"] } diff --git a/crates/rattler_conda_types/Cargo.toml b/crates/rattler_conda_types/Cargo.toml index 60b869832..1f23964ac 100644 --- a/crates/rattler_conda_types/Cargo.toml +++ b/crates/rattler_conda_types/Cargo.toml @@ -37,7 +37,7 @@ url = { workspace = true, features = ["serde"] } [dev-dependencies] rand = { workspace = true } insta = { workspace = true, features = ["yaml", "redactions", "toml"] } -rattler_package_streaming = { path = "../rattler_package_streaming", default-features = false, features = ["rustls-tls"] } +rattler_package_streaming = { path = "../rattler_package_streaming", default-features = false, features = ["rustls-tls", "zstdmt"] } tempfile = { workspace = true } rstest = { workspace = true } assert_matches = { workspace = true } diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index 08fb14d49..7481a274d 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -14,7 +14,7 @@ readme.workspace = true fs-err = { workspace = true } rattler_conda_types = { path="../rattler_conda_types", version = "0.19.0", default-features = false } rattler_digest = { path="../rattler_digest", version = "0.19.0", default-features = false } -rattler_package_streaming = { path="../rattler_package_streaming", version = "0.19.0", default-features = false } +rattler_package_streaming = { path="../rattler_package_streaming", version = "0.19.0", default-features = false, features = ["zstdmt"] } serde_json = { workspace = true } tracing = { workspace = true } walkdir = { workspace = true } diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index ebc8d3b90..e7c32eb87 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -11,15 +11,17 @@ license.workspace = true readme.workspace = true [features] +default = ["fslock"] native-tls = ['reqwest/native-tls'] rustls-tls = ['reqwest/rustls-tls'] +wasm = [] [dependencies] anyhow = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } dirs = { workspace = true } -fslock = { workspace = true } +fslock = { workspace = true, optional = true } itertools = { workspace = true } keyring = { workspace = true } lazy_static = { workspace = true } diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index a60514c19..d56d675c7 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -16,7 +16,8 @@ pub struct AuthenticationMiddleware { auth_storage: AuthenticationStorage, } -#[async_trait] +#[cfg_attr(not(feature = "wasm"), async_trait)] +#[cfg_attr(feature = "wasm", async_trait(?Send))] impl Middleware for AuthenticationMiddleware { async fn handle( &self, diff --git a/crates/rattler_networking/src/authentication_storage/backends/file.rs b/crates/rattler_networking/src/authentication_storage/backends/file.rs index ce640f3f1..9f8a299bf 100644 --- a/crates/rattler_networking/src/authentication_storage/backends/file.rs +++ b/crates/rattler_networking/src/authentication_storage/backends/file.rs @@ -1,5 +1,6 @@ //! file storage for passwords. use anyhow::Result; +#[cfg(feature = "fslock")] use fslock::LockFile; use once_cell::sync::Lazy; use std::collections::{BTreeMap, HashSet}; @@ -38,6 +39,7 @@ impl FileStorage { Self { path } } + #[cfg(feature = "fslock")] /// Lock the file storage file for reading and writing. This will block until the lock is /// acquired. fn lock(&self) -> Result { @@ -60,6 +62,12 @@ impl FileStorage { Ok(lock) } + #[cfg(not(feature = "fslock"))] + /// Fake lock + fn lock(&self) -> Result<(), FileStorageError> { + Ok(()) + } + /// Read the JSON file and deserialize it into a `BTreeMap`, or return an empty `BTreeMap` if the /// file does not exist fn read_json(&self) -> Result, FileStorageError> { diff --git a/crates/rattler_package_streaming/Cargo.toml b/crates/rattler_package_streaming/Cargo.toml index 6573af264..9038df179 100644 --- a/crates/rattler_package_streaming/Cargo.toml +++ b/crates/rattler_package_streaming/Cargo.toml @@ -25,17 +25,18 @@ serde_json = { workspace = true } tar = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["fs"] } +tokio = { workspace = true, default-features = false } tokio-util = { workspace = true, features = ["io-util"] } url = { workspace = true } zip = { workspace = true, features = ["deflate", "time"] } -zstd = { workspace = true, features = ["zstdmt"] } +zstd = { workspace = true } [features] -default = ["native-tls"] +default = ["native-tls", "tokio/fs", "zstdmt"] +zstdmt = ["zstd/zstdmt"] native-tls = ["rattler_networking/native-tls"] rustls-tls = ["rattler_networking/rustls-tls"] -wasm = ["zstd/wasm"] +wasm = ["zstd/wasm", "rattler_networking/wasm"] reqwest = ["dep:reqwest-middleware", "dep:reqwest"] [dev-dependencies] diff --git a/crates/rattler_package_streaming/src/write.rs b/crates/rattler_package_streaming/src/write.rs index 6e5f62b01..9fcca00ec 100644 --- a/crates/rattler_package_streaming/src/write.rs +++ b/crates/rattler_package_streaming/src/write.rs @@ -239,7 +239,11 @@ fn write_zst_archive( let tar_file = File::open(&tar_path)?; let compression_level = compression_level.to_zstd_level()?; let mut zst_encoder = zstd::Encoder::new(writer, compression_level)?; + #[cfg(feature = "zstdmt")] zst_encoder.multithread(num_threads.unwrap_or_else(|| num_cpus::get() as u32))?; + // mark as "used" to avoid "unused" warning + #[cfg(not(feature = "zstdmt"))] + let _ = num_threads; progress_bar_wrapper.reset_position(); if let Ok(tar_total_size) = tar_file.metadata().map(|v| v.len()) {