feat: Use opendal in rattler-index and add executable

.gitattributes

@@ -0,0 +1,2 @@
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true
+Cargo.lock merge=binary linguist-generated=true

.github/workflows/python-bindings.yml

@@ -35,6 +35,7 @@ jobs:
       - uses: prefix-dev/setup-pixi@v0.8.3
         with:
           manifest-path: py-rattler/pixi.toml
+          environments: test
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           components: clippy, rustfmt

Cargo.toml

@@ -52,6 +52,7 @@ chrono = { version = "0.4.39", default-features = false, features = [
     "alloc",
 ] }
 clap = { version = "4.5.29", features = ["derive"] }
+clap-verbosity-flag = "3.0.1"
 cmake = "0.1.54"
 console = { version = "0.15.10", features = ["windows-console-colors"] }
 criterion = "0.5"
@@ -105,6 +106,7 @@ memmap2 = "0.9.5"
 netrc-rs = "0.1.2"
 nom = "7.1.3"
 num_cpus = "1.16.0"
+opendal = { version = "0.51.2", default-features = false }
 once_cell = "1.20.3"
 ouroboros = "0.18.5"
 parking_lot = "0.12.3"
@@ -161,6 +163,7 @@ tokio-util = "0.7.13"
 tower = { version = "0.5.2", default-features = false }
 tower-http = { version = "0.6.2", default-features = false }
 tracing = "0.1.41"
+tracing-log = "0.2.0"
 tracing-subscriber = { version = "0.3.19", default-features = false }
 tracing-test = { version = "0.2.5" }
 trybuild = { version = "1.0.103" }

crates/rattler_conda_types/Cargo.toml

@@ -24,33 +24,48 @@ itertools = { workspace = true }
 lazy-regex = { workspace = true }
 nom = { workspace = true }
 purl = { workspace = true, features = ["serde"] }
-rattler_digest = { path = "../rattler_digest", version = "1.0.6", default-features = false, features = ["serde"] }
+rattler_digest = { path = "../rattler_digest", version = "1.0.6", default-features = false, features = [
+  "serde",
+] }
 rattler_macros = { path = "../rattler_macros", version = "1.0.6", default-features = false }
+rattler_redaction = { version = "0.1.6", path = "../rattler_redaction", default-features = false }
 regex = { workspace = true }
-simd-json = { workspace = true , features = ["serde_impl"]}
+simd-json = { workspace = true, features = ["serde_impl"] }
 serde = { workspace = true, features = ["derive", "rc"] }
 serde_json = { workspace = true }
 serde_repr = { workspace = true }
 serde_with = { workspace = true, features = ["indexmap_2"] }
 serde-untagged = { workspace = true }
 serde_yaml = { workspace = true }
-smallvec = { workspace = true, features = ["serde", "const_new", "const_generics", "union"] }
+smallvec = { workspace = true, features = [
+  "serde",
+  "const_new",
+  "const_generics",
+  "union",
+] }
 strum = { workspace = true, features = ["derive"] }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 tracing = { workspace = true }
 typed-path = { workspace = true }
 url = { workspace = true, features = ["serde"] }
 indexmap = { workspace = true }
-rattler_redaction = { version = "0.1.6", path = "../rattler_redaction" }
 dirs = { workspace = true }
 rayon = { workspace = true, optional = true }
 fs-err = { workspace = true }
 
 [dev-dependencies]
 rand = { workspace = true }
-insta = { workspace = true, features = ["yaml", "redactions", "toml", "glob", "filters"] }
-rattler_package_streaming = { path = "../rattler_package_streaming", default-features = false, features = ["rustls-tls"] }
+insta = { workspace = true, features = [
+  "yaml",
+  "redactions",
+  "toml",
+  "glob",
+  "filters",
+] }
+rattler_package_streaming = { path = "../rattler_package_streaming", default-features = false, features = [
+  "rustls-tls",
+] }
 rstest = { workspace = true }
 assert_matches = { workspace = true }
 hex-literal = { workspace = true }

crates/rattler_index/Cargo.toml

@@ -3,23 +3,64 @@ name = "rattler_index"
 version = "0.20.13"
 edition.workspace = true
 authors = []
-description = "A crate that indexes directories containing conda packages to create local conda channels"
+description = "A crate to index conda channels and create a repodata.json file."
 categories.workspace = true
 homepage.workspace = true
 repository.workspace = true
 license.workspace = true
 readme.workspace = true
+default-run = "rattler-index"
+
+[features]
+default = ["rustls-tls"]
+native-tls = [
+  "reqwest/native-tls",
+  "reqwest/native-tls-alpn",
+  "rattler_package_streaming/native-tls",
+  "rattler_networking/native-tls",
+  "tools/native-tls",
+]
+rustls-tls = [
+  "reqwest/rustls-tls",
+  "reqwest/rustls-tls-native-roots",
+  "rattler_package_streaming/rustls-tls",
+  "rattler_networking/rustls-tls",
+  "tools/rustls-tls",
+]
+
+[[bin]]
+name = "rattler-index"
+path = "src/main.rs"
 
 [dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true, features = ["derive", "env"] }
+clap-verbosity-flag = { workspace = true }
+console = { workspace = true }
 fs-err = { workspace = true }
-rattler_conda_types = { path="../rattler_conda_types", version = "0.31.0", default-features = false }
-rattler_digest = { path="../rattler_digest", version = "1.0.6", default-features = false }
-rattler_package_streaming = { path="../rattler_package_streaming", version = "0.22.29", default-features = false }
+fxhash = { workspace = true }
+futures = { workspace = true }
+indicatif = { workspace = true }
+opendal = { workspace = true, features = [
+  "services-s3",
+  "services-fs",
+], default-features = false }
+rattler_networking = { path = "../rattler_networking", version = "0.22.4", default-features = false }
+rattler_conda_types = { path = "../rattler_conda_types", version = "0.31.0" }
+rattler_digest = { path = "../rattler_digest", version = "1.0.6", default-features = false }
+rattler_package_streaming = { path = "../rattler_package_streaming", version = "0.22.29", default-features = false }
+reqwest = { workspace = true, default-features = false, features = [
+  "http2",
+  "macos-system-configuration",
+  "charset",
+] }
 serde_json = { workspace = true }
-tempfile = { workspace = true }
+tokio = { workspace = true, features = ["full"] }
 tracing = { workspace = true }
-walkdir = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] }
+tracing-log = { workspace = true }
+url = { workspace = true }
 
 [dev-dependencies]
 tempfile = { workspace = true }
-tools = { path = "../tools" }
+tools = { path = "../tools", default-features = false }