Merge remote-tracking branch 'origin/main' into warn-user-on-fork-take-2

Cargo.toml

@@ -64,12 +64,13 @@ memmap = { package = "memmap2", version = "0.7" }
 multiversion = "0.7"
 ndarray = { version = "0.15", default-features = false }
 num-traits = "0.2"
+numpy = "0.22"
 object_store = { version = "0.10", default-features = false }
 once_cell = "1"
 parking_lot = "0.12"
 percent-encoding = "2.3"
 pin-project-lite = "0.2"
-pyo3 = "0.21"
+pyo3 = "0.22"
 rand = "0.8"
 rand_distr = "0.4"
 raw-cpuid = "11"

crates/polars-arrow/Cargo.toml

@@ -55,9 +55,6 @@ zstd = { workspace = true, optional = true }
 # to write to parquet as a stream
 futures = { workspace = true, optional = true }
 
-# to read IPC as a stream
-async-stream = { version = "0.3.2", optional = true }
-
 # avro support
 avro-schema = { workspace = true, optional = true }
 
@@ -75,6 +72,8 @@ arrow-array = { workspace = true, optional = true }
 arrow-buffer = { workspace = true, optional = true }
 arrow-data = { workspace = true, optional = true }
 arrow-schema = { workspace = true, optional = true }
+async-stream = { version = "0.3", optional = true }
+tokio = { workspace = true, optional = true, features = ["io-util"] }
 
 [dev-dependencies]
 criterion = "0.5"
@@ -105,8 +104,6 @@ full = [
   "arrow_rs",
   "io_ipc",
   "io_flight",
-  "io_ipc_write_async",
-  "io_ipc_read_async",
   "io_ipc_compression",
   "io_avro",
   "io_avro_compression",
@@ -119,10 +116,8 @@ full = [
 ]
 arrow_rs = ["arrow-buffer", "arrow-schema", "arrow-data", "arrow-array"]
 io_ipc = ["arrow-format", "polars-error/arrow-format"]
-io_ipc_write_async = ["io_ipc", "futures"]
-io_ipc_read_async = ["io_ipc", "futures", "async-stream"]
 io_ipc_compression = ["lz4", "zstd", "io_ipc"]
-io_flight = ["io_ipc", "arrow-format/flight-data"]
+io_flight = ["io_ipc", "arrow-format/flight-data", "async-stream", "futures", "tokio"]
 
 io_avro = ["avro-schema", "polars-error/avro-schema"]
 io_avro_compression = [

crates/polars-arrow/src/bitmap/immutable.rs

@@ -344,6 +344,7 @@ impl Bitmap {
     /// Unsound iff `i >= self.len()`.
     #[inline]
     pub unsafe fn get_bit_unchecked(&self, i: usize) -> bool {
+        debug_assert!(i < self.len());
         get_bit_unchecked(&self.storage, self.offset + i)
     }
 

crates/polars-arrow/src/bitmap/mutable.rs

@@ -362,6 +362,7 @@ impl MutableBitmap {
     /// Caller must ensure that `index < self.len()`
     #[inline]
     pub unsafe fn set_unchecked(&mut self, index: usize, value: bool) {
+        debug_assert!(index < self.len());
         let byte = self.buffer.get_unchecked_mut(index / 8);
         *byte = set_bit_in_byte(*byte, index % 8, value);
     }

crates/polars-arrow/src/ffi/array.rs

@@ -104,6 +104,7 @@ impl ArrowArray {
             ArrowDataType::BinaryView | ArrowDataType::Utf8View
         );
 
+        #[allow(unused_mut)]
         let (offset, mut buffers, children, dictionary) =
             offset_buffers_children_dictionary(array.as_ref());