From 57f8e2bf5aaf3e52be160d7f90660bbb2b92dd49 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Sun, 28 Jul 2024 12:47:31 +1000 Subject: [PATCH] Bump `zarrs` to 0.16.0 --- CHANGELOG.md | 3 +++ Cargo.lock | 8 +++++--- Cargo.toml | 4 ++-- README.md | 4 ++-- src/bin/zarrs_benchmark_read_async.rs | 4 ++-- src/bin/zarrs_benchmark_read_async_as_sync.rs | 4 ++-- src/bin/zarrs_benchmark_read_sync.rs | 4 ++-- src/bin/zarrs_binary2zarr.rs | 7 +++++-- src/bin/zarrs_ncvar2zarr.rs | 15 +++++++++++---- src/bin/zarrs_ome.rs | 14 +++++++------- src/filter/filters/clamp.rs | 2 +- src/filter/filters/crop.rs | 10 +++++----- src/filter/filters/downsample.rs | 8 ++++---- src/filter/filters/equal.rs | 11 +++++++---- src/filter/filters/gaussian.rs | 14 ++++++++------ src/filter/filters/gradient_magnitude.rs | 14 ++++++++------ src/filter/filters/guided_filter.rs | 12 ++++++------ src/filter/filters/reencode.rs | 10 +++++----- src/filter/filters/replace_value.rs | 2 +- src/filter/filters/rescale.rs | 10 ++++++---- src/filter/filters/summed_area_table.rs | 8 ++++---- src/info/histogram.rs | 5 ++--- src/info/range.rs | 13 ++++++------- src/lib.rs | 10 +++------- 24 files changed, 107 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea339ab..ff85fae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + - Bump `zarrs` to 0.16.0 + ## [0.5.3] - 2024-07-24 ### Added diff --git a/Cargo.lock b/Cargo.lock index 82b452e..6e0b70c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1254,6 +1254,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ + "bytemuck", "num-traits", ] @@ -2649,9 +2650,9 @@ dependencies = [ [[package]] name = "zarrs" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff909141506bec11500efe57b0874627d36dc50fc05c96aa57cd841071b15ba6" +checksum = "d3e13209785de81409deb78dce3bf405cb7725fba163b2817c2615ce78e2a273" dependencies = [ "async-lock", "async-recursion", @@ -2670,6 +2671,7 @@ dependencies = [ "monostate", "ndarray", "num", + "num-complex", "object_store", "opendal", "parking_lot", @@ -2686,7 +2688,7 @@ dependencies = [ [[package]] name = "zarrs_tools" -version = "0.5.3" +version = "0.5.4" dependencies = [ "approx", "async-scoped", diff --git a/Cargo.toml b/Cargo.toml index a22e94e..ceedec4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zarrs_tools" -version = "0.5.3" +version = "0.5.4" authors = ["Lachlan Deakin "] edition = "2021" rust-version = "1.75" @@ -48,7 +48,7 @@ sysinfo = "0.30.6" tempfile = "3.10.1" thiserror = "1.0.57" tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread"] } -zarrs = { version = "0.15.0" } +zarrs = { version = "0.16.0" } [[bin]] name = "zarrs_benchmark_read_sync" diff --git a/README.md b/README.md index 3173046..9751cd9 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ Various tools for creating and manipulating [Zarr v3](https://zarr.dev) data wit A changelog can be found [here](https://github.com/LDeakin/zarrs_tools/blob/main/CHANGELOG.md). ## Tools -All tools support input and output [Zarr V3](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html) data. -Some tools additionally support input of a [V3 compatible subset](https://docs.rs/zarrs/latest/zarrs/#arrays-zarr-v3-and-zarr-v2) of [Zarr V2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html). +All tools support input and output of [Zarr V3](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html) data. +Some tools additionally support input of a [V3 compatible subset](https://docs.rs/zarrs/latest/zarrs/#implementation-status) of [Zarr V2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html). - [zarrs_reencode](https://github.com/LDeakin/zarrs_tools/blob/main/docs/zarrs_reencode.md): reencode an array. Manipulate the chunk size, shard size, codecs, fill value, chunk key encoding separator, and attributes. - [zarrs_filter](https://github.com/LDeakin/zarrs_tools/blob/main/docs/zarrs_filter.md) (feature `filter`): apply simple image filters (transformations) to an array. diff --git a/src/bin/zarrs_benchmark_read_async.rs b/src/bin/zarrs_benchmark_read_async.rs index 98fca01..8346ff4 100644 --- a/src/bin/zarrs_benchmark_read_async.rs +++ b/src/bin/zarrs_benchmark_read_async.rs @@ -83,7 +83,7 @@ async fn main() -> Result<(), Box> { let array_shape = array.shape().to_vec(); let array_subset = ArraySubset::new_with_shape(array_shape.to_vec()); let array_data = array.async_retrieve_array_subset(&array_subset).await?; - bytes_decoded += array_data.len(); + bytes_decoded += array_data.size(); } else { // Calculate chunk/codec concurrency let chunk_representation = @@ -120,7 +120,7 @@ async fn main() -> Result<(), Box> { async move { array .async_retrieve_chunk_opt(&chunk_indices, &codec_options) - .map(|bytes| bytes.map(|bytes| bytes.len())) + .map(|bytes| bytes.map(|bytes| bytes.size())) .await } }) diff --git a/src/bin/zarrs_benchmark_read_async_as_sync.rs b/src/bin/zarrs_benchmark_read_async_as_sync.rs index 5e26cba..29171ad 100644 --- a/src/bin/zarrs_benchmark_read_async_as_sync.rs +++ b/src/bin/zarrs_benchmark_read_async_as_sync.rs @@ -93,7 +93,7 @@ fn main() -> Result<(), Box> { let bytes_decoded = Mutex::new(0); if args.read_all { let subset = ArraySubset::new_with_shape(array.shape().to_vec()); - *bytes_decoded.lock().unwrap() += array.retrieve_array_subset(&subset)?.len(); + *bytes_decoded.lock().unwrap() += array.retrieve_array_subset(&subset)?.size(); } else { let chunk_representation = array.chunk_array_representation(&vec![0; array.chunk_grid().dimensionality()])?; @@ -134,7 +134,7 @@ fn main() -> Result<(), Box> { let bytes = array .retrieve_chunk_opt(&chunk_indices, &codec_options) .unwrap(); - *bytes_decoded.lock().unwrap() += bytes.len(); + *bytes_decoded.lock().unwrap() += bytes.size(); } ); } diff --git a/src/bin/zarrs_benchmark_read_sync.rs b/src/bin/zarrs_benchmark_read_sync.rs index 870b60f..f334fbe 100644 --- a/src/bin/zarrs_benchmark_read_sync.rs +++ b/src/bin/zarrs_benchmark_read_sync.rs @@ -62,7 +62,7 @@ fn main() -> Result<(), Box> { let bytes_decoded = Mutex::new(0); if args.read_all { let subset = ArraySubset::new_with_shape(array.shape().to_vec()); - *bytes_decoded.lock().unwrap() += array.retrieve_array_subset(&subset)?.len(); + *bytes_decoded.lock().unwrap() += array.retrieve_array_subset(&subset)?.size(); } else { let chunk_representation = array.chunk_array_representation(&vec![0; array.chunk_grid().dimensionality()])?; @@ -103,7 +103,7 @@ fn main() -> Result<(), Box> { let bytes = array .retrieve_chunk_opt(&chunk_indices, &codec_options) .unwrap(); - *bytes_decoded.lock().unwrap() += bytes.len(); + *bytes_decoded.lock().unwrap() += bytes.size(); } ); } diff --git a/src/bin/zarrs_binary2zarr.rs b/src/bin/zarrs_binary2zarr.rs index f3c9f34..e6925b7 100644 --- a/src/bin/zarrs_binary2zarr.rs +++ b/src/bin/zarrs_binary2zarr.rs @@ -81,7 +81,10 @@ fn stdin_to_array( endianness: Option, concurrent_chunks: Option, ) -> usize { - let data_type_size = array.data_type().size(); + let data_type_size = array + .data_type() + .fixed_size() + .expect("data type should be fixed size"); let dimensionality = array.chunk_grid().dimensionality(); let array_shape = array.shape(); let array_shape_n = *array_shape.first().unwrap(); @@ -159,7 +162,7 @@ fn stdin_to_array( } array - .store_array_subset_opt(&array_subset, &subset_bytes, &codec_options) + .store_array_subset_opt(&array_subset, subset_bytes, &codec_options) .unwrap(); }; iter_concurrent_limit!(concurrent_chunks, 0..n_blocks, for_each, op); diff --git a/src/bin/zarrs_ncvar2zarr.rs b/src/bin/zarrs_ncvar2zarr.rs index 451f6c5..fd9212c 100644 --- a/src/bin/zarrs_ncvar2zarr.rs +++ b/src/bin/zarrs_ncvar2zarr.rs @@ -94,7 +94,14 @@ fn ncfiles_to_array( let mut start = vec![0u64; array.chunk_grid().dimensionality()]; start[concat_dim] = offsets[idx]; let array_subset = ArraySubset::new_with_start_shape(start, dim_sizes_u64.clone()).unwrap(); - let mut buf = vec![0u8; array.data_type().size() * array_subset.num_elements_usize()]; + let mut buf = vec![ + 0u8; + array + .data_type() + .fixed_size() + .expect("data type should be fixed size") + * array_subset.num_elements_usize() + ]; // println!("{array_subset:?} {dim_sizes:?} {}", buf.len()); nc_var .get_raw_values( @@ -110,15 +117,15 @@ fn ncfiles_to_array( if validate { array - .store_array_subset_opt(&array_subset, &buf, &codec_options) + .store_array_subset_opt(&array_subset, buf.clone(), &codec_options) .unwrap(); let buf_validate = array .retrieve_array_subset_opt(&array_subset, &codec_options) .unwrap(); - assert!(buf == buf_validate); + assert!(buf == buf_validate.into_fixed().unwrap().into_owned()); } else { array - .store_array_subset_opt(&array_subset, &buf, &codec_options) + .store_array_subset_opt(&array_subset, buf, &codec_options) .unwrap(); } }; diff --git a/src/bin/zarrs_ome.rs b/src/bin/zarrs_ome.rs index 3d40f1a..83b7602 100644 --- a/src/bin/zarrs_ome.rs +++ b/src/bin/zarrs_ome.rs @@ -16,7 +16,7 @@ use ome_zarr_metadata::v0_5_dev::{ }; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use zarrs::{ - array::{Array, ArrayCodecTraits, ArrayMetadata, ChunkRepresentation}, + array::{Array, ArrayCodecTraits, ArrayMetadata, ChunkRepresentation, Element, ElementOwned}, array_subset::ArraySubset, group::{Group, GroupMetadata, GroupMetadataV3}, storage::{store::FilesystemStore, StorePrefix, WritableStorageTraits}, @@ -180,7 +180,7 @@ fn apply_chunk_discrete( progress: &Progress, ) -> Result<(), FilterError> where - T: bytemuck::Pod + Copy + Send + Sync + Eq + PartialEq + Hash + AsPrimitive, + T: Element + ElementOwned + Copy + Send + Sync + Eq + PartialEq + Hash + AsPrimitive, { let output_subset = array_output.chunk_subset_bounded(chunk_indices).unwrap(); let downsample_input_subset = @@ -191,7 +191,7 @@ where downsample_filter.apply_ndarray_discrete(input_chunk, progress) }; progress.write(|| { - array_output.store_array_subset_ndarray::(output_subset.start(), output_chunk) + array_output.store_array_subset_ndarray::(output_subset.start(), output_chunk) })?; Ok(()) } @@ -204,7 +204,7 @@ fn apply_chunk_continuous( progress: &Progress, ) -> Result<(), FilterError> where - T: bytemuck::Pod + Copy + Send + Sync + AsPrimitive + std::iter::Sum, + T: Element + ElementOwned + Copy + Send + Sync + AsPrimitive + std::iter::Sum, f64: AsPrimitive, { let output_subset = array_output.chunk_subset_bounded(chunk_indices).unwrap(); @@ -216,7 +216,7 @@ where downsample_filter.apply_ndarray_continuous(input_chunk, progress) }; progress.write(|| { - array_output.store_array_subset_ndarray::(output_subset.start(), output_chunk) + array_output.store_array_subset_ndarray::(output_subset.start(), output_chunk) })?; Ok(()) } @@ -230,7 +230,7 @@ fn apply_chunk_continuous_gaussian( progress: &Progress, ) -> Result<(), FilterError> where - T: bytemuck::Pod + Copy + Send + Sync + AsPrimitive + std::iter::Sum, + T: Element + ElementOwned + Copy + Send + Sync + AsPrimitive + std::iter::Sum, f64: AsPrimitive, { let output_subset = array_output.chunk_subset_bounded(chunk_indices).unwrap(); @@ -253,7 +253,7 @@ where }; let output_chunk = downsample_filter.apply_ndarray_continuous(gaussian_chunk, progress); progress.write(|| { - array_output.store_array_subset_ndarray::(output_subset.start(), output_chunk) + array_output.store_array_subset_ndarray::(output_subset.start(), output_chunk) })?; Ok(()) } diff --git a/src/filter/filters/clamp.rs b/src/filter/filters/clamp.rs index b5f65d7..c3aa5e0 100644 --- a/src/filter/filters/clamp.rs +++ b/src/filter/filters/clamp.rs @@ -102,7 +102,7 @@ impl FilterTraits for Clamp { chunk_input: &zarrs::array::ChunkRepresentation, chunk_output: &zarrs::array::ChunkRepresentation, ) -> usize { - chunk_input.size_usize() + chunk_output.size_usize() + chunk_input.fixed_element_size().unwrap() + chunk_output.fixed_element_size().unwrap() } fn apply( diff --git a/src/filter/filters/crop.rs b/src/filter/filters/crop.rs index d494ec2..2ced63a 100644 --- a/src/filter/filters/crop.rs +++ b/src/filter/filters/crop.rs @@ -3,7 +3,7 @@ use num_traits::AsPrimitive; use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use zarrs::{ - array::{data_type::UnsupportedDataTypeError, Array, DataType}, + array::{data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned}, array_subset::ArraySubset, storage::store::FilesystemStore, }; @@ -84,7 +84,7 @@ impl Crop { ) -> Result<(), FilterError> { let (input_subset, output_subset) = self.get_input_output_subset(output, chunk_indices); let output_bytes = progress.read(|| input.retrieve_array_subset(&input_subset))?; - progress.write(|| output.store_array_subset(&output_subset, &output_bytes))?; + progress.write(|| output.store_array_subset(&output_subset, output_bytes))?; progress.next(); Ok(()) } @@ -97,8 +97,8 @@ impl Crop { progress: &Progress, ) -> Result<(), FilterError> where - TIn: bytemuck::Pod + Send + Sync + AsPrimitive, - TOut: bytemuck::Pod + Send + Sync, + TIn: ElementOwned + Send + Sync + AsPrimitive, + TOut: Element + Send + Sync + Copy + 'static, { let (input_subset, output_subset) = self.get_input_output_subset(output, chunk_indices); @@ -154,7 +154,7 @@ impl FilterTraits for Crop { _chunk_input: &zarrs::array::ChunkRepresentation, chunk_output: &zarrs::array::ChunkRepresentation, ) -> usize { - chunk_output.size_usize() + chunk_output.fixed_element_size().unwrap() } fn output_shape(&self, _input: &Array) -> Option> { diff --git a/src/filter/filters/downsample.rs b/src/filter/filters/downsample.rs index 600d193..87b472e 100644 --- a/src/filter/filters/downsample.rs +++ b/src/filter/filters/downsample.rs @@ -153,9 +153,9 @@ impl FilterTraits for Downsample { chunk_output: &zarrs::array::ChunkRepresentation, ) -> usize { debug_assert_eq!(_chunk_input.data_type(), chunk_output.data_type()); - let input = chunk_output.size_usize() + let input = chunk_output.fixed_element_size().unwrap() * usize::try_from(self.stride.iter().product::()).unwrap(); - let output = chunk_output.size_usize(); + let output = chunk_output.fixed_element_size().unwrap(); input + output } @@ -207,7 +207,7 @@ impl FilterTraits for Downsample { self.apply_ndarray_continuous(input_array, &progress) }; progress.write(|| { - output.store_array_subset_ndarray::<$t_out, _, _>( + output.store_array_subset_ndarray::<$t_out, _>( output_subset.start(), output_array, ) @@ -220,7 +220,7 @@ impl FilterTraits for Downsample { .read(|| input.retrieve_array_subset_ndarray::<$t_in>(&input_subset))?; let output_array = self.apply_ndarray_continuous(input_array, &progress); progress.write(|| { - output.store_array_subset_ndarray::<$t_out, _, _>( + output.store_array_subset_ndarray::<$t_out, _>( output_subset.start(), output_array, ) diff --git a/src/filter/filters/equal.rs b/src/filter/filters/equal.rs index 4bbb73c..081a1da 100644 --- a/src/filter/filters/equal.rs +++ b/src/filter/filters/equal.rs @@ -3,7 +3,10 @@ use num_traits::AsPrimitive; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use zarrs::{ - array::{data_type::UnsupportedDataTypeError, Array, DataType, FillValue, FillValueMetadata}, + array::{ + data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned, FillValue, + FillValueMetadata, + }, array_subset::ArraySubset, storage::store::FilesystemStore, }; @@ -64,8 +67,8 @@ impl Equal { equal: &TIn, ) -> Result, FilterError> where - TIn: bytemuck::Pod + Copy + Send + Sync + PartialEq, - TOut: bytemuck::Pod + Send + Sync, + TIn: ElementOwned + Copy + Send + Sync + PartialEq, + TOut: Element + Send + Sync + Copy + 'static, bool: AsPrimitive, { let output_elements = input_elements @@ -114,7 +117,7 @@ impl FilterTraits for Equal { chunk_input: &zarrs::array::ChunkRepresentation, chunk_output: &zarrs::array::ChunkRepresentation, ) -> usize { - chunk_input.size_usize() + chunk_output.size_usize() + chunk_input.fixed_element_size().unwrap() + chunk_output.fixed_element_size().unwrap() } fn output_data_type(&self, _input: &Array) -> Option<(DataType, FillValue)> { diff --git a/src/filter/filters/gaussian.rs b/src/filter/filters/gaussian.rs index e6a9e1b..5ae2e97 100644 --- a/src/filter/filters/gaussian.rs +++ b/src/filter/filters/gaussian.rs @@ -5,7 +5,7 @@ use num_traits::AsPrimitive; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use zarrs::{ - array::{data_type::UnsupportedDataTypeError, Array, DataType}, + array::{data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned}, array_subset::ArraySubset, storage::store::FilesystemStore, }; @@ -77,8 +77,8 @@ impl Gaussian { progress: &Progress, ) -> Result<(), FilterError> where - TIn: bytemuck::Pod + Send + Sync + AsPrimitive, - TOut: bytemuck::Pod + Send + Sync, + TIn: ElementOwned + Send + Sync + AsPrimitive, + TOut: Element + Send + Sync + Copy + 'static, f32: AsPrimitive, { let subset_output = output.chunk_subset_bounded(chunk_indices).unwrap(); @@ -98,7 +98,7 @@ impl Gaussian { progress.write(|| { output - .store_array_subset_ndarray::(subset_output.start(), output_array) + .store_array_subset_ndarray::(subset_output.start(), output_array) .unwrap() }); @@ -160,8 +160,10 @@ impl FilterTraits for Gaussian { ) .unwrap(); let num_output_elements = chunk_output.num_elements_usize(); - num_input_elements * (chunk_input.data_type().size() + core::mem::size_of::() * 2) - + num_output_elements * (core::mem::size_of::() + chunk_output.data_type().size()) + num_input_elements + * (chunk_input.data_type().fixed_size().unwrap() + core::mem::size_of::() * 2) + + num_output_elements + * (core::mem::size_of::() + chunk_output.data_type().fixed_size().unwrap()) } fn apply( diff --git a/src/filter/filters/gradient_magnitude.rs b/src/filter/filters/gradient_magnitude.rs index e86481b..5f1b328 100644 --- a/src/filter/filters/gradient_magnitude.rs +++ b/src/filter/filters/gradient_magnitude.rs @@ -4,7 +4,7 @@ use num_traits::AsPrimitive; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use zarrs::{ - array::{data_type::UnsupportedDataTypeError, Array, DataType}, + array::{data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned}, array_subset::ArraySubset, storage::store::FilesystemStore, }; @@ -54,8 +54,8 @@ impl GradientMagnitude { progress: &Progress, ) -> Result<(), FilterError> where - TIn: bytemuck::Pod + AsPrimitive, - TOut: bytemuck::Pod, + TIn: ElementOwned + AsPrimitive, + TOut: Element + Copy + 'static, f32: AsPrimitive, { // Determine the input and output subset @@ -79,7 +79,7 @@ impl GradientMagnitude { progress.write(|| { output - .store_array_subset_ndarray::(subset_output.start(), gradient_magnitude) + .store_array_subset_ndarray::(subset_output.start(), gradient_magnitude) .unwrap() }); @@ -156,8 +156,10 @@ impl FilterTraits for GradientMagnitude { ) .unwrap(); let num_output_elements = chunk_input.num_elements_usize(); - num_input_elements * (chunk_input.data_type().size() + core::mem::size_of::() * 4) - + num_output_elements * (core::mem::size_of::() + chunk_output.data_type().size()) + num_input_elements + * (chunk_input.data_type().fixed_size().unwrap() + core::mem::size_of::() * 4) + + num_output_elements + * (core::mem::size_of::() + chunk_output.data_type().fixed_size().unwrap()) } fn apply( diff --git a/src/filter/filters/guided_filter.rs b/src/filter/filters/guided_filter.rs index 3fe01b8..2366efd 100644 --- a/src/filter/filters/guided_filter.rs +++ b/src/filter/filters/guided_filter.rs @@ -5,7 +5,7 @@ use rayon::iter::{ }; use serde::{Deserialize, Serialize}; use zarrs::{ - array::{data_type::UnsupportedDataTypeError, Array, DataType}, + array::{data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned}, array_subset::ArraySubset, storage::store::FilesystemStore, }; @@ -77,8 +77,8 @@ impl GuidedFilter { progress: &Progress, ) -> Result<(), FilterError> where - TIn: bytemuck::Pod + Send + Sync + AsPrimitive, - TOut: bytemuck::Pod + Send + Sync, + TIn: ElementOwned + Send + Sync + AsPrimitive, + TOut: Element + Send + Sync + Copy + 'static, f32: AsPrimitive, { let subset_output = output.chunk_subset_bounded(chunk_indices).unwrap(); @@ -102,7 +102,7 @@ impl GuidedFilter { progress.write(|| { output - .store_array_subset_ndarray::(subset_output.start(), output_array) + .store_array_subset_ndarray::(subset_output.start(), output_array) .unwrap() }); @@ -227,8 +227,8 @@ impl FilterTraits for GuidedFilter { chunk_input: &zarrs::array::ChunkRepresentation, chunk_output: &zarrs::array::ChunkRepresentation, ) -> usize { - chunk_input.size_usize() - + chunk_output.size_usize() + chunk_input.fixed_element_size().unwrap() + + chunk_output.fixed_element_size().unwrap() + chunk_output.num_elements_usize() * (core::mem::size_of::() + core::mem::size_of::() * 2) } diff --git a/src/filter/filters/reencode.rs b/src/filter/filters/reencode.rs index 9f5658d..eece78b 100644 --- a/src/filter/filters/reencode.rs +++ b/src/filter/filters/reencode.rs @@ -3,7 +3,7 @@ use num_traits::AsPrimitive; use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use zarrs::{ - array::{data_type::UnsupportedDataTypeError, Array, DataType}, + array::{data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned}, array_subset::ArraySubset, storage::store::FilesystemStore, }; @@ -50,7 +50,7 @@ impl Reencode { ) -> Result<(), FilterError> { let input_output_subset = output.chunk_subset_bounded(chunk_indices).unwrap(); let subset_bytes = progress.read(|| input.retrieve_array_subset(&input_output_subset))?; - progress.write(|| output.store_array_subset(&input_output_subset, &subset_bytes))?; + progress.write(|| output.store_array_subset(&input_output_subset, subset_bytes))?; progress.next(); Ok(()) } @@ -63,8 +63,8 @@ impl Reencode { progress: &Progress, ) -> Result<(), FilterError> where - TIn: bytemuck::Pod + Send + Sync + AsPrimitive, - TOut: bytemuck::Pod + Send + Sync, + TIn: ElementOwned + Send + Sync + AsPrimitive, + TOut: Element + Send + Sync + Copy + 'static, { let input_output_subset = output.chunk_subset_bounded(chunk_indices).unwrap(); @@ -120,7 +120,7 @@ impl FilterTraits for Reencode { _chunk_input: &zarrs::array::ChunkRepresentation, chunk_output: &zarrs::array::ChunkRepresentation, ) -> usize { - chunk_output.size_usize() + chunk_output.fixed_element_size().unwrap() } fn apply( diff --git a/src/filter/filters/replace_value.rs b/src/filter/filters/replace_value.rs index fec07df..74e674e 100644 --- a/src/filter/filters/replace_value.rs +++ b/src/filter/filters/replace_value.rs @@ -128,7 +128,7 @@ impl FilterTraits for ReplaceValue { chunk_input: &zarrs::array::ChunkRepresentation, chunk_output: &zarrs::array::ChunkRepresentation, ) -> usize { - chunk_input.size_usize() + chunk_output.size_usize() + chunk_input.fixed_element_size().unwrap() + chunk_output.fixed_element_size().unwrap() } fn apply( diff --git a/src/filter/filters/rescale.rs b/src/filter/filters/rescale.rs index 6d020ef..30ec3a0 100644 --- a/src/filter/filters/rescale.rs +++ b/src/filter/filters/rescale.rs @@ -3,7 +3,7 @@ use num_traits::AsPrimitive; use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use zarrs::{ - array::{data_type::UnsupportedDataTypeError, Array, DataType}, + array::{data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned}, array_subset::ArraySubset, storage::store::FilesystemStore, }; @@ -19,8 +19,10 @@ use crate::{ #[derive(Debug, Clone, Parser, Serialize, Deserialize)] pub struct RescaleArguments { /// Multiplier term. + #[arg(allow_hyphen_values(true))] pub multiply: f64, /// Addition term. + #[arg(allow_hyphen_values(true))] pub add: f64, /// Perform the addition before multiplication. #[arg(long)] @@ -71,8 +73,8 @@ impl Rescale { progress: &Progress, ) -> Result<(), FilterError> where - TIn: bytemuck::Pod + Send + Sync + AsPrimitive, - TOut: bytemuck::Pod + Copy + Send + Sync + 'static, + TIn: ElementOwned + Send + Sync + AsPrimitive, + TOut: Element + Send + Sync + Copy + 'static, f64: AsPrimitive, { // Determine the input and output subset @@ -152,7 +154,7 @@ impl FilterTraits for Rescale { chunk_input: &zarrs::array::ChunkRepresentation, chunk_output: &zarrs::array::ChunkRepresentation, ) -> usize { - chunk_input.size_usize() + chunk_output.size_usize() + chunk_input.fixed_element_size().unwrap() + chunk_output.fixed_element_size().unwrap() } fn apply( diff --git a/src/filter/filters/summed_area_table.rs b/src/filter/filters/summed_area_table.rs index 52cfc6d..8ef66a8 100644 --- a/src/filter/filters/summed_area_table.rs +++ b/src/filter/filters/summed_area_table.rs @@ -6,7 +6,7 @@ use num_traits::{AsPrimitive, Zero}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use zarrs::{ - array::{data_type::UnsupportedDataTypeError, Array, DataType}, + array::{data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned}, array_subset::ArraySubset, storage::store::FilesystemStore, }; @@ -54,8 +54,8 @@ impl SummedAreaTable { progress: &Progress, ) -> Result<(), FilterError> where - TIn: bytemuck::Pod + Send + Sync, - TOut: bytemuck::Pod + Send + Sync + Zero + AddAssign, + TIn: ElementOwned + Send + Sync, + TOut: Element + ElementOwned + Send + Sync + Zero + AddAssign + Copy + 'static, TIn: AsPrimitive, { let dimensionality = chunk_start_dim.len(); @@ -138,7 +138,7 @@ impl FilterTraits for SummedAreaTable { chunk_input: &zarrs::array::ChunkRepresentation, chunk_output: &zarrs::array::ChunkRepresentation, ) -> usize { - chunk_input.size_usize() + chunk_output.size_usize() + chunk_input.fixed_element_size().unwrap() + chunk_output.fixed_element_size().unwrap() } fn apply( diff --git a/src/info/histogram.rs b/src/info/histogram.rs index 5452e62..0d92d29 100644 --- a/src/info/histogram.rs +++ b/src/info/histogram.rs @@ -1,9 +1,8 @@ -use bytemuck::Pod; use half::{bf16, f16}; use num_traits::AsPrimitive; use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; use zarrs::{ - array::{Array, ArrayError, DataType}, + array::{Array, ArrayError, DataType, ElementOwned}, array_subset::ArraySubset, storage::ReadableStorageTraits, }; @@ -39,7 +38,7 @@ pub fn calculate_histogram( pub fn calculate_histogram_t< TStorage: ReadableStorageTraits + 'static, - T: Pod + PartialOrd + Send + Sync + AsPrimitive, + T: ElementOwned + PartialOrd + Send + Sync + AsPrimitive, >( array: &Array, n_bins: usize, diff --git a/src/info/range.rs b/src/info/range.rs index 1da6d7f..6f63575 100644 --- a/src/info/range.rs +++ b/src/info/range.rs @@ -1,9 +1,8 @@ -use bytemuck::Pod; use half::{bf16, f16}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use rayon_iter_concurrent_limit::iter_concurrent_limit; use zarrs::{ - array::{Array, ArrayError, DataType}, + array::{Array, ArrayError, DataType, ElementOwned}, array_subset::ArraySubset, storage::ReadableStorageTraits, }; @@ -99,7 +98,7 @@ pub fn calculate_range( pub fn calculate_range_t< TStorage: ReadableStorageTraits + 'static, - T: Pod + PartialOrd + Send + Sync, + T: ElementOwned + PartialOrd + Send + Sync, >( array: &Array, t_min: T, @@ -111,10 +110,10 @@ pub fn calculate_range_t< let chunk_min_max = |chunk_indices: Vec| { // TODO: Codec concurrent limit let elements = array.retrieve_chunk_elements::(&chunk_indices)?; - let (mut min, mut max) = (t_min, t_max); - for element in elements { - min = if element < min { element } else { min }; - max = if element > max { element } else { max }; + let (mut min, mut max) = (t_min.clone(), t_max.clone()); + for element in &elements { + min = if element < &min { element.clone() } else { min }; + max = if element > &max { element.clone() } else { max }; } Ok::<_, ArrayError>((min, max)) }; diff --git a/src/lib.rs b/src/lib.rs index 07ed209..9e206c7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -556,10 +556,6 @@ pub fn get_array_builder_reencode( let fill_value = convert_fill_value(array.data_type(), array.fill_value(), &data_type); array_builder.fill_value(fill_value); } - assert_eq!( - array_builder.data_type.size(), - array_builder.fill_value.size() - ); if let Some(shard_shape) = shard_shape { array_builder.chunk_grid(shard_shape.try_into().unwrap()); @@ -638,11 +634,11 @@ pub fn do_reencode( let chunk_subset = array_out.chunk_subset(&chunk_indices).unwrap(); let bytes = progress .read(|| array_in.retrieve_array_subset_opt(&chunk_subset, &codec_options))?; - *bytes_decoded.lock().unwrap() += bytes.len(); + *bytes_decoded.lock().unwrap() += bytes.size(); if validate { progress.write(|| { - array_out.store_chunk_opt(&chunk_indices, &bytes, &codec_options) + array_out.store_chunk_opt(&chunk_indices, bytes.clone(), &codec_options) })?; let bytes_out = array_out .retrieve_chunk_opt(&chunk_indices, &codec_options) @@ -650,7 +646,7 @@ pub fn do_reencode( assert!(bytes == bytes_out); } else { progress.write(|| { - array_out.store_chunk_opt(&chunk_indices, &bytes, &codec_options) + array_out.store_chunk_opt(&chunk_indices, bytes, &codec_options) })?; } progress.next();