Skip to content

Commit

Permalink
refactor!: add CodecMetadataOptions
Browse files Browse the repository at this point in the history
  • Loading branch information
LDeakin committed Jan 13, 2025
1 parent 682cdc0 commit d0d7f9c
Show file tree
Hide file tree
Showing 22 changed files with 129 additions and 89 deletions.
2 changes: 1 addition & 1 deletion zarrs/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -603,7 +603,7 @@ impl<TStorage: ?Sized> Array<TStorage> {
// Codec metadata manipulation
match &mut metadata {
ArrayMetadata::V3(metadata) => {
metadata.codecs = self.codecs().create_metadatas_opt(options);
metadata.codecs = self.codecs().create_metadatas_opt(options.codec_options());
}
ArrayMetadata::V2(_metadata) => {
// NOTE: The codec related options in ArrayMetadataOptions do not impact V2 codecs
Expand Down
28 changes: 10 additions & 18 deletions zarrs/src/array/array_metadata_options.rs
Original file line number Diff line number Diff line change
@@ -1,44 +1,36 @@
use crate::config::{global_config, MetadataConvertVersion};

use super::codec::CodecMetadataOptions;

/// Options for writing array metadata.
#[derive(Debug, Clone)]
pub struct ArrayMetadataOptions {
experimental_codec_store_metadata_if_encode_only: bool,
codec_options: CodecMetadataOptions,
convert_version: MetadataConvertVersion,
include_zarrs_metadata: bool,
}

impl Default for ArrayMetadataOptions {
fn default() -> Self {
Self {
experimental_codec_store_metadata_if_encode_only: false,
codec_options: CodecMetadataOptions::default(),
convert_version: global_config().metadata_convert_version(),
include_zarrs_metadata: global_config().include_zarrs_metadata(),
}
}
}

impl ArrayMetadataOptions {
/// Return the [experimental codec store metadata if encode only](crate::config::Config#experimental-codec-store-metadata-if-encode-only) setting.
/// Return the codec options.
#[must_use]
pub fn experimental_codec_store_metadata_if_encode_only(&self) -> bool {
self.experimental_codec_store_metadata_if_encode_only
pub fn codec_options(&self) -> &CodecMetadataOptions {
&self.codec_options
}

/// Set the [experimental codec store metadata if encode only](crate::config::Config#experimental-codec-store-metadata-if-encode-only) setting.
/// Return a mutable reference to the codec options.
#[must_use]
pub fn with_experimental_codec_store_metadata_if_encode_only(mut self, enabled: bool) -> Self {
self.experimental_codec_store_metadata_if_encode_only = enabled;
self
}

/// Set the [experimental codec store metadata if encode only](crate::config::Config#experimental-codec-store-metadata-if-encode-only) setting.
pub fn set_experimental_codec_store_metadata_if_encode_only(
&mut self,
enabled: bool,
) -> &mut Self {
self.experimental_codec_store_metadata_if_encode_only = enabled;
self
pub fn codec_options_mut(&mut self) -> &mut CodecMetadataOptions {
&mut self.codec_options

Check warning on line 33 in zarrs/src/array/array_metadata_options.rs

View check run for this annotation

Codecov / codecov/patch

zarrs/src/array/array_metadata_options.rs#L32-L33

Added lines #L32 - L33 were not covered by tests
}

/// Get the [metadata convert version](crate::config::Config#metadata-convert-version) configuration.
Expand Down
10 changes: 6 additions & 4 deletions zarrs/src/array/codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@
pub mod array_to_array;
pub mod array_to_bytes;
pub mod bytes_to_bytes;
pub mod metadata_options;
pub mod options;

pub use metadata_options::CodecMetadataOptions;
pub use options::{CodecOptions, CodecOptionsBuilder};

// Array to array
Expand Down Expand Up @@ -96,8 +98,8 @@ use std::sync::Arc;

use super::array_bytes::update_bytes_flen;
use super::{
concurrency::RecommendedConcurrency, ArrayMetadataOptions, BytesRepresentation,
ChunkRepresentation, ChunkShape, DataType,
concurrency::RecommendedConcurrency, BytesRepresentation, ChunkRepresentation, ChunkShape,
DataType,
};
use super::{ArrayBytes, RawBytes};

Expand Down Expand Up @@ -199,13 +201,13 @@ pub trait CodecTraits: Send + Sync {
/// Create metadata.
///
/// A hidden codec (e.g. a cache) will return [`None`], since it will not have any associated metadata.
fn create_metadata_opt(&self, options: &ArrayMetadataOptions) -> Option<MetadataV3>;
fn create_metadata_opt(&self, options: &CodecMetadataOptions) -> Option<MetadataV3>;

/// Create metadata with default options.
///
/// A hidden codec (e.g. a cache) will return [`None`], since it will not have any associated metadata.
fn create_metadata(&self) -> Option<MetadataV3> {
self.create_metadata_opt(&ArrayMetadataOptions::default())
self.create_metadata_opt(&CodecMetadataOptions::default())
}

/// Indicates if the input to a codecs partial decoder should be cached for optimal performance.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ use crate::{
codec::{
options::CodecOptions, ArrayBytes, ArrayCodecTraits, ArrayPartialDecoderTraits,
ArrayPartialEncoderTraits, ArrayToArrayCodecTraits, ArrayToArrayPartialEncoderDefault,
CodecError, CodecTraits, RecommendedConcurrency,
CodecError, CodecMetadataOptions, CodecTraits, RecommendedConcurrency,
},
ArrayMetadataOptions, ChunkRepresentation, ChunkShape, DataType,
ChunkRepresentation, ChunkShape, DataType,
},
config::global_config,
metadata::v3::MetadataV3,
Expand Down Expand Up @@ -47,7 +47,7 @@ impl BitroundCodec {
}

impl CodecTraits for BitroundCodec {
fn create_metadata_opt(&self, options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, options: &CodecMetadataOptions) -> Option<MetadataV3> {

Check warning on line 50 in zarrs/src/array/codec/array_to_array/bitround/bitround_codec.rs

View check run for this annotation

Codecov / codecov/patch

zarrs/src/array/codec/array_to_array/bitround/bitround_codec.rs#L50

Added line #L50 was not covered by tests
if options.experimental_codec_store_metadata_if_encode_only() {
let configuration = BitroundCodecConfigurationV1 {
keepbits: self.keepbits,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ use crate::{
codec::{
options::CodecOptions, ArrayBytes, ArrayCodecTraits, ArrayPartialDecoderTraits,
ArrayPartialEncoderTraits, ArrayToArrayCodecTraits, ArrayToArrayPartialEncoderDefault,
CodecError, CodecTraits, RecommendedConcurrency,
CodecError, CodecMetadataOptions, CodecTraits, RecommendedConcurrency,
},
ArrayMetadataOptions, ChunkRepresentation, ChunkShape,
ChunkRepresentation, ChunkShape,
},
metadata::v3::{array::codec::transpose::TransposeCodecConfigurationV1, MetadataV3},
plugin::PluginCreateError,
Expand Down Expand Up @@ -48,7 +48,7 @@ impl TransposeCodec {
}

impl CodecTraits for TransposeCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = TransposeCodecConfigurationV1 {
order: self.order.clone(),
};
Expand Down
7 changes: 3 additions & 4 deletions zarrs/src/array/codec/array_to_bytes/bytes/bytes_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@ use crate::{
codec::{
ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderDefault,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesPartialDecoderTraits,
BytesPartialEncoderTraits, CodecError, CodecOptions, CodecTraits,
BytesPartialEncoderTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RecommendedConcurrency,
},
ArrayBytes, ArrayMetadataOptions, BytesRepresentation, ChunkRepresentation, DataTypeSize,
RawBytes,
ArrayBytes, BytesRepresentation, ChunkRepresentation, DataTypeSize, RawBytes,
},
metadata::v3::MetadataV3,
};
Expand Down Expand Up @@ -101,7 +100,7 @@ impl BytesCodec {
}

impl CodecTraits for BytesCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = BytesCodecConfigurationV1 {
endian: self.endian,
};
Expand Down
12 changes: 6 additions & 6 deletions zarrs/src/array/codec/array_to_bytes/codec_chain.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ use crate::{
ArrayCodecTraits, ArrayPartialDecoderCache, ArrayPartialDecoderTraits,
ArrayPartialEncoderTraits, ArrayToArrayCodecTraits, ArrayToBytesCodecTraits,
BytesPartialDecoderCache, BytesPartialDecoderTraits, BytesPartialEncoderTraits,
BytesToBytesCodecTraits, Codec, CodecError, CodecOptions, CodecTraits,
BytesToBytesCodecTraits, Codec, CodecError, CodecMetadataOptions, CodecOptions,
CodecTraits,
},
concurrency::RecommendedConcurrency,
ArrayBytes, ArrayMetadataOptions, BytesRepresentation, ChunkRepresentation, ChunkShape,
RawBytes,
ArrayBytes, BytesRepresentation, ChunkRepresentation, ChunkShape, RawBytes,
},
array_subset::ArraySubset,
metadata::v3::MetadataV3,
Expand Down Expand Up @@ -137,7 +137,7 @@ impl CodecChain {

/// Create codec chain metadata.
#[must_use]
pub fn create_metadatas_opt(&self, options: &ArrayMetadataOptions) -> Vec<MetadataV3> {
pub fn create_metadatas_opt(&self, options: &CodecMetadataOptions) -> Vec<MetadataV3> {
let mut metadatas =
Vec::with_capacity(self.array_to_array.len() + 1 + self.bytes_to_bytes.len());
for codec in &self.array_to_array {
Expand All @@ -159,7 +159,7 @@ impl CodecChain {
/// Create codec chain metadata with default options.
#[must_use]
pub fn create_metadatas(&self) -> Vec<MetadataV3> {
self.create_metadatas_opt(&ArrayMetadataOptions::default())
self.create_metadatas_opt(&CodecMetadataOptions::default())
}

/// Get the array to array codecs
Expand Down Expand Up @@ -215,7 +215,7 @@ impl CodecTraits for CodecChain {
/// Returns [`None`] since a codec chain does not have standard codec metadata.
///
/// Note that usage of the codec chain is explicit in [`Array`](crate::array::Array) and [`CodecChain::create_metadatas_opt()`] will call [`CodecTraits::create_metadata_opt()`] from for each codec.
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {

Check warning on line 218 in zarrs/src/array/codec/array_to_bytes/codec_chain.rs

View check run for this annotation

Codecov / codecov/patch

zarrs/src/array/codec/array_to_bytes/codec_chain.rs#L218

Added line #L218 was not covered by tests
None
}

Expand Down
10 changes: 5 additions & 5 deletions zarrs/src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ use crate::{
codec::{
ArrayBytes, ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderDefault,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesPartialDecoderTraits,
BytesPartialEncoderTraits, CodecError, CodecOptions, CodecTraits, RawBytes,
RecommendedConcurrency,
BytesPartialEncoderTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RawBytes, RecommendedConcurrency,
},
convert_from_bytes_slice, transmute_to_bytes_vec, ArrayMetadataOptions,
BytesRepresentation, ChunkRepresentation, DataType,
convert_from_bytes_slice, transmute_to_bytes_vec, BytesRepresentation, ChunkRepresentation,
DataType,
},
config::global_config,
metadata::v3::{array::codec::pcodec::PcodecModeSpecConfiguration, MetadataV3},
Expand Down Expand Up @@ -84,7 +84,7 @@ impl PcodecCodec {
}

impl CodecTraits for PcodecCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let mode_spec = mode_spec_pco_to_config(&self.chunk_config.mode_spec);
let (delta_spec, delta_encoding_order) = match self.chunk_config.delta_spec {
DeltaSpec::Auto => (PcodecDeltaSpecConfiguration::Auto, None),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@ use crate::{
codec::{
ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderTraits,
ArrayToBytesCodecTraits, BytesPartialDecoderTraits, BytesPartialEncoderTraits,
CodecChain, CodecError, CodecOptions, CodecTraits, RecommendedConcurrency,
CodecChain, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RecommendedConcurrency,
},
concurrency::calc_concurrency_outer_inner,
transmute_to_bytes_vec, unravel_index, ArrayBytes, ArrayMetadataOptions, ArraySize,
BytesRepresentation, ChunkRepresentation, ChunkShape, DataTypeSize, FillValue, RawBytes,
transmute_to_bytes_vec, unravel_index, ArrayBytes, ArraySize, BytesRepresentation,
ChunkRepresentation, ChunkShape, DataTypeSize, FillValue, RawBytes,
},
array_subset::ArraySubset,
metadata::v3::MetadataV3,
Expand Down Expand Up @@ -85,7 +86,7 @@ impl ShardingCodec {
}

impl CodecTraits for ShardingCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = ShardingCodecConfigurationV1 {
chunk_shape: self.chunk_shape.clone(),
codecs: self.inner_codecs.create_metadatas(),
Expand Down
10 changes: 5 additions & 5 deletions zarrs/src/array/codec/array_to_bytes/vlen/vlen_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ use crate::{
codec::{
ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderDefault,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesCodec,
BytesPartialDecoderTraits, BytesPartialEncoderTraits, CodecError, CodecOptions,
CodecTraits, RecommendedConcurrency,
BytesPartialDecoderTraits, BytesPartialEncoderTraits, CodecError, CodecMetadataOptions,
CodecOptions, CodecTraits, RecommendedConcurrency,
},
transmute_to_bytes_vec, ArrayBytes, ArrayMetadataOptions, BytesRepresentation,
ChunkRepresentation, CodecChain, DataType, DataTypeSize, Endianness, FillValue, RawBytes,
transmute_to_bytes_vec, ArrayBytes, BytesRepresentation, ChunkRepresentation, CodecChain,
DataType, DataTypeSize, Endianness, FillValue, RawBytes,
},
config::global_config,
metadata::v3::{array::codec::vlen::VlenIndexDataType, MetadataV3},
Expand Down Expand Up @@ -83,7 +83,7 @@ impl VlenCodec {
}

impl CodecTraits for VlenCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = VlenCodecConfigurationV1 {
index_codecs: self.index_codecs.create_metadatas(),
data_codecs: self.data_codecs.create_metadatas(),
Expand Down
7 changes: 3 additions & 4 deletions zarrs/src/array/codec/array_to_bytes/vlen_v2/vlen_v2_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@ use crate::{
codec::{
ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderDefault,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesPartialDecoderTraits,
BytesPartialEncoderTraits, CodecError, CodecOptions, CodecTraits,
BytesPartialEncoderTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RecommendedConcurrency,
},
ArrayBytes, ArrayMetadataOptions, BytesRepresentation, ChunkRepresentation, DataTypeSize,
RawBytes,
ArrayBytes, BytesRepresentation, ChunkRepresentation, DataTypeSize, RawBytes,
},
config::global_config,
metadata::v3::MetadataV3,
Expand All @@ -35,7 +34,7 @@ impl VlenV2Codec {
}

impl CodecTraits for VlenV2Codec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let config = global_config();
let name = config
.experimental_codec_names()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,11 @@ macro_rules! vlen_v2_codec {
codec::{
array_to_bytes::vlen_v2::VlenV2Codec, ArrayPartialDecoderTraits,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesPartialDecoderTraits,
BytesPartialEncoderTraits, CodecError, CodecOptions, CodecTraits,
BytesPartialEncoderTraits, CodecError, CodecMetadataOptions, CodecOptions,
CodecTraits,
},
ArrayBytes, ArrayCodecTraits, ArrayMetadataOptions, BytesRepresentation,
ChunkRepresentation, RawBytes, RecommendedConcurrency,
ArrayBytes, ArrayCodecTraits, BytesRepresentation, ChunkRepresentation, RawBytes,
RecommendedConcurrency,
};

#[cfg(feature = "async")]
Expand Down Expand Up @@ -77,7 +78,7 @@ macro_rules! vlen_v2_codec {
}

impl CodecTraits for $struct {
fn create_metadata_opt(&self, options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, options: &CodecMetadataOptions) -> Option<MetadataV3> {
self.inner.create_metadata_opt(options)
}

Expand Down
8 changes: 4 additions & 4 deletions zarrs/src/array/codec/array_to_bytes/zfp/zfp_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ use crate::{
codec::{
ArrayBytes, ArrayCodecTraits, ArrayPartialDecoderTraits, ArrayPartialEncoderDefault,
ArrayPartialEncoderTraits, ArrayToBytesCodecTraits, BytesPartialDecoderTraits,
BytesPartialEncoderTraits, CodecError, CodecOptions, CodecTraits, RawBytes,
RecommendedConcurrency,
BytesPartialEncoderTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RawBytes, RecommendedConcurrency,
},
ArrayMetadataOptions, BytesRepresentation, ChunkRepresentation, DataType,
BytesRepresentation, ChunkRepresentation, DataType,
},
config::global_config,
metadata::v3::{array::codec::zfp::ZfpMode, MetadataV3},
Expand Down Expand Up @@ -129,7 +129,7 @@ impl ZfpCodec {
}

impl CodecTraits for ZfpCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = ZfpCodecConfigurationV1 {
write_header: Some(self.write_header),
mode: self.mode,
Expand Down
7 changes: 4 additions & 3 deletions zarrs/src/array/codec/bytes_to_bytes/blosc/blosc_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@ use crate::{
array::{
codec::{
BytesPartialDecoderTraits, BytesPartialEncoderDefault, BytesPartialEncoderTraits,
BytesToBytesCodecTraits, CodecError, CodecOptions, CodecTraits, RecommendedConcurrency,
BytesToBytesCodecTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RecommendedConcurrency,
},
ArrayMetadataOptions, BytesRepresentation, RawBytes,
BytesRepresentation, RawBytes,
},
metadata::v3::MetadataV3,
plugin::PluginCreateError,
Expand Down Expand Up @@ -131,7 +132,7 @@ impl BloscCodec {
}

impl CodecTraits for BloscCodec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = BloscCodecConfigurationV1 {
cname: self.cname,
clevel: self.clevel,
Expand Down
7 changes: 4 additions & 3 deletions zarrs/src/array/codec/bytes_to_bytes/bz2/bz2_codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ use crate::{
array::{
codec::{
BytesPartialDecoderTraits, BytesPartialEncoderDefault, BytesPartialEncoderTraits,
BytesToBytesCodecTraits, CodecError, CodecOptions, CodecTraits, RecommendedConcurrency,
BytesToBytesCodecTraits, CodecError, CodecMetadataOptions, CodecOptions, CodecTraits,
RecommendedConcurrency,
},
ArrayMetadataOptions, BytesRepresentation, RawBytes,
BytesRepresentation, RawBytes,
},
config::global_config,
metadata::v3::MetadataV3,
Expand Down Expand Up @@ -46,7 +47,7 @@ impl Bz2Codec {
}

impl CodecTraits for Bz2Codec {
fn create_metadata_opt(&self, _options: &ArrayMetadataOptions) -> Option<MetadataV3> {
fn create_metadata_opt(&self, _options: &CodecMetadataOptions) -> Option<MetadataV3> {
let configuration = Bz2CodecConfigurationV1 {
level: Bz2CompressionLevel::try_from(self.compression.level())
.expect("checked on init"),
Expand Down
Loading

0 comments on commit d0d7f9c

Please sign in to comment.