Skip to content

Commit

Permalink
refactor(rust): Purge unused code
Browse files Browse the repository at this point in the history
  • Loading branch information
@orlp
orlp committed Oct 16, 2024
1 parent 3255066 commit 753363a
Show file tree
Hide file tree
Showing 9 changed files with 8 additions and 593 deletions.
15 changes: 1 addition & 14 deletions crates/polars-arrow/src/compute/aggregate/sum.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use polars_error::PolarsResult;
use crate::array::{Array, PrimitiveArray};
use crate::bitmap::utils::{BitChunkIterExact, BitChunksExact};
use crate::bitmap::Bitmap;
use crate::datatypes::{ArrowDataType, PhysicalType, PrimitiveType};
use crate::datatypes::PhysicalType;
use crate::scalar::*;
use crate::types::simd::*;
use crate::types::NativeType;
Expand Down Expand Up @@ -102,19 +102,6 @@ where
}
}

/// Whether [`sum`] supports `dtype`
pub fn can_sum(dtype: &ArrowDataType) -> bool {
if let PhysicalType::Primitive(primitive) = dtype.to_physical_type() {
use PrimitiveType::*;
matches!(
primitive,
Int8 | Int16 | Int64 | Int128 | UInt8 | UInt16 | UInt32 | UInt64 | Float32 | Float64
)
} else {
false
}
}

/// Returns the sum of all elements in `array` as a [`Scalar`] of the same physical
/// and logical types as `array`.
/// # Error
Expand Down
213 changes: 0 additions & 213 deletions crates/polars-arrow/src/compute/arity.rs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
//! Defines kernels suitable to perform operations to primitive arrays.
use polars_error::PolarsResult;

use super::utils::{check_same_len, combine_validities_and};
use crate::array::PrimitiveArray;
use crate::bitmap::{Bitmap, MutableBitmap};
use crate::datatypes::ArrowDataType;
use crate::types::NativeType;

Expand All @@ -29,104 +26,6 @@ where
PrimitiveArray::<O>::new(dtype, values.into(), array.validity().cloned())
}

/// Version of unary that checks for errors in the closure used to create the
/// buffer
pub fn try_unary<I, F, O>(
array: &PrimitiveArray<I>,
op: F,
dtype: ArrowDataType,
) -> PolarsResult<PrimitiveArray<O>>
where
I: NativeType,
O: NativeType,
F: Fn(I) -> PolarsResult<O>,
{
let values = array
.values()
.iter()
.map(|v| op(*v))
.collect::<PolarsResult<Vec<_>>>()?
.into();

Ok(PrimitiveArray::<O>::new(
dtype,
values,
array.validity().cloned(),
))
}

/// Version of unary that returns an array and bitmap. Used when working with
/// overflowing operations
pub fn unary_with_bitmap<I, F, O>(
array: &PrimitiveArray<I>,
op: F,
dtype: ArrowDataType,
) -> (PrimitiveArray<O>, Bitmap)
where
I: NativeType,
O: NativeType,
F: Fn(I) -> (O, bool),
{
let mut mut_bitmap = MutableBitmap::with_capacity(array.len());

let values = array
.values()
.iter()
.map(|v| {
let (res, over) = op(*v);
mut_bitmap.push(over);
res
})
.collect::<Vec<_>>()
.into();

(
PrimitiveArray::<O>::new(dtype, values, array.validity().cloned()),
mut_bitmap.into(),
)
}

/// Version of unary that creates a mutable bitmap that is used to keep track
/// of checked operations. The resulting bitmap is compared with the array
/// bitmap to create the final validity array.
pub fn unary_checked<I, F, O>(
array: &PrimitiveArray<I>,
op: F,
dtype: ArrowDataType,
) -> PrimitiveArray<O>
where
I: NativeType,
O: NativeType,
F: Fn(I) -> Option<O>,
{
let mut mut_bitmap = MutableBitmap::with_capacity(array.len());

let values = array
.values()
.iter()
.map(|v| match op(*v) {
Some(val) => {
mut_bitmap.push(true);
val
},
None => {
mut_bitmap.push(false);
O::default()
},
})
.collect::<Vec<_>>()
.into();

// The validity has to be checked against the bitmap created during the
// creation of the values with the iterator. If an error was found during
// the iteration, then the validity is changed to None to mark the value
// as Null
let bitmap: Bitmap = mut_bitmap.into();
let validity = combine_validities_and(array.validity(), Some(&bitmap));

PrimitiveArray::<O>::new(dtype, values, validity)
}

/// Applies a binary operations to two primitive arrays.
///
/// This is the fastest way to perform an operation on two primitive array when the benefits of a
Expand Down Expand Up @@ -169,115 +68,3 @@ where

PrimitiveArray::<T>::new(dtype, values, validity)
}

/// Version of binary that checks for errors in the closure used to create the
/// buffer
pub fn try_binary<T, D, F>(
lhs: &PrimitiveArray<T>,
rhs: &PrimitiveArray<D>,
dtype: ArrowDataType,
op: F,
) -> PolarsResult<PrimitiveArray<T>>
where
T: NativeType,
D: NativeType,
F: Fn(T, D) -> PolarsResult<T>,
{
check_same_len(lhs, rhs)?;

let validity = combine_validities_and(lhs.validity(), rhs.validity());

let values = lhs
.values()
.iter()
.zip(rhs.values().iter())
.map(|(l, r)| op(*l, *r))
.collect::<PolarsResult<Vec<_>>>()?
.into();

Ok(PrimitiveArray::<T>::new(dtype, values, validity))
}

/// Version of binary that returns an array and bitmap. Used when working with
/// overflowing operations
pub fn binary_with_bitmap<T, D, F>(
lhs: &PrimitiveArray<T>,
rhs: &PrimitiveArray<D>,
dtype: ArrowDataType,
op: F,
) -> (PrimitiveArray<T>, Bitmap)
where
T: NativeType,
D: NativeType,
F: Fn(T, D) -> (T, bool),
{
check_same_len(lhs, rhs).unwrap();

let validity = combine_validities_and(lhs.validity(), rhs.validity());

let mut mut_bitmap = MutableBitmap::with_capacity(lhs.len());

let values = lhs
.values()
.iter()
.zip(rhs.values().iter())
.map(|(l, r)| {
let (res, over) = op(*l, *r);
mut_bitmap.push(over);
res
})
.collect::<Vec<_>>()
.into();

(
PrimitiveArray::<T>::new(dtype, values, validity),
mut_bitmap.into(),
)
}

/// Version of binary that creates a mutable bitmap that is used to keep track
/// of checked operations. The resulting bitmap is compared with the array
/// bitmap to create the final validity array.
pub fn binary_checked<T, D, F>(
lhs: &PrimitiveArray<T>,
rhs: &PrimitiveArray<D>,
dtype: ArrowDataType,
op: F,
) -> PrimitiveArray<T>
where
T: NativeType,
D: NativeType,
F: Fn(T, D) -> Option<T>,
{
check_same_len(lhs, rhs).unwrap();

let mut mut_bitmap = MutableBitmap::with_capacity(lhs.len());

let values = lhs
.values()
.iter()
.zip(rhs.values().iter())
.map(|(l, r)| match op(*l, *r) {
Some(val) => {
mut_bitmap.push(true);
val
},
None => {
mut_bitmap.push(false);
T::default()
},
})
.collect::<Vec<_>>()
.into();

let bitmap: Bitmap = mut_bitmap.into();
let validity = combine_validities_and(lhs.validity(), rhs.validity());

// The validity has to be checked against the bitmap created during the
// creation of the values with the iterator. If an error was found during
// the iteration, then the validity is changed to None to mark the value
// as Null
let validity = combine_validities_and(validity.as_ref(), Some(&bitmap));

PrimitiveArray::<T>::new(dtype, values, validity)
}
13 changes: 0 additions & 13 deletions crates/polars-arrow/src/compute/cast/binary_to.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,19 +92,6 @@ pub fn binary_to_utf8<O: Offset>(
)
}

/// Conversion to utf8
/// # Errors
/// This function errors if the values are not valid utf8
pub fn binary_to_large_utf8(
from: &BinaryArray<i32>,
to_dtype: ArrowDataType,
) -> PolarsResult<Utf8Array<i64>> {
let values = from.values().clone();
let offsets = from.offsets().into();

Utf8Array::<i64>::try_new(to_dtype, offsets, values, from.validity().cloned())
}

/// Casts a [`BinaryArray`] to a [`PrimitiveArray`], making any uncastable value a Null.
pub(super) fn binary_to_primitive<O: Offset, T>(
from: &BinaryArray<O>,
Expand Down
97 changes: 1 addition & 96 deletions crates/polars-arrow/src/compute/cast/dictionary_to.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use polars_error::{polars_bail, PolarsResult};

use super::{primitive_as_primitive, primitive_to_primitive, CastOptionsImpl};
use super::{primitive_to_primitive, CastOptionsImpl};
use crate::array::{Array, DictionaryArray, DictionaryKey};
use crate::compute::cast::cast;
use crate::datatypes::ArrowDataType;
Expand All @@ -23,101 +23,6 @@ macro_rules! key_cast {
}};
}

/// Casts a [`DictionaryArray`] to a new [`DictionaryArray`] by keeping the
/// keys and casting the values to `values_type`.
/// # Errors
/// This function errors if the values are not castable to `values_type`
pub fn dictionary_to_dictionary_values<K: DictionaryKey>(
from: &DictionaryArray<K>,
values_type: &ArrowDataType,
) -> PolarsResult<DictionaryArray<K>> {
let keys = from.keys();
let values = from.values();
let length = values.len();

let values = cast(values.as_ref(), values_type, CastOptionsImpl::default())?;

assert_eq!(values.len(), length); // this is guaranteed by `cast`
unsafe {
DictionaryArray::try_new_unchecked(from.dtype().clone(), keys.clone(), values.clone())
}
}

/// Similar to dictionary_to_dictionary_values, but overflowing cast is wrapped
pub fn wrapping_dictionary_to_dictionary_values<K: DictionaryKey>(
from: &DictionaryArray<K>,
values_type: &ArrowDataType,
) -> PolarsResult<DictionaryArray<K>> {
let keys = from.keys();
let values = from.values();
let length = values.len();

let values = cast(
values.as_ref(),
values_type,
CastOptionsImpl {
wrapped: true,
partial: false,
},
)?;
assert_eq!(values.len(), length); // this is guaranteed by `cast`
unsafe {
DictionaryArray::try_new_unchecked(from.dtype().clone(), keys.clone(), values.clone())
}
}

/// Casts a [`DictionaryArray`] to a new [`DictionaryArray`] backed by a
/// different physical type of the keys, while keeping the values equal.
/// # Errors
/// Errors if any of the old keys' values is larger than the maximum value
/// supported by the new physical type.
pub fn dictionary_to_dictionary_keys<K1, K2>(
from: &DictionaryArray<K1>,
) -> PolarsResult<DictionaryArray<K2>>
where
K1: DictionaryKey + num_traits::NumCast,
K2: DictionaryKey + num_traits::NumCast,
{
let keys = from.keys();
let values = from.values();
let is_ordered = from.is_ordered();

let casted_keys = primitive_to_primitive::<K1, K2>(keys, &K2::PRIMITIVE.into());

if casted_keys.null_count() > keys.null_count() {
polars_bail!(ComputeError: "overflow")
} else {
let dtype =
ArrowDataType::Dictionary(K2::KEY_TYPE, Box::new(values.dtype().clone()), is_ordered);
// SAFETY: this is safe because given a type `T` that fits in a `usize`, casting it to type `P` either overflows or also fits in a `usize`
unsafe { DictionaryArray::try_new_unchecked(dtype, casted_keys, values.clone()) }
}
}

/// Similar to dictionary_to_dictionary_keys, but overflowing cast is wrapped
pub fn wrapping_dictionary_to_dictionary_keys<K1, K2>(
from: &DictionaryArray<K1>,
) -> PolarsResult<DictionaryArray<K2>>
where
K1: DictionaryKey + num_traits::AsPrimitive<K2>,
K2: DictionaryKey,
{
let keys = from.keys();
let values = from.values();
let is_ordered = from.is_ordered();

let casted_keys = primitive_as_primitive::<K1, K2>(keys, &K2::PRIMITIVE.into());

if casted_keys.null_count() > keys.null_count() {
polars_bail!(ComputeError: "overflow")
} else {
let dtype =
ArrowDataType::Dictionary(K2::KEY_TYPE, Box::new(values.dtype().clone()), is_ordered);
// some of the values may not fit in `usize` and thus this needs to be checked
DictionaryArray::try_new(dtype, casted_keys, values.clone())
}
}

pub(super) fn dictionary_cast_dyn<K: DictionaryKey + num_traits::NumCast>(
array: &dyn Array,
to_type: &ArrowDataType,
Expand Down
2 changes: 1 addition & 1 deletion crates/polars-arrow/src/compute/cast/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use binview_to::binview_to_primitive_dyn;
pub use binview_to::utf8view_to_utf8;
pub use boolean_to::*;
pub use decimal_to::*;
pub use dictionary_to::*;
use dictionary_to::*;
use polars_error::{polars_bail, polars_ensure, polars_err, PolarsResult};
use polars_utils::IdxSize;
pub use primitive_to::*;
Expand Down
Loading

0 comments on commit 753363a

Please sign in to comment.