Skip to content

Commit

Permalink
fix: Implement to_arrow functionality properly for Arrays (#19077)
Browse files Browse the repository at this point in the history
  • Loading branch information
siddharth-vi authored Oct 11, 2024
1 parent dbbd93f commit b792033
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 0 deletions.
28 changes: 28 additions & 0 deletions crates/polars-core/src/series/into.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,34 @@ impl Series {
);
Box::new(arr)
},
#[cfg(feature = "dtype-array")]
DataType::Array(inner, width) => {
let ca = self.array().unwrap();
let arr = ca.chunks[chunk_idx].clone();
let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();

let new_values = if let DataType::Null = &**inner {
arr.values().clone()
} else {
let s = unsafe {
Series::from_chunks_and_dtype_unchecked(
PlSmallStr::EMPTY,
vec![arr.values().clone()],
&inner.to_physical(),
)
.cast_unchecked(inner)
.unwrap()
};

s.to_arrow(0, compat_level)
};

let dtype =
FixedSizeListArray::default_datatype(inner.to_arrow(compat_level), *width);
let arr =
FixedSizeListArray::new(dtype, arr.len(), new_values, arr.validity().cloned());
Box::new(arr)
},
#[cfg(feature = "dtype-categorical")]
dt @ (DataType::Categorical(_, ordering) | DataType::Enum(_, ordering)) => {
let ca = self.categorical().unwrap();
Expand Down
26 changes: 26 additions & 0 deletions py-polars/tests/unit/interop/test_interop.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,32 @@ def test_arrow_list_chunked_array() -> None:
assert s.dtype == pl.List


# Test that polars convert Arrays of logical types correctly to arrow
def test_arrow_array_logical() -> None:
# cast to large string and uint32 indices because polars converts to those
pa_data1 = (
pa.array(["a", "b", "c", "d"])
.dictionary_encode()
.cast(pa.dictionary(pa.uint32(), pa.large_string()))
)
pa_array_logical1 = pa.FixedSizeListArray.from_arrays(pa_data1, 2)

s1 = pl.Series(
values=[["a", "b"], ["c", "d"]],
dtype=pl.Array(pl.Enum(["a", "b", "c", "d"]), shape=2),
)
assert s1.to_arrow() == pa_array_logical1

pa_data2 = pa.array([date(2024, 1, 1), date(2024, 1, 2)])
pa_array_logical2 = pa.FixedSizeListArray.from_arrays(pa_data2, 1)

s2 = pl.Series(
values=[[date(2024, 1, 1)], [date(2024, 1, 2)]],
dtype=pl.Array(pl.Date, shape=1),
)
assert s2.to_arrow() == pa_array_logical2


def test_from_dict() -> None:
data = {"a": [1, 2], "b": [3, 4]}
df = pl.from_dict(data)
Expand Down

0 comments on commit b792033

Please sign in to comment.