From a4f9b6c1973ce4bf43e17c822ba013fe78616e05 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Wed, 5 Mar 2025 17:14:55 -0800 Subject: [PATCH 1/7] fixEmptyArrayHasAll --- datafusion/functions-nested/src/array_has.rs | 8 ++++++++ datafusion/sqllogictest/test_files/array.slt | 8 +++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/datafusion/functions-nested/src/array_has.rs b/datafusion/functions-nested/src/array_has.rs index 5a29cf962817..09090244da98 100644 --- a/datafusion/functions-nested/src/array_has.rs +++ b/datafusion/functions-nested/src/array_has.rs @@ -439,6 +439,14 @@ fn array_has_all_and_any_dispatch( ) -> Result { let haystack = as_generic_list_array::(haystack)?; let needle = as_generic_list_array::(needle)?; + if needle.values().len() == 0 { + let result_value = match comparison_type { + ComparisonType::All => true, + ComparisonType::Any => false, + }; + let result = BooleanArray::from(vec![result_value; haystack.len()]); + return Ok(Arc::new(result)); + } match needle.data_type() { DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => { array_has_all_and_any_string_internal::(haystack, needle, comparison_type) diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index c8f6a985bb22..a47389af7ba0 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -5818,14 +5818,16 @@ false false false false false false false false false false false false -query BBBBBBBBBBBBB -select array_has_all(make_array(1,2,3), make_array(1,3)), +query BBBBBBBBBBBBBBB +select array_has_all(make_array(1,2,3), []), + array_has_all(make_array(1,2,3), make_array(1,3)), array_has_all(make_array(1,2,3), make_array(1,4)), array_has_all(make_array([1,2], [3,4]), make_array([1,2])), array_has_all(make_array([1,2], [3,4]), make_array([1,3])), array_has_all(make_array([1,2], [3,4]), make_array([1,2], [3,4], [5,6])), array_has_all(make_array([[1,2,3]]), make_array([[1]])), array_has_all(make_array([[1,2,3]]), make_array([[1,2,3]])), + array_has_any(make_array(1,2,3), []), array_has_any(make_array(1,2,3), make_array(1,10,100)), array_has_any(make_array(1,2,3), make_array(10,100)), array_has_any(make_array([1,2], [3,4]), make_array([1,10], [10,4])), @@ -5834,7 +5836,7 @@ select array_has_all(make_array(1,2,3), make_array(1,3)), array_has_any(make_array([[1,2,3]]), make_array([[1,2,3]], [[4,5,6]])) ; ---- -true false true false false false true true false false true false true +true true false true false false false false true true false false true false true query BBBBBBBBBBBBB select array_has_all(arrow_cast(make_array(1,2,3), 'LargeList(Int64)'), arrow_cast(make_array(1,3), 'LargeList(Int64)')), From af2c50dc0edbf062e0593d532dec8f1a1c279b3e Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Wed, 5 Mar 2025 17:18:14 -0800 Subject: [PATCH 2/7] Fix the array --- datafusion/sqllogictest/test_files/array.slt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index a47389af7ba0..084cb9abf239 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -5836,7 +5836,7 @@ select array_has_all(make_array(1,2,3), []), array_has_any(make_array([[1,2,3]]), make_array([[1,2,3]], [[4,5,6]])) ; ---- -true true false true false false false false true true false false true false true +true true false true false false false true false true false false true false true query BBBBBBBBBBBBB select array_has_all(arrow_cast(make_array(1,2,3), 'LargeList(Int64)'), arrow_cast(make_array(1,3), 'LargeList(Int64)')), From 41f77d1d06f1e2a46a073271562ed6d9ac282138 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Thu, 6 Mar 2025 10:07:41 -0800 Subject: [PATCH 3/7] Change to more efficient BooleanBuffer --- datafusion/functions-nested/src/array_has.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/datafusion/functions-nested/src/array_has.rs b/datafusion/functions-nested/src/array_has.rs index 09090244da98..3d2d6782b946 100644 --- a/datafusion/functions-nested/src/array_has.rs +++ b/datafusion/functions-nested/src/array_has.rs @@ -440,12 +440,14 @@ fn array_has_all_and_any_dispatch( let haystack = as_generic_list_array::(haystack)?; let needle = as_generic_list_array::(needle)?; if needle.values().len() == 0 { - let result_value = match comparison_type { - ComparisonType::All => true, - ComparisonType::Any => false, + return match comparison_type { + ComparisonType::All => Ok(Arc::new(BooleanArray::from( + BooleanBuffer::new_set(haystack.len()), + ))), + ComparisonType::Any => Ok(Arc::new(BooleanArray::from( + BooleanBuffer::new_unset(haystack.len()), + ))), }; - let result = BooleanArray::from(vec![result_value; haystack.len()]); - return Ok(Arc::new(result)); } match needle.data_type() { DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => { From e927d659f28da06f79ce5c84833de33522641547 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Thu, 6 Mar 2025 10:47:28 -0800 Subject: [PATCH 4/7] Separate test case --- datafusion/sqllogictest/test_files/array.slt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index 084cb9abf239..ffaf12e11771 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -5818,16 +5818,20 @@ false false false false false false false false false false false false -query BBBBBBBBBBBBBBB +query BB select array_has_all(make_array(1,2,3), []), - array_has_all(make_array(1,2,3), make_array(1,3)), - array_has_all(make_array(1,2,3), make_array(1,4)), + array_has_any(make_array(1,2,3), []) +; +---- +true false + +query BBBBBBBBBBBBB +select array_has_all(make_array(1,2,3), make_array(1,3)), array_has_all(make_array([1,2], [3,4]), make_array([1,2])), array_has_all(make_array([1,2], [3,4]), make_array([1,3])), array_has_all(make_array([1,2], [3,4]), make_array([1,2], [3,4], [5,6])), array_has_all(make_array([[1,2,3]]), make_array([[1]])), array_has_all(make_array([[1,2,3]]), make_array([[1,2,3]])), - array_has_any(make_array(1,2,3), []), array_has_any(make_array(1,2,3), make_array(1,10,100)), array_has_any(make_array(1,2,3), make_array(10,100)), array_has_any(make_array([1,2], [3,4]), make_array([1,10], [10,4])), @@ -5836,7 +5840,7 @@ select array_has_all(make_array(1,2,3), []), array_has_any(make_array([[1,2,3]]), make_array([[1,2,3]], [[4,5,6]])) ; ---- -true true false true false false false true false true false false true false true +true false true false false false true true false false true false true query BBBBBBBBBBBBB select array_has_all(arrow_cast(make_array(1,2,3), 'LargeList(Int64)'), arrow_cast(make_array(1,3), 'LargeList(Int64)')), From 8c87e18b611851d6dda64e410c75988839cbd4e4 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Thu, 6 Mar 2025 10:48:23 -0800 Subject: [PATCH 5/7] small fix --- datafusion/sqllogictest/test_files/array.slt | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index ffaf12e11771..7c8e33018a00 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -5827,6 +5827,7 @@ true false query BBBBBBBBBBBBB select array_has_all(make_array(1,2,3), make_array(1,3)), + array_has_all(make_array(1,2,3), make_array(1,4)), array_has_all(make_array([1,2], [3,4]), make_array([1,2])), array_has_all(make_array([1,2], [3,4]), make_array([1,3])), array_has_all(make_array([1,2], [3,4]), make_array([1,2], [3,4], [5,6])), From ff316b6fa778fdff317a3855d75d6994dbc370b8 Mon Sep 17 00:00:00 2001 From: LuQQiu Date: Fri, 7 Mar 2025 10:05:31 -0800 Subject: [PATCH 6/7] Update datafusion/functions-nested/src/array_has.rs Co-authored-by: Alex Huang --- datafusion/functions-nested/src/array_has.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/datafusion/functions-nested/src/array_has.rs b/datafusion/functions-nested/src/array_has.rs index 3d2d6782b946..1857ead8c547 100644 --- a/datafusion/functions-nested/src/array_has.rs +++ b/datafusion/functions-nested/src/array_has.rs @@ -440,14 +440,11 @@ fn array_has_all_and_any_dispatch( let haystack = as_generic_list_array::(haystack)?; let needle = as_generic_list_array::(needle)?; if needle.values().len() == 0 { - return match comparison_type { - ComparisonType::All => Ok(Arc::new(BooleanArray::from( - BooleanBuffer::new_set(haystack.len()), - ))), - ComparisonType::Any => Ok(Arc::new(BooleanArray::from( - BooleanBuffer::new_unset(haystack.len()), - ))), + let buffer = match comparison_type { + ComparisonType::All => BooleanBuffer::new_set(haystack.len()), + ComparisonType::Any => BooleanBuffer::new_unset(haystack.len()), }; + return Ok(Arc::new(BooleanArray::from(buffer))); } match needle.data_type() { DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => { From ba49e8bd97504b74bea100d651b87bbeeee95368 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Fri, 7 Mar 2025 14:51:34 -0800 Subject: [PATCH 7/7] Add the string failure test cases --- datafusion/sqllogictest/test_files/array.slt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index 7c8e33018a00..e8748aae127b 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -5818,12 +5818,14 @@ false false false false false false false false false false false false -query BB +query BBBB select array_has_all(make_array(1,2,3), []), - array_has_any(make_array(1,2,3), []) + array_has_any(make_array(1,2,3), []), + array_has_all(make_array('aa','bb','cc'), []), + array_has_any(make_array('aa','bb','cc'), []), ; ---- -true false +true false true false query BBBBBBBBBBBBB select array_has_all(make_array(1,2,3), make_array(1,3)),