From 6b1b0e46439c8a6bdd7605b2f71e1cba5899319f Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Fri, 2 Jan 2026 14:49:23 +0530 Subject: [PATCH 1/9] refactor: Use TypeSignatureClass::Float for isnan/iszero/nanvl signatures --- datafusion/functions/src/math/iszero.rs | 13 ++++++++----- datafusion/functions/src/math/nans.rs | 17 +++++++++-------- datafusion/functions/src/math/nanvl.rs | 7 +++---- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/datafusion/functions/src/math/iszero.rs b/datafusion/functions/src/math/iszero.rs index 6349551ca0a43..ae4fbe9c64fdb 100644 --- a/datafusion/functions/src/math/iszero.rs +++ b/datafusion/functions/src/math/iszero.rs @@ -22,8 +22,8 @@ use arrow::array::{ArrayRef, AsArray, BooleanArray}; use arrow::datatypes::DataType::{Boolean, Float32, Float64}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; -use datafusion_common::{Result, exec_err}; -use datafusion_expr::TypeSignature::Exact; +use datafusion_common::{Result, ScalarValue, exec_err}; +use datafusion_expr::{Coercion, TypeSignatureClass}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -59,10 +59,9 @@ impl Default for IsZeroFunc { impl IsZeroFunc { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::one_of( - vec![Exact(vec![Float32]), Exact(vec![Float64])], + signature: Signature::coercible( + vec![Coercion::new_exact(TypeSignatureClass::Float)], Volatility::Immutable, ), } @@ -87,6 +86,10 @@ impl ScalarUDFImpl for IsZeroFunc { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + // Handle NULL input + if args.args[0].data_type().is_null() { + return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(None))); + } make_scalar_function(iszero, vec![])(&args.args) } diff --git a/datafusion/functions/src/math/nans.rs b/datafusion/functions/src/math/nans.rs index be21cfde0aa6c..1cf7bee5e5a0d 100644 --- a/datafusion/functions/src/math/nans.rs +++ b/datafusion/functions/src/math/nans.rs @@ -18,8 +18,8 @@ //! Math function: `isnan()`. use arrow::datatypes::{DataType, Float32Type, Float64Type}; -use datafusion_common::{Result, exec_err}; -use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, TypeSignature}; +use datafusion_common::{Result, ScalarValue, exec_err}; +use datafusion_expr::{Coercion, ColumnarValue, ScalarFunctionArgs, TypeSignatureClass}; use arrow::array::{ArrayRef, AsArray, BooleanArray}; use datafusion_expr::{Documentation, ScalarUDFImpl, Signature, Volatility}; @@ -54,13 +54,9 @@ impl Default for IsNanFunc { impl IsNanFunc { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::one_of( - vec![ - TypeSignature::Exact(vec![Float32]), - TypeSignature::Exact(vec![Float64]), - ], + signature: Signature::coercible( + vec![Coercion::new_exact(TypeSignatureClass::Float)], Volatility::Immutable, ), } @@ -84,6 +80,11 @@ impl ScalarUDFImpl for IsNanFunc { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + // Handle NULL input + if args.args[0].data_type().is_null() { + return Ok(ColumnarValue::Scalar(ScalarValue::Boolean(None))); + } + let args = ColumnarValue::values_to_arrays(&args.args)?; let arr: ArrayRef = match args[0].data_type() { diff --git a/datafusion/functions/src/math/nanvl.rs b/datafusion/functions/src/math/nanvl.rs index 345b1a5b71aef..12a2bfea6bd7b 100644 --- a/datafusion/functions/src/math/nanvl.rs +++ b/datafusion/functions/src/math/nanvl.rs @@ -24,7 +24,6 @@ use arrow::array::{ArrayRef, AsArray, Float32Array, Float64Array}; use arrow::datatypes::DataType::{Float32, Float64}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; use datafusion_common::{DataFusionError, Result, exec_err}; -use datafusion_expr::TypeSignature::Exact; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -66,10 +65,10 @@ impl Default for NanvlFunc { impl NanvlFunc { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::one_of( - vec![Exact(vec![Float32, Float32]), Exact(vec![Float64, Float64])], + signature: Signature::uniform( + 2, + vec![Float32, Float64], Volatility::Immutable, ), } From f4eb9cbc588116f73d090573e5956f8bc4ae176d Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Fri, 2 Jan 2026 15:09:52 +0530 Subject: [PATCH 2/9] change nanvl for consistency --- datafusion/functions/src/math/nanvl.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/datafusion/functions/src/math/nanvl.rs b/datafusion/functions/src/math/nanvl.rs index 12a2bfea6bd7b..a6fc2c05ae6db 100644 --- a/datafusion/functions/src/math/nanvl.rs +++ b/datafusion/functions/src/math/nanvl.rs @@ -24,6 +24,7 @@ use arrow::array::{ArrayRef, AsArray, Float32Array, Float64Array}; use arrow::datatypes::DataType::{Float32, Float64}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; use datafusion_common::{DataFusionError, Result, exec_err}; +use datafusion_expr::{Coercion, TypeSignatureClass}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -65,10 +66,10 @@ impl Default for NanvlFunc { impl NanvlFunc { pub fn new() -> Self { + let float = Coercion::new_exact(TypeSignatureClass::Float); Self { - signature: Signature::uniform( - 2, - vec![Float32, Float64], + signature: Signature::coercible( + vec![float.clone(), float], Volatility::Immutable, ), } From 7b482aa6c5787def031db9af2f90dae5c1644c7a Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Fri, 2 Jan 2026 16:53:17 +0530 Subject: [PATCH 3/9] fix sql test error --- datafusion/functions/src/math/nanvl.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/datafusion/functions/src/math/nanvl.rs b/datafusion/functions/src/math/nanvl.rs index a6fc2c05ae6db..1ef705c17562c 100644 --- a/datafusion/functions/src/math/nanvl.rs +++ b/datafusion/functions/src/math/nanvl.rs @@ -23,7 +23,8 @@ use crate::utils::make_scalar_function; use arrow::array::{ArrayRef, AsArray, Float32Array, Float64Array}; use arrow::datatypes::DataType::{Float32, Float64}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; -use datafusion_common::{DataFusionError, Result, exec_err}; +use datafusion_common::types::NativeType; +use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err}; use datafusion_expr::{Coercion, TypeSignatureClass}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, @@ -66,7 +67,11 @@ impl Default for NanvlFunc { impl NanvlFunc { pub fn new() -> Self { - let float = Coercion::new_exact(TypeSignatureClass::Float); + let float = Coercion::new_implicit( + TypeSignatureClass::Float, + vec![TypeSignatureClass::Numeric], + NativeType::Float64, + ); Self { signature: Signature::coercible( vec![float.clone(), float], @@ -97,6 +102,11 @@ impl ScalarUDFImpl for NanvlFunc { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + if args.arg_fields.iter().any(|f| f.data_type().is_null()) { + return ColumnarValue::Scalar(ScalarValue::Null) + .cast_to(args.return_type(), None); + } + make_scalar_function(nanvl, vec![])(&args.args) } From f2b196e615c6ddce8b1617dcfe37fb839bb42663 Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Fri, 2 Jan 2026 17:14:17 +0530 Subject: [PATCH 4/9] fix panic --- datafusion/functions/src/math/nanvl.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/datafusion/functions/src/math/nanvl.rs b/datafusion/functions/src/math/nanvl.rs index 1ef705c17562c..a423af0aab4aa 100644 --- a/datafusion/functions/src/math/nanvl.rs +++ b/datafusion/functions/src/math/nanvl.rs @@ -107,7 +107,23 @@ impl ScalarUDFImpl for NanvlFunc { .cast_to(args.return_type(), None); } - make_scalar_function(nanvl, vec![])(&args.args) + let target_type = if args + .args + .iter() + .all(|arg| matches!(arg.data_type(), Float32)) + { + Float32 + } else { + Float64 + }; + + let casted_args = args + .args + .iter() + .map(|arg| arg.cast_to(&target_type, None)) + .collect::>>()?; + + make_scalar_function(nanvl, vec![])(&casted_args) } fn documentation(&self) -> Option<&Documentation> { From bd86ee5e6822ad4c7ddccdf9c113aad41291e77d Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Fri, 2 Jan 2026 17:40:37 +0530 Subject: [PATCH 5/9] fix issues with nanvl --- datafusion/functions/src/math/nanvl.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/datafusion/functions/src/math/nanvl.rs b/datafusion/functions/src/math/nanvl.rs index a423af0aab4aa..3df37481d2b75 100644 --- a/datafusion/functions/src/math/nanvl.rs +++ b/datafusion/functions/src/math/nanvl.rs @@ -107,20 +107,17 @@ impl ScalarUDFImpl for NanvlFunc { .cast_to(args.return_type(), None); } - let target_type = if args - .args - .iter() - .all(|arg| matches!(arg.data_type(), Float32)) - { - Float32 - } else { - Float64 - }; + let target_type = args.return_type(); + if !matches!(target_type, Float32 | Float64) { + return exec_err!( + "Unsupported return type {target_type:?} for function nanvl" + ); + } let casted_args = args .args .iter() - .map(|arg| arg.cast_to(&target_type, None)) + .map(|arg| arg.cast_to(target_type, None)) .collect::>>()?; make_scalar_function(nanvl, vec![])(&casted_args) From c6a65fe0a0e3522256f814c28f2f972c70bb592b Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Fri, 2 Jan 2026 18:11:20 +0530 Subject: [PATCH 6/9] switch to new_implicit --- datafusion/functions/src/math/iszero.rs | 12 ++++++++---- datafusion/functions/src/math/nans.rs | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/datafusion/functions/src/math/iszero.rs b/datafusion/functions/src/math/iszero.rs index ae4fbe9c64fdb..fde25d1878c14 100644 --- a/datafusion/functions/src/math/iszero.rs +++ b/datafusion/functions/src/math/iszero.rs @@ -22,6 +22,7 @@ use arrow::array::{ArrayRef, AsArray, BooleanArray}; use arrow::datatypes::DataType::{Boolean, Float32, Float64}; use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use datafusion_common::types::NativeType; use datafusion_common::{Result, ScalarValue, exec_err}; use datafusion_expr::{Coercion, TypeSignatureClass}; use datafusion_expr::{ @@ -59,11 +60,14 @@ impl Default for IsZeroFunc { impl IsZeroFunc { pub fn new() -> Self { + // Accept any numeric type and coerce to float + let float = Coercion::new_implicit( + TypeSignatureClass::Float, + vec![TypeSignatureClass::Numeric], + NativeType::Float64, + ); Self { - signature: Signature::coercible( - vec![Coercion::new_exact(TypeSignatureClass::Float)], - Volatility::Immutable, - ), + signature: Signature::coercible(vec![float], Volatility::Immutable), } } } diff --git a/datafusion/functions/src/math/nans.rs b/datafusion/functions/src/math/nans.rs index 1cf7bee5e5a0d..6c82bc4dab587 100644 --- a/datafusion/functions/src/math/nans.rs +++ b/datafusion/functions/src/math/nans.rs @@ -18,6 +18,7 @@ //! Math function: `isnan()`. use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use datafusion_common::types::NativeType; use datafusion_common::{Result, ScalarValue, exec_err}; use datafusion_expr::{Coercion, ColumnarValue, ScalarFunctionArgs, TypeSignatureClass}; @@ -54,11 +55,14 @@ impl Default for IsNanFunc { impl IsNanFunc { pub fn new() -> Self { + // Accept any numeric type and coerce to float + let float = Coercion::new_implicit( + TypeSignatureClass::Float, + vec![TypeSignatureClass::Numeric], + NativeType::Float64, + ); Self { - signature: Signature::coercible( - vec![Coercion::new_exact(TypeSignatureClass::Float)], - Volatility::Immutable, - ), + signature: Signature::coercible(vec![float], Volatility::Immutable), } } } From 8dd92ebd4d0d41fcde269ba1b900db5bcb82cbdb Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Sun, 4 Jan 2026 21:36:46 +0530 Subject: [PATCH 7/9] add float16 support --- datafusion/functions/src/math/iszero.rs | 9 +++++++-- datafusion/functions/src/math/nans.rs | 7 ++++++- datafusion/functions/src/math/nanvl.rs | 22 ++++++++++++++++++---- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/datafusion/functions/src/math/iszero.rs b/datafusion/functions/src/math/iszero.rs index fde25d1878c14..97d06c2c4d9d2 100644 --- a/datafusion/functions/src/math/iszero.rs +++ b/datafusion/functions/src/math/iszero.rs @@ -19,8 +19,8 @@ use std::any::Any; use std::sync::Arc; use arrow::array::{ArrayRef, AsArray, BooleanArray}; -use arrow::datatypes::DataType::{Boolean, Float32, Float64}; -use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use arrow::datatypes::DataType::{Boolean, Float16, Float32, Float64}; +use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type}; use datafusion_common::types::NativeType; use datafusion_common::{Result, ScalarValue, exec_err}; @@ -115,6 +115,11 @@ fn iszero(args: &[ArrayRef]) -> Result { |x| x == 0.0, )) as ArrayRef), + Float16 => Ok(Arc::new(BooleanArray::from_unary( + args[0].as_primitive::(), + |x| x.to_f32() == 0.0, + )) as ArrayRef), + other => exec_err!("Unsupported data type {other:?} for function iszero"), } } diff --git a/datafusion/functions/src/math/nans.rs b/datafusion/functions/src/math/nans.rs index 6c82bc4dab587..03f246c28be19 100644 --- a/datafusion/functions/src/math/nans.rs +++ b/datafusion/functions/src/math/nans.rs @@ -17,7 +17,7 @@ //! Math function: `isnan()`. -use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type}; use datafusion_common::types::NativeType; use datafusion_common::{Result, ScalarValue, exec_err}; use datafusion_expr::{Coercion, ColumnarValue, ScalarFunctionArgs, TypeSignatureClass}; @@ -101,6 +101,11 @@ impl ScalarUDFImpl for IsNanFunc { args[0].as_primitive::(), f32::is_nan, )) as ArrayRef, + + DataType::Float16 => Arc::new(BooleanArray::from_unary( + args[0].as_primitive::(), + |x| x.is_nan(), + )) as ArrayRef, other => { return exec_err!( "Unsupported data type {other:?} for function {}", diff --git a/datafusion/functions/src/math/nanvl.rs b/datafusion/functions/src/math/nanvl.rs index 3df37481d2b75..907e404b73aee 100644 --- a/datafusion/functions/src/math/nanvl.rs +++ b/datafusion/functions/src/math/nanvl.rs @@ -20,9 +20,9 @@ use std::sync::Arc; use crate::utils::make_scalar_function; -use arrow::array::{ArrayRef, AsArray, Float32Array, Float64Array}; -use arrow::datatypes::DataType::{Float32, Float64}; -use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use arrow::array::{ArrayRef, AsArray, Float16Array, Float32Array, Float64Array}; +use arrow::datatypes::DataType::{Float16, Float32, Float64}; +use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type}; use datafusion_common::types::NativeType; use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err}; use datafusion_expr::{Coercion, TypeSignatureClass}; @@ -96,6 +96,7 @@ impl ScalarUDFImpl for NanvlFunc { fn return_type(&self, arg_types: &[DataType]) -> Result { match &arg_types[0] { + Float16 => Ok(Float16), Float32 => Ok(Float32), _ => Ok(Float64), } @@ -108,7 +109,7 @@ impl ScalarUDFImpl for NanvlFunc { } let target_type = args.return_type(); - if !matches!(target_type, Float32 | Float64) { + if !matches!(target_type, Float16 | Float32 | Float64) { return exec_err!( "Unsupported return type {target_type:?} for function nanvl" ); @@ -153,6 +154,19 @@ fn nanvl(args: &[ArrayRef]) -> Result { .map(|res| Arc::new(res) as _) .map_err(DataFusionError::from) } + Float16 => { + let compute_nanvl = + |x: ::Native, + y: ::Native| { + if x.is_nan() { y } else { x } + }; + + let x = args[0].as_primitive() as &Float16Array; + let y = args[1].as_primitive() as &Float16Array; + arrow::compute::binary::<_, _, _, Float16Type>(x, y, compute_nanvl) + .map(|res| Arc::new(res) as _) + .map_err(DataFusionError::from) + } other => exec_err!("Unsupported data type {other:?} for function nanvl"), } } From 1b1dfa60fedd14e1ec7542f22351ab4fc3ec1b14 Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Mon, 5 Jan 2026 19:55:26 +0530 Subject: [PATCH 8/9] suggestion from jefffrey Co-authored-by: Jeffrey Vo --- datafusion/functions/src/math/iszero.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/functions/src/math/iszero.rs b/datafusion/functions/src/math/iszero.rs index 97d06c2c4d9d2..5dbb56b9ed67d 100644 --- a/datafusion/functions/src/math/iszero.rs +++ b/datafusion/functions/src/math/iszero.rs @@ -117,7 +117,7 @@ fn iszero(args: &[ArrayRef]) -> Result { Float16 => Ok(Arc::new(BooleanArray::from_unary( args[0].as_primitive::(), - |x| x.to_f32() == 0.0, + |x| x.is_zero(), )) as ArrayRef), other => exec_err!("Unsupported data type {other:?} for function iszero"), From 28a242ba61d3ea39b66fa077e603f246365d1c33 Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Tue, 6 Jan 2026 10:37:44 +0530 Subject: [PATCH 9/9] reverted coercible --- datafusion/functions/src/math/iszero.rs | 2 +- datafusion/functions/src/math/nanvl.rs | 38 ++++++------------------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/datafusion/functions/src/math/iszero.rs b/datafusion/functions/src/math/iszero.rs index 5dbb56b9ed67d..ba4afc5622eb3 100644 --- a/datafusion/functions/src/math/iszero.rs +++ b/datafusion/functions/src/math/iszero.rs @@ -18,7 +18,7 @@ use std::any::Any; use std::sync::Arc; -use arrow::array::{ArrayRef, AsArray, BooleanArray}; +use arrow::array::{ArrayRef, ArrowNativeTypeOp, AsArray, BooleanArray}; use arrow::datatypes::DataType::{Boolean, Float16, Float32, Float64}; use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type}; diff --git a/datafusion/functions/src/math/nanvl.rs b/datafusion/functions/src/math/nanvl.rs index 907e404b73aee..6daf476e250d3 100644 --- a/datafusion/functions/src/math/nanvl.rs +++ b/datafusion/functions/src/math/nanvl.rs @@ -23,9 +23,8 @@ use crate::utils::make_scalar_function; use arrow::array::{ArrayRef, AsArray, Float16Array, Float32Array, Float64Array}; use arrow::datatypes::DataType::{Float16, Float32, Float64}; use arrow::datatypes::{DataType, Float16Type, Float32Type, Float64Type}; -use datafusion_common::types::NativeType; -use datafusion_common::{DataFusionError, Result, ScalarValue, exec_err}; -use datafusion_expr::{Coercion, TypeSignatureClass}; +use datafusion_common::{DataFusionError, Result, exec_err}; +use datafusion_expr::TypeSignature::Exact; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -67,14 +66,13 @@ impl Default for NanvlFunc { impl NanvlFunc { pub fn new() -> Self { - let float = Coercion::new_implicit( - TypeSignatureClass::Float, - vec![TypeSignatureClass::Numeric], - NativeType::Float64, - ); Self { - signature: Signature::coercible( - vec![float.clone(), float], + signature: Signature::one_of( + vec![ + Exact(vec![Float16, Float16]), + Exact(vec![Float32, Float32]), + Exact(vec![Float64, Float64]), + ], Volatility::Immutable, ), } @@ -103,25 +101,7 @@ impl ScalarUDFImpl for NanvlFunc { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - if args.arg_fields.iter().any(|f| f.data_type().is_null()) { - return ColumnarValue::Scalar(ScalarValue::Null) - .cast_to(args.return_type(), None); - } - - let target_type = args.return_type(); - if !matches!(target_type, Float16 | Float32 | Float64) { - return exec_err!( - "Unsupported return type {target_type:?} for function nanvl" - ); - } - - let casted_args = args - .args - .iter() - .map(|arg| arg.cast_to(target_type, None)) - .collect::>>()?; - - make_scalar_function(nanvl, vec![])(&casted_args) + make_scalar_function(nanvl, vec![])(&args.args) } fn documentation(&self) -> Option<&Documentation> {