Merge commit '2eb38bd5e0aeac69dd7b032386729379a160883e' into chunchun…

…/update-df-may-week-4

Cargo.toml

@@ -59,6 +59,9 @@ version = "38.0.0"
 # for the inherited dependency but cannot do the reverse (override from true to false).
 #
 # See for more detaiils: https://github.com/rust-lang/cargo/issues/11329
+ahash = { version = "0.8", default-features = false, features = [
+    "runtime-rng",
+] }
 arrow = { version = "51.0.0", features = ["prettyprint"] }
 arrow-array = { version = "51.0.0", default-features = false, features = ["chrono-tz"] }
 arrow-buffer = { version = "51.0.0", default-features = false }

datafusion-cli/src/main.rs

@@ -304,7 +304,7 @@ enum ByteUnit {
 }
 
 impl ByteUnit {
-    fn multiplier(&self) -> usize {
+    fn multiplier(&self) -> u64 {
         match self {
             ByteUnit::Byte => 1,
             ByteUnit::KiB => 1 << 10,
@@ -349,8 +349,12 @@ fn extract_memory_pool_size(size: &str) -> Result<usize, String> {
         let unit = byte_suffixes()
             .get(suffix)
             .ok_or_else(|| format!("Invalid memory pool size '{}'", size))?;
+        let memory_pool_size = usize::try_from(unit.multiplier())
+            .ok()
+            .and_then(|multiplier| num.checked_mul(multiplier))
+            .ok_or_else(|| format!("Memory pool size '{}' is too large", size))?;
 
-        Ok(num * unit.multiplier())
+        Ok(memory_pool_size)
     } else {
         Err(format!("Invalid memory pool size '{}'", size))
     }

datafusion-examples/examples/advanced_udf.rs

@@ -181,7 +181,7 @@ impl ScalarUDFImpl for PowUdf {
         &self.aliases
     }
 
-    fn monotonicity(&self, input: &[ExprProperties]) -> Result<SortProperties> {
+    fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {
         // The POW function preserves the order of its argument.
         Ok(input[0].sort_properties)
     }

datafusion-examples/examples/function_factory.rs

@@ -157,7 +157,7 @@ impl ScalarUDFImpl for ScalarFunctionWrapper {
         &[]
     }
 
-    fn monotonicity(&self, _input: &[ExprProperties]) -> Result<SortProperties> {
+    fn output_ordering(&self, _input: &[ExprProperties]) -> Result<SortProperties> {
         Ok(SortProperties::Unordered)
     }
 }

datafusion/common/Cargo.toml

@@ -41,9 +41,7 @@ backtrace = []
 pyarrow = ["pyo3", "arrow/pyarrow", "parquet"]
 
 [dependencies]
-ahash = { version = "0.8", default-features = false, features = [
-    "runtime-rng",
-] }
+ahash = { workspace = true }
 apache-avro = { version = "0.16", default-features = false, features = [
     "bzip",
     "snappy",

datafusion/core/Cargo.toml

@@ -77,7 +77,7 @@ unicode_expressions = [
 ]
 
 [dependencies]
-ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
+ahash = { workspace = true }
 apache-avro = { version = "0.16", optional = true }
 arrow = { workspace = true }
 arrow-array = { workspace = true }

datafusion/core/src/physical_planner.rs

@@ -2819,7 +2819,11 @@ mod tests {
             write!(f, "NoOp")
         }
 
-        fn from_template(&self, _exprs: &[Expr], _inputs: &[LogicalPlan]) -> Self {
+        fn with_exprs_and_inputs(
+            &self,
+            _exprs: Vec<Expr>,
+            _inputs: Vec<LogicalPlan>,
+        ) -> Result<Self> {
             unimplemented!("NoOp");
         }
     }

datafusion/core/tests/user_defined/user_defined_plan.rs

@@ -365,14 +365,18 @@ impl UserDefinedLogicalNodeCore for TopKPlanNode {
         write!(f, "TopK: k={}", self.k)
     }
 
-    fn from_template(&self, exprs: &[Expr], inputs: &[LogicalPlan]) -> Self {
+    fn with_exprs_and_inputs(
+        &self,
+        mut exprs: Vec<Expr>,
+        mut inputs: Vec<LogicalPlan>,
+    ) -> Result<Self> {
         assert_eq!(inputs.len(), 1, "input size inconsistent");
         assert_eq!(exprs.len(), 1, "expression size inconsistent");
-        Self {
+        Ok(Self {
             k: self.k,
-            input: inputs[0].clone(),
-            expr: exprs[0].clone(),
-        }
+            input: inputs.swap_remove(0),
+            expr: exprs.swap_remove(0),
+        })
     }
 }
 

datafusion/expr/Cargo.toml

@@ -38,9 +38,7 @@ path = "src/lib.rs"
 [features]
 
 [dependencies]
-ahash = { version = "0.8", default-features = false, features = [
-    "runtime-rng",
-] }
+ahash = { workspace = true }
 arrow = { workspace = true }
 arrow-array = { workspace = true }
 chrono = { workspace = true }

datafusion/expr/src/logical_plan/extension.rs

@@ -215,7 +215,7 @@ impl Eq for dyn UserDefinedLogicalNode {}
 /// [user_defined_plan.rs](../../tests/user_defined_plan.rs) for an
 /// example of how to use this extension API.
 pub trait UserDefinedLogicalNodeCore:
-    fmt::Debug + Eq + Hash + Send + Sync + 'static
+    fmt::Debug + Eq + Hash + Sized + Send + Sync + 'static
 {
     /// Return the plan's name.
     fn name(&self) -> &str;
@@ -248,23 +248,27 @@ pub trait UserDefinedLogicalNodeCore:
     /// For example: `TopK: k=10`
     fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result;
 
-    /// Create a new `ExtensionPlanNode` with the specified children
+    #[deprecated(since = "39.0.0", note = "use with_exprs_and_inputs instead")]
+    #[allow(clippy::wrong_self_convention)]
+    fn from_template(&self, exprs: &[Expr], inputs: &[LogicalPlan]) -> Self {
+        self.with_exprs_and_inputs(exprs.to_vec(), inputs.to_vec())
+            .unwrap()
+    }
+
+    /// Create a new `UserDefinedLogicalNode` with the specified children
     /// and expressions. This function is used during optimization
     /// when the plan is being rewritten and a new instance of the
-    /// `ExtensionPlanNode` must be created.
+    /// `UserDefinedLogicalNode` must be created.
     ///
     /// Note that exprs and inputs are in the same order as the result
     /// of self.inputs and self.exprs.
     ///
-    /// So, `self.from_template(exprs, ..).expressions() == exprs
-    //
-    // TODO(clippy): This should probably be renamed to use a `with_*` prefix. Something
-    // like `with_template`, or `with_exprs_and_inputs`.
-    //
-    // Also, I think `ExtensionPlanNode` has been renamed to `UserDefinedLogicalNode`
-    // but the doc comments have not been updated.
-    #[allow(clippy::wrong_self_convention)]
-    fn from_template(&self, exprs: &[Expr], inputs: &[LogicalPlan]) -> Self;
+    /// So, `self.with_exprs_and_inputs(exprs, ..).expressions() == exprs
+    fn with_exprs_and_inputs(
+        &self,
+        exprs: Vec<Expr>,
+        inputs: Vec<LogicalPlan>,
+    ) -> Result<Self>;
 
     /// Returns the necessary input columns for this node required to compute
     /// the columns in the output schema
@@ -321,7 +325,7 @@ impl<T: UserDefinedLogicalNodeCore> UserDefinedLogicalNode for T {
         exprs: Vec<Expr>,
         inputs: Vec<LogicalPlan>,
     ) -> Result<Arc<dyn UserDefinedLogicalNode>> {
-        Ok(Arc::new(self.from_template(&exprs, &inputs)))
+        Ok(Arc::new(self.with_exprs_and_inputs(exprs, inputs)?))
     }
 
     fn necessary_children_exprs(

datafusion/expr/src/udf.rs

@@ -256,8 +256,8 @@ impl ScalarUDF {
 
     /// Calculates the [`SortProperties`] of this function based on its
     /// children's properties.
-    pub fn monotonicity(&self, inputs: &[ExprProperties]) -> Result<SortProperties> {
-        self.inner.monotonicity(inputs)
+    pub fn output_ordering(&self, inputs: &[ExprProperties]) -> Result<SortProperties> {
+        self.inner.output_ordering(inputs)
     }
 
     /// See [`ScalarUDFImpl::coerce_types`] for more details.
@@ -516,7 +516,7 @@ pub trait ScalarUDFImpl: Debug + Send + Sync {
 
     /// Calculates the [`SortProperties`] of this function based on its
     /// children's properties.
-    fn monotonicity(&self, _inputs: &[ExprProperties]) -> Result<SortProperties> {
+    fn output_ordering(&self, _inputs: &[ExprProperties]) -> Result<SortProperties> {
         Ok(SortProperties::Unordered)
     }
 

datafusion/functions/src/datetime/date_bin.rs

@@ -147,7 +147,7 @@ impl ScalarUDFImpl for DateBinFunc {
         }
     }
 
-    fn monotonicity(&self, input: &[ExprProperties]) -> Result<SortProperties> {
+    fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {
         // The DATE_BIN function preserves the order of its second argument.
         let step = &input[0];
         let date_value = &input[1];

datafusion/functions/src/datetime/date_trunc.rs

@@ -204,7 +204,7 @@ impl ScalarUDFImpl for DateTruncFunc {
         &self.aliases
     }
 
-    fn monotonicity(&self, input: &[ExprProperties]) -> Result<SortProperties> {
+    fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {
         // The DATE_TRUNC function preserves the order of its second argument.
         let precision = &input[0];
         let date_value = &input[1];

datafusion/functions/src/macros.rs

@@ -154,9 +154,9 @@ macro_rules! downcast_arg {
 /// $GNAME: a singleton instance of the UDF
 /// $NAME: the name of the function
 /// $UNARY_FUNC: the unary function to apply to the argument
-/// $MONOTONIC_FUNC: the monotonicity of the function
+/// $OUTPUT_ORDERING: the output ordering calculation method of the function
 macro_rules! make_math_unary_udf {
-    ($UDF:ident, $GNAME:ident, $NAME:ident, $UNARY_FUNC:ident, $MONOTONICITY:expr) => {
+    ($UDF:ident, $GNAME:ident, $NAME:ident, $UNARY_FUNC:ident, $OUTPUT_ORDERING:expr) => {
         make_udf_function!($NAME::$UDF, $GNAME, $NAME);
 
         mod $NAME {
@@ -209,11 +209,11 @@ macro_rules! make_math_unary_udf {
                     }
                 }
 
-                fn monotonicity(
+                fn output_ordering(
                     &self,
                     input: &[ExprProperties],
                 ) -> Result<SortProperties> {
-                    $MONOTONICITY(input)
+                    $OUTPUT_ORDERING(input)
                 }
 
                 fn invoke(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> {
@@ -261,9 +261,9 @@ macro_rules! make_math_unary_udf {
 /// $GNAME: a singleton instance of the UDF
 /// $NAME: the name of the function
 /// $BINARY_FUNC: the binary function to apply to the argument
-/// $MONOTONIC_FUNC: the monotonicity of the function
+/// $OUTPUT_ORDERING: the output ordering calculation method of the function
 macro_rules! make_math_binary_udf {
-    ($UDF:ident, $GNAME:ident, $NAME:ident, $BINARY_FUNC:ident, $MONOTONICITY:expr) => {
+    ($UDF:ident, $GNAME:ident, $NAME:ident, $BINARY_FUNC:ident, $OUTPUT_ORDERING:expr) => {
         make_udf_function!($NAME::$UDF, $GNAME, $NAME);
 
         mod $NAME {
@@ -319,11 +319,11 @@ macro_rules! make_math_binary_udf {
                     }
                 }
 
-                fn monotonicity(
+                fn output_ordering(
                     &self,
                     input: &[ExprProperties],
                 ) -> Result<SortProperties> {
-                    $MONOTONICITY(input)
+                    $OUTPUT_ORDERING(input)
                 }
 
                 fn invoke(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> {

datafusion/functions/src/math/abs.rs

@@ -170,7 +170,7 @@ impl ScalarUDFImpl for AbsFunc {
         abs_fun(&args).map(ColumnarValue::Array)
     }
 
-    fn monotonicity(&self, input: &[ExprProperties]) -> Result<SortProperties> {
+    fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {
         // Non-decreasing for x ≥ 0 and symmetrically non-increasing for x ≤ 0.
         let arg = &input[0];
         let range = &arg.range;

datafusion/functions/src/math/log.rs

@@ -81,7 +81,7 @@ impl ScalarUDFImpl for LogFunc {
         }
     }
 
-    fn monotonicity(&self, input: &[ExprProperties]) -> Result<SortProperties> {
+    fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {
         match (input[0].sort_properties, input[1].sort_properties) {
             (first @ SortProperties::Ordered(value), SortProperties::Ordered(base))
                 if !value.descending && base.descending

datafusion/functions/src/math/mod.rs

@@ -41,36 +41,36 @@ pub mod trunc;
 
 // Create UDFs
 make_udf_function!(abs::AbsFunc, ABS, abs);
-make_math_unary_udf!(AcosFunc, ACOS, acos, acos, super::acos_monotonicity);
-make_math_unary_udf!(AcoshFunc, ACOSH, acosh, acosh, super::acosh_monotonicity);
-make_math_unary_udf!(AsinFunc, ASIN, asin, asin, super::asin_monotonicity);
-make_math_unary_udf!(AsinhFunc, ASINH, asinh, asinh, super::asinh_monotonicity);
-make_math_unary_udf!(AtanFunc, ATAN, atan, atan, super::atan_monotonicity);
-make_math_unary_udf!(AtanhFunc, ATANH, atanh, atanh, super::atanh_monotonicity);
-make_math_binary_udf!(Atan2, ATAN2, atan2, atan2, super::atan2_monotonicity);
-make_math_unary_udf!(CbrtFunc, CBRT, cbrt, cbrt, super::cbrt_monotonicity);
-make_math_unary_udf!(CeilFunc, CEIL, ceil, ceil, super::ceil_monotonicity);
-make_math_unary_udf!(CosFunc, COS, cos, cos, super::cos_monotonicity);
-make_math_unary_udf!(CoshFunc, COSH, cosh, cosh, super::cosh_monotonicity);
+make_math_unary_udf!(AcosFunc, ACOS, acos, acos, super::acos_order);
+make_math_unary_udf!(AcoshFunc, ACOSH, acosh, acosh, super::acosh_order);
+make_math_unary_udf!(AsinFunc, ASIN, asin, asin, super::asin_order);
+make_math_unary_udf!(AsinhFunc, ASINH, asinh, asinh, super::asinh_order);
+make_math_unary_udf!(AtanFunc, ATAN, atan, atan, super::atan_order);
+make_math_unary_udf!(AtanhFunc, ATANH, atanh, atanh, super::atanh_order);
+make_math_binary_udf!(Atan2, ATAN2, atan2, atan2, super::atan2_order);
+make_math_unary_udf!(CbrtFunc, CBRT, cbrt, cbrt, super::cbrt_order);
+make_math_unary_udf!(CeilFunc, CEIL, ceil, ceil, super::ceil_order);
+make_math_unary_udf!(CosFunc, COS, cos, cos, super::cos_order);
+make_math_unary_udf!(CoshFunc, COSH, cosh, cosh, super::cosh_order);
 make_udf_function!(cot::CotFunc, COT, cot);
 make_math_unary_udf!(
     DegreesFunc,
     DEGREES,
     degrees,
     to_degrees,
-    super::degrees_monotonicity
+    super::degrees_order
 );
-make_math_unary_udf!(ExpFunc, EXP, exp, exp, super::exp_monotonicity);
+make_math_unary_udf!(ExpFunc, EXP, exp, exp, super::exp_order);
 make_udf_function!(factorial::FactorialFunc, FACTORIAL, factorial);
-make_math_unary_udf!(FloorFunc, FLOOR, floor, floor, super::floor_monotonicity);
+make_math_unary_udf!(FloorFunc, FLOOR, floor, floor, super::floor_order);
 make_udf_function!(log::LogFunc, LOG, log);
 make_udf_function!(gcd::GcdFunc, GCD, gcd);
 make_udf_function!(nans::IsNanFunc, ISNAN, isnan);
 make_udf_function!(iszero::IsZeroFunc, ISZERO, iszero);
 make_udf_function!(lcm::LcmFunc, LCM, lcm);
-make_math_unary_udf!(LnFunc, LN, ln, ln, super::ln_monotonicity);
-make_math_unary_udf!(Log2Func, LOG2, log2, log2, super::log2_monotonicity);
-make_math_unary_udf!(Log10Func, LOG10, log10, log10, super::log10_monotonicity);
+make_math_unary_udf!(LnFunc, LN, ln, ln, super::ln_order);
+make_math_unary_udf!(Log2Func, LOG2, log2, log2, super::log2_order);
+make_math_unary_udf!(Log10Func, LOG10, log10, log10, super::log10_order);
 make_udf_function!(nanvl::NanvlFunc, NANVL, nanvl);
 make_udf_function!(pi::PiFunc, PI, pi);
 make_udf_function!(power::PowerFunc, POWER, power);
@@ -79,22 +79,16 @@ make_math_unary_udf!(
     RADIANS,
     radians,
     to_radians,
-    super::radians_monotonicity
+    super::radians_order
 );
 make_udf_function!(random::RandomFunc, RANDOM, random);
 make_udf_function!(round::RoundFunc, ROUND, round);
-make_math_unary_udf!(
-    SignumFunc,
-    SIGNUM,
-    signum,
-    signum,
-    super::signum_monotonicity
-);
-make_math_unary_udf!(SinFunc, SIN, sin, sin, super::sin_monotonicity);
-make_math_unary_udf!(SinhFunc, SINH, sinh, sinh, super::sinh_monotonicity);
-make_math_unary_udf!(SqrtFunc, SQRT, sqrt, sqrt, super::sqrt_monotonicity);
-make_math_unary_udf!(TanFunc, TAN, tan, tan, super::tan_monotonicity);
-make_math_unary_udf!(TanhFunc, TANH, tanh, tanh, super::tanh_monotonicity);
+make_math_unary_udf!(SignumFunc, SIGNUM, signum, signum, super::signum_order);
+make_math_unary_udf!(SinFunc, SIN, sin, sin, super::sin_order);
+make_math_unary_udf!(SinhFunc, SINH, sinh, sinh, super::sinh_order);
+make_math_unary_udf!(SqrtFunc, SQRT, sqrt, sqrt, super::sqrt_order);
+make_math_unary_udf!(TanFunc, TAN, tan, tan, super::tan_order);
+make_math_unary_udf!(TanhFunc, TANH, tanh, tanh, super::tanh_order);
 make_udf_function!(trunc::TruncFunc, TRUNC, trunc);
 
 pub mod expr_fn {