Merge commit 'fe268bcab9e1ff93bf0f9c8ee72a252d0440b614' into chunchun…

…/update-df-apr-week-4

datafusion-cli/src/catalog.rs

@@ -345,10 +345,9 @@ mod tests {
             if cfg!(windows) { "USERPROFILE" } else { "HOME" },
             test_home_path,
         );
-        let input =
-            "~/Code/arrow-datafusion/benchmarks/data/tpch_sf1/part/part-0.parquet";
+        let input = "~/Code/datafusion/benchmarks/data/tpch_sf1/part/part-0.parquet";
         let expected = format!(
-            "{}{}Code{}arrow-datafusion{}benchmarks{}data{}tpch_sf1{}part{}part-0.parquet",
+            "{}{}Code{}datafusion{}benchmarks{}data{}tpch_sf1{}part{}part-0.parquet",
             test_home_path,
             MAIN_SEPARATOR,
             MAIN_SEPARATOR,

datafusion-examples/README.md

@@ -31,7 +31,7 @@ To run the examples, use the `cargo run` command, such as:
 
 ```bash
 git clone https://github.com/apache/datafusion
-cd arrow-datafusion
+cd datafusion
 # Download test data
 git submodule update --init
 

datafusion-examples/examples/flight/flight_sql_server.rs

@@ -73,7 +73,7 @@ macro_rules! status {
 ///
 /// JDBC connection string: "jdbc:arrow-flight-sql://127.0.0.1:50051/"
 ///
-/// Based heavily on Ballista's implementation: https://github.com/apache/arrow-ballista/blob/main/ballista/scheduler/src/flight_sql.rs
+/// Based heavily on Ballista's implementation: https://github.com/apache/datafusion-ballista/blob/main/ballista/scheduler/src/flight_sql.rs
 /// and the example in arrow-rs: https://github.com/apache/arrow-rs/blob/master/arrow-flight/examples/flight_sql_server.rs
 ///
 #[tokio::main]

datafusion/functions-array/src/rewrite.rs

@@ -152,9 +152,8 @@ impl FunctionRewrite for ArrayFunctionRewriter {
                 expr,
                 field: GetFieldAccess::NamedStructField { name },
             }) => {
-                let expr = *expr.clone();
                 let name = Expr::Literal(name);
-                Transformed::yes(get_field(expr, name.clone()))
+                Transformed::yes(get_field(*expr, name))
             }
 
             // expr[idx] ==> array_element(expr, idx)

datafusion/functions/src/math/coalesce.rs → datafusion/functions/src/core/coalesce.rs

@@ -132,11 +132,11 @@ mod test {
 
     use datafusion_expr::ScalarUDFImpl;
 
-    use crate::math;
+    use crate::core;
 
     #[test]
     fn test_coalesce_return_types() {
-        let coalesce = math::coalesce::CoalesceFunc::new();
+        let coalesce = core::coalesce::CoalesceFunc::new();
         let return_type = coalesce
             .return_type(&[DataType::Date32, DataType::Date32])
             .unwrap();

datafusion/functions/src/core/mod.rs

@@ -19,6 +19,7 @@
 
 pub mod arrow_cast;
 pub mod arrowtypeof;
+pub mod coalesce;
 pub mod getfield;
 pub mod named_struct;
 pub mod nullif;
@@ -35,6 +36,7 @@ make_udf_function!(arrowtypeof::ArrowTypeOfFunc, ARROWTYPEOF, arrow_typeof);
 make_udf_function!(r#struct::StructFunc, STRUCT, r#struct);
 make_udf_function!(named_struct::NamedStructFunc, NAMED_STRUCT, named_struct);
 make_udf_function!(getfield::GetFieldFunc, GET_FIELD, get_field);
+make_udf_function!(coalesce::CoalesceFunc, COALESCE, coalesce);
 
 // Export the functions out of this package, both as expr_fn as well as a list of functions
 export_functions!(
@@ -45,5 +47,6 @@ export_functions!(
     (arrow_typeof, arg_1, "Returns the Arrow type of the input expression."),
     (r#struct, args, "Returns a struct with the given arguments"),
     (named_struct, args, "Returns a struct with the given names and arguments pairs"),
-    (get_field, arg_1 arg_2, "Returns the value of the field with the given name from the struct")
+    (get_field, arg_1 arg_2, "Returns the value of the field with the given name from the struct"),
+    (coalesce, args, "Returns `coalesce(args...)`, which evaluates to the value of the first expr which is not NULL")
 );

datafusion/functions/src/math/mod.rs

@@ -21,7 +21,6 @@ use datafusion_expr::ScalarUDF;
 use std::sync::Arc;
 
 pub mod abs;
-pub mod coalesce;
 pub mod cot;
 pub mod factorial;
 pub mod gcd;
@@ -47,7 +46,6 @@ make_math_unary_udf!(AtanhFunc, ATANH, atanh, atanh, Some(vec![Some(true)]));
 make_math_binary_udf!(Atan2, ATAN2, atan2, atan2, Some(vec![Some(true)]));
 make_math_unary_udf!(CbrtFunc, CBRT, cbrt, cbrt, None);
 make_math_unary_udf!(CeilFunc, CEIL, ceil, ceil, Some(vec![Some(true)]));
-make_udf_function!(coalesce::CoalesceFunc, COALESCE, coalesce);
 make_math_unary_udf!(CosFunc, COS, cos, cos, None);
 make_math_unary_udf!(CoshFunc, COSH, cosh, cosh, None);
 make_udf_function!(cot::CotFunc, COT, cot);
@@ -130,11 +128,6 @@ pub mod expr_fn {
         super::ceil().call(vec![num])
     }
 
-    #[doc = "returns `coalesce(args...)`, which evaluates to the value of the first [Expr] which is not NULL"]
-    pub fn coalesce(args: Vec<Expr>) -> Expr {
-        super::coalesce().call(args)
-    }
-
     #[doc = "cosine"]
     pub fn cos(num: Expr) -> Expr {
         super::cos().call(vec![num])
@@ -289,7 +282,6 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
         atanh(),
         cbrt(),
         ceil(),
-        coalesce(),
         cos(),
         cosh(),
         cot(),

datafusion/proto/proto/datafusion.proto

@@ -389,7 +389,6 @@ message LogicalExprNode {
     NegativeNode negative = 13;
     InListNode in_list = 14;
     Wildcard wildcard = 15;
-    ScalarFunctionNode scalar_function = 16;
     TryCastNode try_cast = 17;
 
     // window expressions
@@ -538,153 +537,6 @@ message InListNode {
   bool negated = 3;
 }
 
-enum ScalarFunction {
-  //  0 was Abs before
-  //  The first enum value must be zero for open enums
-  unknown = 0;
-  //  1 was Acos
-  //  2 was Asin
-  // 3 was Atan
-  // 4 was Ascii
-  // 5 was Ceil
-  // 6 was Cos
-  // 7 was Digest
-  // 8 was Exp
-  // 9 was Floor
-  // 10 was Ln
-  // 11 was Log
-  // 12 was Log10
-  // 13 was Log2
-  // 14 was Round
-  // 15 was Signum
-  // 16 was Sin
-  // 17 was Sqrt
-  // Tan = 18;
-  // 19 was Trunc
-  // 20 was Array
-  // RegexpMatch = 21;
-  // 22 was BitLength
-  // 23 was Btrim
-  // 24 was CharacterLength
-  // 25 was Chr
-  // 26 was Concat
-  // 27 was ConcatWithSeparator
-  // 28 was DatePart
-  // 29 was DateTrunc
-  // 30 was InitCap
-  // 31 was Left
-  // 32 was Lpad
-  // 33 was Lower
-  // 34 was Ltrim
-  // 35 was MD5
-  //  36 was NullIf
-  // 37 was OctetLength 
-  // 38 was Random
-  // 39 was RegexpReplace
-  // 40 was Repeat
-  // 41 was Replace
-  // 42 was Reverse
-  // 43 was Right
-  // 44 was Rpad
-  // 45 was Rtrim
-  // 46 was SHA224
-  // 47 was SHA256
-  // 48 was SHA384
-  // 49 was SHA512
-  // 50 was SplitPart
-  // StartsWith = 51;
-  // 52 was Strpos
-  // 53 was Substr
-  // ToHex = 54;
-  // 55 was ToTimestamp
-  // 56 was ToTimestampMillis
-  // 57 was ToTimestampMicros
-  // 58 was ToTimestampSeconds
-  // 59 was Now
-  // 60 was Translate
-  // Trim = 61;
-  // Upper = 62;
-  // 63 was Coalesce
-  // 64 was Power
-  // 65 was StructFun
-  // 66 was FromUnixtime
-  // 67 Atan2
-  // 68 was DateBin
-  // 69 was ArrowTypeof
-  // 70 was CurrentDate
-  // 71 was CurrentTime
-  // 72 was Uuid
-  // 73 was Cbrt
-  // 74 Acosh
-  // 75 was Asinh
-  // 76 was Atanh
-  // 77 was Sinh
-  // 78 was Cosh
-  // Tanh = 79
-  // 80 was Pi
-  // 81 was Degrees
-  // 82 was Radians
-  // 83 was Factorial
-  // 84 was Lcm
-  // 85 was Gcd
-  // 86 was ArrayAppend
-  // 87 was ArrayConcat
-  // 88 was ArrayDims
-  // 89 was ArrayRepeat
-  // 90 was ArrayLength
-  // 91 was ArrayNdims
-  // 92 was ArrayPosition
-  // 93 was ArrayPositions
-  // 94 was ArrayPrepend
-  // 95 was ArrayRemove
-  // 96 was ArrayReplace
-  // 97 was ArrayToString
-  // 98 was Cardinality
-  // 99 was ArrayElement
-  // 100 was ArraySlice
-  // 103 was Cot
-  // 104 was ArrayHas
-  // 105 was ArrayHasAny
-  // 106 was ArrayHasAll
-  // 107 was ArrayRemoveN
-  // 108 was ArrayReplaceN
-  // 109 was ArrayRemoveAll
-  // 110 was ArrayReplaceAll
-  // 111 was Nanvl
-  // 112 was Flatten
-  // 113 was IsNan
-  // 114 was Iszero
-  // 115 was ArrayEmpty
-  // 116 was ArrayPopBack
-  // 117 was StringToArray
-  // 118 was ToTimestampNanos
-  // 119 was ArrayIntersect
-  // 120 was ArrayUnion
-  // 121 was OverLay
-  // 122 is Range
-  // 123 is ArrayExcept
-  // 124 was ArrayPopFront
-  // 125 was Levenshtein
-  // 126 was SubstrIndex
-  // 127 was FindInSet
-  // 128 was ArraySort
-  // 129 was ArrayDistinct
-  // 130 was ArrayResize
-  // 131 was EndsWith
-  // 132 was InStr
-  // 133 was MakeDate
-  // 134 was ArrayReverse
-  // 135 is RegexpLike
-  // 136 was ToChar
-  // 137 was ToDate
-  // 138 was ToUnixtime
-}
-
-message ScalarFunctionNode {
-  ScalarFunction fun = 1;
-  repeated LogicalExprNode args = 2;
-}
-
 enum AggregateFunction {
   MIN = 0;
   MAX = 1;
@@ -1458,7 +1310,6 @@ message PhysicalExprNode {
     PhysicalSortExprNode sort = 10;
     PhysicalNegativeNode negative = 11;
     PhysicalInListNode in_list = 12;
-    PhysicalScalarFunctionNode scalar_function = 13;
     PhysicalTryCastNode try_cast = 14;
 
     // window expressions
@@ -1559,13 +1410,6 @@ message PhysicalCaseNode {
   PhysicalExprNode else_expr = 3;
 }
 
-message PhysicalScalarFunctionNode {
-  string name = 1;
-  ScalarFunction fun = 2;
-  repeated PhysicalExprNode args = 3;
-  ArrowType return_type = 4;
-}
-
 message PhysicalTryCastNode {
   PhysicalExprNode expr = 1;
   ArrowType arrow_type = 2;