Move the to_timestamp* functions to datafusion-functions (apache#9388)

* WIP

* WIP

* cargo fmt updates.

* Migrate to_timestamp* functions to new functions crate.

* update to allow wasm run to complete.

* Fix expr_api example

* cargo fmt.

* Update datafusion/core/tests/simplification.rs

Co-authored-by: Andrew Lamb <[email protected]>

* Revert changes to sql in tests to restore to_timestamp(..) calls.

* Rust fmt apparently can't make up it's mind.

* Updates for merge and moving some expression simplifier tests from optimizer to core/tests.

* Revert test changes

* Update datafusion/core/tests/simplification.rs

* Move cast_column to `ColumnarValue::cast_to`

* Remove datafusion-physical-expr dependency

* Move dependencies to dev

* Fix merge conflict by migrating to_date to new functions module alongside to_timestamp.

* Cargo lock update from merge.

* Add missing licenses.

---------

Co-authored-by: Andrew Lamb <[email protected]>

.github/workflows/rust.yml

@@ -88,6 +88,9 @@ jobs:
       - name: Check function packages (array_expressions)
         run: cargo check --no-default-features --features=array_expressions -p datafusion
 
+      - name: Check function packages (datetime_expressions)
+        run: cargo check --no-default-features --features=datetime_expressions -p datafusion
+
       - name: Check Cargo.lock for datafusion-cli
         run: |
           # If this test fails, try running `cargo update` in the `datafusion-cli` directory

README.md

@@ -78,6 +78,7 @@ Default features:
 - `array_expressions`: functions for working with arrays such as `array_to_string`
 - `compression`: reading files compressed with `xz2`, `bzip2`, `flate2`, and `zstd`
 - `crypto_expressions`: cryptographic functions such as `md5` and `sha256`
+- `datetime_expressions`: date and time functions such as `to_timestamp`
 - `encoding_expressions`: `encode` and `decode` functions
 - `parquet`: support for reading the [Apache Parquet] format
 - `regex_expressions`: regular expression functions, such as `regexp_match`

datafusion-cli/Cargo.toml

@@ -38,6 +38,7 @@ clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "36.0.0", features = [
     "avro",
     "crypto_expressions",
+    "datetime_expressions",
     "encoding_expressions",
     "parquet",
     "regex_expressions",

datafusion-examples/examples/expr_api.rs

@@ -15,8 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::collections::HashMap;
+use std::sync::Arc;
+
 use arrow::array::{BooleanArray, Int32Array};
 use arrow::record_batch::RecordBatch;
+
 use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit};
 use datafusion::common::{DFField, DFSchema};
 use datafusion::error::Result;
@@ -30,8 +34,6 @@ use datafusion_common::{ScalarValue, ToDFSchema};
 use datafusion_expr::expr::BinaryExpr;
 use datafusion_expr::interval_arithmetic::Interval;
 use datafusion_expr::{ColumnarValue, ExprSchemable, Operator};
-use std::collections::HashMap;
-use std::sync::Arc;
 
 /// This example demonstrates the DataFusion [`Expr`] API.
 ///
@@ -113,10 +115,7 @@ fn evaluate_demo() -> Result<()> {
 fn simplify_demo() -> Result<()> {
     // For example, lets say you have has created an expression such
     // ts = to_timestamp("2020-09-08T12:00:00+00:00")
-    let expr = col("ts").eq(call_fn(
-        "to_timestamp",
-        vec![lit("2020-09-08T12:00:00+00:00")],
-    )?);
+    let expr = col("ts").eq(to_timestamp(vec![lit("2020-09-08T12:00:00+00:00")]));
 
     // Naively evaluating such an expression against a large number of
     // rows would involve re-converting "2020-09-08T12:00:00+00:00" to a

datafusion/common/src/format.rs

@@ -15,8 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use arrow::compute::CastOptions;
 use arrow::util::display::{DurationFormat, FormatOptions};
 
 /// The default [`FormatOptions`] to use within DataFusion
 pub const DEFAULT_FORMAT_OPTIONS: FormatOptions<'static> =
     FormatOptions::new().with_duration_format(DurationFormat::Pretty);
+
+/// The default [`CastOptions`] to use within DataFusion
+pub const DEFAULT_CAST_OPTIONS: CastOptions<'static> = CastOptions {
+    safe: false,
+    format_options: DEFAULT_FORMAT_OPTIONS,
+};

datafusion/core/Cargo.toml

@@ -43,9 +43,11 @@ avro = ["apache-avro", "num-traits", "datafusion-common/avro"]
 backtrace = ["datafusion-common/backtrace"]
 compression = ["xz2", "bzip2", "flate2", "zstd", "async-compression", "tokio-util"]
 crypto_expressions = ["datafusion-physical-expr/crypto_expressions", "datafusion-optimizer/crypto_expressions"]
+datetime_expressions = ["datafusion-functions/datetime_expressions"]
 default = [
     "array_expressions",
     "crypto_expressions",
+    "datetime_expressions",
     "encoding_expressions",
     "regex_expressions",
     "unicode_expressions",