save progress

apache · Dec 22, 2023 · 0b53811 · 0b53811
1 parent 1e8e3fe
commit 0b53811
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 7 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -34,9 +34,9 @@ arrow-flight = { version = "49.0.0", features = ["flight-sql-experimental"] }
 arrow-schema = { version = "49.0.0", default-features = false }
 configure_me = { version = "0.4.0" }
 configure_me_codegen = { version = "0.4.4" }
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "0ff5305db6b03128282d31afac69fa727e1fe7c4" }
-datafusion-cli = { git = "https://github.com/apache/arrow-datafusion", rev = "0ff5305db6b03128282d31afac69fa727e1fe7c4" }
-datafusion-proto = { git = "https://github.com/apache/arrow-datafusion", rev = "0ff5305db6b03128282d31afac69fa727e1fe7c4" }
+datafusion = { git = "https://github.com/andygrove/arrow-datafusion", branch = "parquet-writer-props-serde" }
+datafusion-cli = { git = "https://github.com/andygrove/arrow-datafusion", branch = "parquet-writer-props-serde" }
+datafusion-proto = { git = "https://github.com/andygrove/arrow-datafusion", branch = "parquet-writer-props-serde" }
 object_store = "0.8.0"
 sqlparser = "0.40.0"
 tonic = { version = "0.10" }

diff --git a/ballista/client/src/context.rs b/ballista/client/src/context.rs
@@ -470,6 +470,8 @@ mod tests {
     #[cfg(feature = "standalone")]
     use datafusion::datasource::listing::ListingTableUrl;
     #[cfg(feature = "standalone")]
+    use datafusion::parquet::file::properties::WriterProperties;
+    #[cfg(feature = "standalone")]
     use tempfile::TempDir;
 
     #[tokio::test]
@@ -493,8 +495,12 @@ mod tests {
         let tmp_dir = TempDir::new().unwrap();
         let file_path =
             format!("{:?}", tmp_dir.path().join("test_write_parquet.parquet"));
-        df.write_parquet(&file_path, DataFrameWriteOptions::default(), None)
-            .await?;
+        df.write_parquet(
+            &file_path,
+            DataFrameWriteOptions::default(),
+            Some(WriterProperties::default()),
+        )
+        .await?;
         Ok(())
     }
 
@@ -637,7 +643,6 @@ mod tests {
                         target_partitions: x.target_partitions,
                         file_sort_order: vec![],
                         file_type_write_options: None,
-                        single_file: false,
                     };
 
                     let table_paths = listing_table

diff --git a/benchmarks/src/bin/tpch.rs b/benchmarks/src/bin/tpch.rs
@@ -845,7 +845,6 @@ async fn get_table(
         table_partition_cols: vec![],
         file_sort_order: vec![],
         file_type_write_options: None,
-        single_file: false,
     };
 
     let url = ListingTableUrl::parse(path)?;