From ae069456b7ed4f858e5027bf19fabd2c3bed8fb3 Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Thu, 25 May 2023 14:07:26 -0700
Subject: [PATCH 01/18] Remove the TEST_BROKER constant and allow the
 `KAFKA_BROKER` env override

---
 tests/helpers/mod.rs | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index 2d682d2..c3b9cff 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -27,12 +27,18 @@ use tokio::task::JoinHandle;
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
 
-pub const TEST_BROKER: &str = "0.0.0.0:9092";
 pub const LOCALSTACK_ENDPOINT: &str = "http://0.0.0.0:4566";
 
+/*
+ * Return the KAFKA_BROKER set in the environment or default ot the local machine's port 9092
+ */
+pub fn test_broker() -> String {
+    std::env::var("KAFKA_BROKER").unwrap_or("0.0.0.0:9092".into())
+}
+
 pub async fn create_topic(topic: &str, num_partitions: i32) {
     let admin_client: AdminClient<_> = ClientConfig::new()
-        .set("bootstrap.servers", TEST_BROKER)
+        .set("bootstrap.servers", test_broker())
         .create()
         .unwrap();
 
@@ -46,7 +52,7 @@ pub async fn create_topic(topic: &str, num_partitions: i32) {
 
 pub async fn delete_topic(topic: &str) {
     let admin_client: AdminClient<_> = ClientConfig::new()
-        .set("bootstrap.servers", TEST_BROKER)
+        .set("bootstrap.servers", test_broker())
         .create()
         .unwrap();
 
@@ -58,7 +64,7 @@ pub async fn delete_topic(topic: &str) {
 
 pub fn create_producer() -> FutureProducer {
     ClientConfig::new()
-        .set("bootstrap.servers", TEST_BROKER)
+        .set("bootstrap.servers", test_broker())
         .create()
         .unwrap()
 }

From 470d637bb80de474967e224ada38fe0f19de92fa Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Thu, 25 May 2023 14:12:35 -0700
Subject: [PATCH 02/18] Switch from relying on a singular endpoint constant for
 dynamodb and S3

---
 tests/emails_s3_tests.rs |  2 +-
 tests/helpers/mod.rs     | 12 +++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tests/emails_s3_tests.rs b/tests/emails_s3_tests.rs
index 853d05b..e1cc606 100644
--- a/tests/emails_s3_tests.rs
+++ b/tests/emails_s3_tests.rs
@@ -228,7 +228,7 @@ impl TestScope {
 }
 
 async fn prepare_table(topic: &str) -> String {
-    env::set_var("AWS_ENDPOINT_URL", helpers::LOCALSTACK_ENDPOINT);
+    env::set_var("AWS_ENDPOINT_URL", helpers::test_s3());
     env::set_var("AWS_ACCESS_KEY_ID", "test");
     env::set_var("AWS_SECRET_ACCESS_KEY", "test");
 
diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index c3b9cff..07a2561 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -27,13 +27,19 @@ use tokio::task::JoinHandle;
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
 
-pub const LOCALSTACK_ENDPOINT: &str = "http://0.0.0.0:4566";
-
 /*
  * Return the KAFKA_BROKER set in the environment or default ot the local machine's port 9092
  */
 pub fn test_broker() -> String {
-    std::env::var("KAFKA_BROKER").unwrap_or("0.0.0.0:9092".into())
+    env::var("KAFKA_BROKER").unwrap_or("0.0.0.0:9092".into())
+}
+
+pub fn test_s3() -> String {
+    env::var("S3_ENDPOINT").unwrap_or("http://0.0.0.0:4506".into())
+}
+
+pub fn test_dynamodb() -> String {
+    env::var("DYNAMODB_ENDPOINT").unwrap_or("http://0.0.0.0:4506".into())
 }
 
 pub async fn create_topic(topic: &str, num_partitions: i32) {

From a042ee06d500dcdc2d7f61fa45efb8af6b5458cd Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Fri, 26 May 2023 11:55:59 -0700
Subject: [PATCH 03/18] Add some debug traits for checking tests

---
 src/lib.rs               | 2 ++
 tests/emails_s3_tests.rs | 8 +++-----
 tests/helpers/mod.rs     | 8 ++------
 3 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index cbe277f..529bec8 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -239,6 +239,7 @@ pub enum SchemaSource {
     File(PathBuf),
 }
 
+#[derive(Debug)]
 /// The enum to represent 'auto.offset.reset' options.
 pub enum AutoOffsetReset {
     /// The "earliest" option. Messages will be ingested from the beginning of a partition on reset.
@@ -253,6 +254,7 @@ impl AutoOffsetReset {
 }
 
 /// Options for configuring the behavior of the run loop executed by the [`start_ingest`] function.
+#[derive(Debug)]
 pub struct IngestOptions {
     /// The Kafka broker string to connect to.
     pub kafka_brokers: String,
diff --git a/tests/emails_s3_tests.rs b/tests/emails_s3_tests.rs
index e1cc606..fcb1510 100644
--- a/tests/emails_s3_tests.rs
+++ b/tests/emails_s3_tests.rs
@@ -20,10 +20,8 @@ use rusoto_core::Region;
 use rusoto_s3::{CopyObjectRequest, S3};
 use tokio::task::JoinHandle;
 
-const TEST_S3_ENDPOINT: &str = "http://localhost:4566";
 const TEST_S3_BUCKET: &str = "tests";
 const TEST_APP_ID: &str = "emails_test";
-const TEST_BROKER: &str = "0.0.0.0:9092";
 const TEST_CONSUMER_GROUP_ID: &str = "kafka_delta_ingest_emails";
 const TEST_PARTITIONS: i32 = 4;
 const TEST_TOTAL_MESSAGES: i32 = 200;
@@ -159,7 +157,7 @@ impl TestScope {
 
         IngestOptions {
             transforms,
-            kafka_brokers: TEST_BROKER.to_string(),
+            kafka_brokers: helpers::test_broker(),
             consumer_group_id: TEST_CONSUMER_GROUP_ID.to_string(),
             app_id: TEST_APP_ID.to_string(),
             additional_kafka_settings,
@@ -228,13 +226,13 @@ impl TestScope {
 }
 
 async fn prepare_table(topic: &str) -> String {
-    env::set_var("AWS_ENDPOINT_URL", helpers::test_s3());
+    env::set_var("AWS_ENDPOINT_URL", helpers::test_aws_endpoint());
     env::set_var("AWS_ACCESS_KEY_ID", "test");
     env::set_var("AWS_SECRET_ACCESS_KEY", "test");
 
     let s3 = rusoto_s3::S3Client::new(Region::Custom {
         name: "custom".to_string(),
-        endpoint: TEST_S3_ENDPOINT.to_string(),
+        endpoint: helpers::test_aws_endpoint(),
     });
 
     s3.copy_object(CopyObjectRequest {
diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index 07a2561..e726db9 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -34,12 +34,8 @@ pub fn test_broker() -> String {
     env::var("KAFKA_BROKER").unwrap_or("0.0.0.0:9092".into())
 }
 
-pub fn test_s3() -> String {
-    env::var("S3_ENDPOINT").unwrap_or("http://0.0.0.0:4506".into())
-}
-
-pub fn test_dynamodb() -> String {
-    env::var("DYNAMODB_ENDPOINT").unwrap_or("http://0.0.0.0:4506".into())
+pub fn test_aws_endpoint() -> String {
+    env::var("AWS_ENDPOINT_URL").unwrap_or("http://0.0.0.0:4506".into())
 }
 
 pub async fn create_topic(topic: &str, num_partitions: i32) {

From 9422a372ba81a391142082ae11fb2140cfa04b4e Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Fri, 26 May 2023 12:37:13 -0700
Subject: [PATCH 04/18] Add some environment variables to allow tests to run
 more flexibly

In my environment I do not have a useful or workable docker-compose setup, so
this allows pointing the tests towards and existing Kafka broker(s) and S3
endpoint.

Currently overriding the docker-compose workloads for Azure tests is not
working.
---
 README.adoc                      | 39 ++---------------
 doc/HACKING.adoc                 | 74 ++++++++++++++++++++++++++++++++
 src/lib.rs                       |  2 +-
 tests/buffer_flush_tests.rs      |  3 ++
 tests/emails_azure_blob_tests.rs | 10 ++---
 tests/emails_s3_tests.rs         |  7 ++-
 tests/helpers/mod.rs             |  8 +++-
 7 files changed, 95 insertions(+), 48 deletions(-)
 create mode 100644 doc/HACKING.adoc

diff --git a/README.adoc b/README.adoc
index 4d78a79..463f8ff 100644
--- a/README.adoc
+++ b/README.adoc
@@ -4,9 +4,12 @@ The kafka-delta-ingest project aims to build a highly efficient daemon for
 streaming data through link:https://kafka.apache.org[Apache Kafka] into
 link:https://delta.io[Delta Lake].
 
-This project is currently highly experimental and evolving in tandem with the
+This project is currently in production in a number of organizations and is
+still actively evolving in tandem with the
 link:https://github.com/delta-io/delta-rs[delta-rs] bindings.
 
+To contribute please look at the link:https://github.com/delta-io/kafka-delta-ingest/blob/main/doc/HACKING.adoc[hacking document].
+
 == Features
 
 * Multiple worker processes per stream
@@ -66,37 +69,6 @@ Make sure to provide the additional option:
 
 when invoking the cli command as well.
 
-==== Example Data
-
-A tarball containing 100K line-delimited JSON messages is included in `tests/json/web_requests-100K.json.tar.gz`. Running `./bin/extract-example-json.sh` will unpack this to the expected location.
-
-===== Pretty-printed example from the file
-
-```json
-{
-  "meta": {
-    "producer": {
-      "timestamp": "2021-03-24T15:06:17.321710+00:00"
-    }
-  },
-  "method": "DELETE",
-  "session_id": "7c28bcf9-be26-4d0b-931a-3374ab4bb458",
-  "status": 204,
-  "url": "http://www.youku.com",
-  "uuid": "831c6afa-375c-4988-b248-096f9ed101f8"
-}
-```
-
-After extracting the example data, you'll need to play this onto the web_requests topic of the local Kafka container.
-
-NOTE: URLs sampled for the test data are sourced from Wikipedia's list of most popular websites - https://en.wikipedia.org/wiki/List_of_most_popular_websites.
-
-===== Inspect example output
-
-* List data files - `ls tests/data/web_requests/date=2021-03-24`
-* List delta log files - `ls tests/data/web_requests/_delta_log`
-* Show some parquet data (using link:https://pypi.org/project/parquet-tools/[parquet-tools])
-** `parquet-tools show tests/data/web_requests/date=2021-03-24/<some file written by your example>`
 
 === Using Azure Event Hubs
 
@@ -279,9 +251,6 @@ Use the Azure Portal to browse the file system:
 * Data files: `web_requests/date=2021-03-24`
 * Delta log files: `web_requests/_delta_log`
 
-== Developing
-
-Make sure the docker-compose setup has been ran, and execute `cargo test` to run unit and integration tests.
 
 == Get Involved
 
diff --git a/doc/HACKING.adoc b/doc/HACKING.adoc
new file mode 100644
index 0000000..0c61db7
--- /dev/null
+++ b/doc/HACKING.adoc
@@ -0,0 +1,74 @@
+ifdef::env-github[]
+:tip-caption: :bulb:
+:note-caption: :information_source:
+:important-caption: :heavy_exclamation_mark:
+:caution-caption: :fire:
+:warning-caption: :warning:
+endif::[]
+:toc: macro
+
+= Hacking on kafka-delta-ingest
+
+kafka-delta-ingest is designed to work well with AWS and Azure, which can add
+some complexity to the development environment. This document outlines how to
+work with kafka-delta-ingest locally for new and existing Rust developers.
+
+toc::[]
+
+== Developing
+
+Make sure the docker-compose setup has been ran, and execute `cargo test` to run unit and integration tests.
+
+== Environment Variables
+
+|===
+| Name | Default | Notes
+
+| `KAFKA_BROKERS`
+| `0.0.0.0:9092`
+| A kafka broker string which can be used during integration testing
+
+
+| `AWS_ENDPOINT_URL`
+| `http://0.0.0.0:4056`
+| AWS endpoint URL for something that can provide stub S3 and DynamoDB operations (e.g. Localstack)
+
+| `AWS_S3_BUCKET`
+| `tests`
+| Bucket to use for test data at the given endpoint
+
+|===
+
+
+== Example Data
+
+A tarball containing 100K line-delimited JSON messages is included in `tests/json/web_requests-100K.json.tar.gz`. Running `./bin/extract-example-json.sh` will unpack this to the expected location.
+
+
+.Pretty-printed example from the file
+[source,json]
+----
+{
+  "meta": {
+    "producer": {
+      "timestamp": "2021-03-24T15:06:17.321710+00:00"
+    }
+  },
+  "method": "DELETE",
+  "session_id": "7c28bcf9-be26-4d0b-931a-3374ab4bb458",
+  "status": 204,
+  "url": "http://www.youku.com",
+  "uuid": "831c6afa-375c-4988-b248-096f9ed101f8"
+}
+----
+
+After extracting the example data, you'll need to play this onto the web_requests topic of the local Kafka container.
+
+NOTE: URLs sampled for the test data are sourced from Wikipedia's list of most popular websites - https://en.wikipedia.org/wiki/List_of_most_popular_websites.
+
+=== Inspect example output
+
+* List data files - `ls tests/data/web_requests/date=2021-03-24`
+* List delta log files - `ls tests/data/web_requests/_delta_log`
+* Show some parquet data (using link:https://pypi.org/project/parquet-tools/[parquet-tools])
+** `parquet-tools show tests/data/web_requests/date=2021-03-24/<some file written by your example>`
diff --git a/src/lib.rs b/src/lib.rs
index 529bec8..52c3044 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -305,7 +305,7 @@ pub struct IngestOptions {
 impl Default for IngestOptions {
     fn default() -> Self {
         IngestOptions {
-            kafka_brokers: "localhost:9092".to_string(),
+            kafka_brokers: std::env::var("KAFKA_BROKERS").unwrap_or("localhost:9092".into()),
             consumer_group_id: "kafka_delta_ingest".to_string(),
             app_id: "kafka_delta_ingest".to_string(),
             seek_offsets: None,
diff --git a/tests/buffer_flush_tests.rs b/tests/buffer_flush_tests.rs
index 9ad9254..cbdeb6d 100644
--- a/tests/buffer_flush_tests.rs
+++ b/tests/buffer_flush_tests.rs
@@ -29,6 +29,7 @@ async fn test_flush_when_latency_expires() {
             max_messages_per_batch: 5000,
             // large value - avoid flushing on file size
             min_bytes_per_file: 1000000,
+            kafka_brokers: helpers::test_broker(),
             ..Default::default()
         }),
     )
@@ -76,6 +77,7 @@ async fn test_dont_write_an_empty_buffer() {
             app_id: "dont_write_an_empty_buffer".to_string(),
             // buffer for 5 seconds before flush
             allowed_latency: 5,
+            kafka_brokers: helpers::test_broker(),
             ..Default::default()
         }),
     )
@@ -124,6 +126,7 @@ async fn test_flush_on_size_without_latency_expiration() {
             max_messages_per_batch: 10,
             // tiny buffer size for write flush
             min_bytes_per_file: 20,
+            kafka_brokers: helpers::test_broker(),
             ..Default::default()
         }),
     )
diff --git a/tests/emails_azure_blob_tests.rs b/tests/emails_azure_blob_tests.rs
index 1679598..b71302a 100644
--- a/tests/emails_azure_blob_tests.rs
+++ b/tests/emails_azure_blob_tests.rs
@@ -20,9 +20,7 @@ use uuid::Uuid;
 use kafka_delta_ingest::{start_ingest, IngestOptions};
 use tokio::task::JoinHandle;
 
-const TEST_S3_BUCKET: &str = "tests";
 const TEST_APP_ID: &str = "emails_test";
-const TEST_BROKER: &str = "0.0.0.0:9092";
 const TEST_CONSUMER_GROUP_ID: &str = "kafka_delta_ingest_emails";
 const TEST_PARTITIONS: i32 = 4;
 const TEST_TOTAL_MESSAGES: i32 = 200;
@@ -158,7 +156,7 @@ impl TestScope {
 
         IngestOptions {
             transforms,
-            kafka_brokers: TEST_BROKER.to_string(),
+            kafka_brokers: helpers::test_broker(),
             consumer_group_id: TEST_CONSUMER_GROUP_ID.to_string(),
             app_id: TEST_APP_ID.to_string(),
             additional_kafka_settings,
@@ -227,8 +225,8 @@ impl TestScope {
 }
 
 async fn prepare_table(topic: &str) -> String {
-    let container_client =
-        azure_storage_blobs::prelude::ClientBuilder::emulator().container_client(TEST_S3_BUCKET);
+    let container_client = azure_storage_blobs::prelude::ClientBuilder::emulator()
+        .container_client(helpers::test_s3_bucket());
     let source_blob =
         container_client.blob_client(format!("emails/_delta_log/00000000000000000000.json"));
     let sas_url = {
@@ -253,7 +251,7 @@ async fn prepare_table(topic: &str) -> String {
         .await
         .unwrap();
 
-    format!("az://{}/{}", TEST_S3_BUCKET, topic)
+    format!("az://{}/{}", helpers::test_s3_bucket(), topic)
 }
 
 fn create_partitions_app_ids(num_p: i32) -> Vec<String> {
diff --git a/tests/emails_s3_tests.rs b/tests/emails_s3_tests.rs
index fcb1510..e9ae38d 100644
--- a/tests/emails_s3_tests.rs
+++ b/tests/emails_s3_tests.rs
@@ -20,7 +20,6 @@ use rusoto_core::Region;
 use rusoto_s3::{CopyObjectRequest, S3};
 use tokio::task::JoinHandle;
 
-const TEST_S3_BUCKET: &str = "tests";
 const TEST_APP_ID: &str = "emails_test";
 const TEST_CONSUMER_GROUP_ID: &str = "kafka_delta_ingest_emails";
 const TEST_PARTITIONS: i32 = 4;
@@ -236,18 +235,18 @@ async fn prepare_table(topic: &str) -> String {
     });
 
     s3.copy_object(CopyObjectRequest {
-        bucket: TEST_S3_BUCKET.to_string(),
+        bucket: helpers::test_s3_bucket(),
         key: format!("{}/_delta_log/00000000000000000000.json", topic),
         copy_source: format!(
             "/{}/emails/_delta_log/00000000000000000000.json",
-            TEST_S3_BUCKET
+            helpers::test_s3_bucket(),
         ),
         ..Default::default()
     })
     .await
     .unwrap();
 
-    format!("s3://{}/{}", TEST_S3_BUCKET, topic)
+    format!("s3://{}/{}", helpers::test_s3_bucket(), topic)
 }
 
 fn create_partitions_app_ids(num_p: i32) -> Vec<String> {
diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index e726db9..2c148d6 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -28,16 +28,20 @@ use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
 
 /*
- * Return the KAFKA_BROKER set in the environment or default ot the local machine's port 9092
+ * Return the KAFKA_BROKERS set in the environment or default ot the local machine's port 9092
  */
 pub fn test_broker() -> String {
-    env::var("KAFKA_BROKER").unwrap_or("0.0.0.0:9092".into())
+    env::var("KAFKA_BROKERS").unwrap_or("0.0.0.0:9092".into())
 }
 
 pub fn test_aws_endpoint() -> String {
     env::var("AWS_ENDPOINT_URL").unwrap_or("http://0.0.0.0:4506".into())
 }
 
+pub fn test_s3_bucket() -> String {
+    env::var("AWS_S3_BUCKET").unwrap_or("tests".into())
+}
+
 pub async fn create_topic(topic: &str, num_partitions: i32) {
     let admin_client: AdminClient<_> = ClientConfig::new()
         .set("bootstrap.servers", test_broker())

From 243ce793fb106bc5ca26f0b74107072399912c19 Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Fri, 26 May 2023 16:00:17 -0700
Subject: [PATCH 05/18] Structuring the Cargo.toml to put dependencies behind
 `s3` and `azure` flags

This will make development iteration times and binaries much smaller, which is
an improvement
---
 .github/workflows/build.yml |  8 +++++---
 Cargo.toml                  | 34 ++++++++++++++++++++++++----------
 build.rs                    |  5 +++++
 src/lib.rs                  |  4 ++--
 4 files changed, 36 insertions(+), 15 deletions(-)
 create mode 100644 build.rs

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fdeb8f0..370bc22 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -39,12 +39,14 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install minimum stable with clippy and rustfmt
       uses: actions-rs/toolchain@v1
-      with: 
+      with:
         profile: default
         toolchain: stable
         override: true
     - name: Teststack setup
       run: docker-compose up setup
-    - name: Run tests
-      run: cargo test
+    - name: Run s3 feature tests
+      run: cargo test --features s3
+    - name: Run azure feature tests
+      run: cargo test --features azure
 
diff --git a/Cargo.toml b/Cargo.toml
index d6851f7..ce97fc2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,9 +20,6 @@ lazy_static = "1"
 log = "0"
 maplit = "1"
 rdkafka = { version = "0.28", features = ["ssl-vendored"] }
-rusoto_core = { version = "0.46" }
-rusoto_credential = { version = "0.46" }
-rusoto_s3 = { version = "0.46" }
 schema_registry_converter = { version = "3.1.0", features = ["easy", "json", "avro"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
@@ -33,26 +30,43 @@ tokio = { version = "1", features = ["full"] }
 tokio-stream = { version = "0", features = ["fs"] }
 tokio-util = "0.6.3"
 uuid = { version = "0.8", features = ["serde", "v4"] }
+url = "2.3"
+
+deltalake = { version = "0.11.0", features = ["json"], optional = true }
 
-deltalake = { version = "0.11.0", features = ["s3", "azure"] }
-dynamodb_lock = "0.4.3"
+# s3 feature enabled
+dynamodb_lock = { version = "^0.4.3", optional = true }
+rusoto_core = { version = "0.46", optional = true }
+rusoto_credential = { version = "0.46", optional = true }
+rusoto_s3 = { version = "0.46", optional = true }
 
 # sentry
 sentry = { version = "0.23.0", optional = true }
-url = "2.3"
+
+# azure feature enabled, mostly used for tests
+azure_core = { version = "0.10.0", optional = true }
+azure_storage = { version = "0.10.0", optional = true }
+azure_storage_blobs = { version = "0.10.0", optional = true }
 
 [features]
+default = []
 sentry-ext = ["sentry"]
 dynamic-linking = [ "rdkafka/dynamic-linking" ]
+azure = [
+    "deltalake/azure",
+]
+s3 = [
+    "deltalake/s3",
+    "dynamodb_lock",
+    "rusoto_core",
+    "rusoto_credential",
+    "rusoto_s3",
+]
 
 [dev-dependencies]
-deltalake = { version = "0.11.0", features = ["s3", "azure", "json"] }
 utime = "0.3"
 serial_test = "*"
 tempfile = "3"
-azure_core = "0.10.0"
-azure_storage = "0.10.0"
-azure_storage_blobs = "0.10.0"
 time = "0.3.20"
 
 [profile.release]
diff --git a/build.rs b/build.rs
new file mode 100644
index 0000000..73e7796
--- /dev/null
+++ b/build.rs
@@ -0,0 +1,5 @@
+#[cfg(not(any(feature = "s3", feature = "azure")))]
+compile_error!(
+    "Either the \"s3\" or the \"azure\" feature must be built to compile kafka-delta-ingest"
+);
+fn main() {}
diff --git a/src/lib.rs b/src/lib.rs
index 52c3044..ae70fe0 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -239,7 +239,7 @@ pub enum SchemaSource {
     File(PathBuf),
 }
 
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 /// The enum to represent 'auto.offset.reset' options.
 pub enum AutoOffsetReset {
     /// The "earliest" option. Messages will be ingested from the beginning of a partition on reset.
@@ -254,7 +254,7 @@ impl AutoOffsetReset {
 }
 
 /// Options for configuring the behavior of the run loop executed by the [`start_ingest`] function.
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct IngestOptions {
     /// The Kafka broker string to connect to.
     pub kafka_brokers: String,

From f9451a989e1639e2a741397e365bd391110e1e34 Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Fri, 26 May 2023 16:01:28 -0700
Subject: [PATCH 06/18] WIP refactoring TestScope code into the helpers

This code is copied and pasted so we need to have it made more common
---
 tests/emails_azure_blob_tests.rs | 184 +++--------------------------
 tests/emails_s3_tests.rs         | 192 +++++++------------------------
 tests/helpers/mod.rs             | 131 +++++++++++++++++++--
 3 files changed, 179 insertions(+), 328 deletions(-)

diff --git a/tests/emails_azure_blob_tests.rs b/tests/emails_azure_blob_tests.rs
index b71302a..e96b053 100644
--- a/tests/emails_azure_blob_tests.rs
+++ b/tests/emails_azure_blob_tests.rs
@@ -1,57 +1,39 @@
 #[allow(dead_code)]
 mod helpers;
 
-use std::collections::HashMap;
-use std::env;
-use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
 
+#[cfg(feature = "azure")]
 use azure_storage::{prelude::BlobSasPermissions, shared_access_signature::SasProtocol};
-use time::OffsetDateTime;
-use tokio::runtime::Runtime;
-
-use chrono::prelude::*;
-use serde_json::json;
+use kafka_delta_ingest::{start_ingest, IngestOptions};
 use serial_test::serial;
-use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
 
-use kafka_delta_ingest::{start_ingest, IngestOptions};
-use tokio::task::JoinHandle;
-
-const TEST_APP_ID: &str = "emails_test";
-const TEST_CONSUMER_GROUP_ID: &str = "kafka_delta_ingest_emails";
-const TEST_PARTITIONS: i32 = 4;
-const TEST_TOTAL_MESSAGES: i32 = 200;
-
-const WORKER_1: &str = "WORKER-1";
-const WORKER_2: &str = "WORKER-2";
+use helpers::*;
 
+#[cfg(feature = "azure")]
 #[tokio::test(flavor = "multi_thread")]
 #[serial]
 async fn when_both_workers_started_simultaneously_azure() {
     run_emails_s3_tests(false).await;
 }
 
+#[cfg(feature = "azure")]
 #[tokio::test(flavor = "multi_thread")]
 #[serial]
 async fn when_rebalance_happens_azure() {
     run_emails_s3_tests(true).await;
 }
 
-struct TestScope {
-    pub topic: String,
-    pub table: String,
-    pub workers_token: Arc<CancellationToken>,
-    pub runtime: HashMap<&'static str, Runtime>,
-}
-
+#[cfg(feature = "azure")]
 async fn run_emails_s3_tests(initiate_rebalance: bool) {
     helpers::init_logger();
-    let scope = TestScope::new().await;
+    let topic = format!("emails_azure-{}", Uuid::new_v4());
+    let table = prepare_table(&topic).await;
+    let scope = TestScope::new(&topic, &table).await;
 
-    let w1 = scope.create_and_start(WORKER_1).await;
+    let w1 = scope.create_and_start(helpers::WORKER_1).await;
 
     // in order to initiate rebalance we first send messages,
     // ensure that worker 1 consumes some of them and then create worker 2,
@@ -59,9 +41,9 @@ async fn run_emails_s3_tests(initiate_rebalance: bool) {
     let w2 = if initiate_rebalance {
         scope.send_messages(TEST_TOTAL_MESSAGES).await;
         thread::sleep(Duration::from_secs(1));
-        scope.create_and_start(WORKER_2).await
+        scope.create_and_start(helpers::WORKER_2).await
     } else {
-        let w = scope.create_and_start(WORKER_2).await;
+        let w = scope.create_and_start(helpers::WORKER_2).await;
         thread::sleep(Duration::from_secs(4));
         scope.send_messages(TEST_TOTAL_MESSAGES).await;
         w
@@ -69,12 +51,12 @@ async fn run_emails_s3_tests(initiate_rebalance: bool) {
 
     // this will end up with more app_ids than actual,
     // since we're not sure which partitions will get each worker
-    let partitions = create_partitions_app_ids(TEST_PARTITIONS);
+    let partitions = create_partitions_app_ids(helpers::TEST_PARTITIONS);
 
     // wait until the destination table will get every expected message, we check this summing up
     // the each offset of each partition to get the TOTAL_MESSAGES value
     scope
-        .wait_on_total_offset(partitions, TEST_TOTAL_MESSAGES)
+        .wait_on_total_offset(partitions, helpers::TEST_TOTAL_MESSAGES)
         .await;
 
     println!("Waiting on workers futures to exit...");
@@ -89,141 +71,7 @@ async fn run_emails_s3_tests(initiate_rebalance: bool) {
     scope.shutdown();
 }
 
-impl TestScope {
-    async fn new() -> Self {
-        let topic = format!("emails-{}", Uuid::new_v4());
-        let table = prepare_table(&topic).await;
-        let workers_token = Arc::new(CancellationToken::new());
-        let mut runtime = HashMap::new();
-        runtime.insert(WORKER_1, helpers::create_runtime(WORKER_1));
-        runtime.insert(WORKER_2, helpers::create_runtime(WORKER_2));
-
-        println!("Topic: {}", &topic);
-        println!("Table: {}", &table);
-        helpers::create_topic(topic.as_str(), TEST_PARTITIONS).await;
-
-        Self {
-            topic,
-            table,
-            workers_token,
-            runtime,
-        }
-    }
-
-    fn shutdown(self) {
-        for (_, rt) in self.runtime {
-            rt.shutdown_background()
-        }
-    }
-
-    async fn create_and_start(&self, name: &str) -> JoinHandle<()> {
-        let rt = self.runtime.get(name).unwrap();
-        let topic = self.topic.clone();
-        let table = self.table.clone();
-        let options = self.create_options();
-        let token = self.workers_token.clone();
-        rt.spawn(async move {
-            let res = start_ingest(topic, table, options, token.clone()).await;
-            res.unwrap_or_else(|e| println!("AN ERROR OCCURED: {}", e));
-            println!("Ingest process exited");
-
-            token.cancel();
-        })
-    }
-
-    fn create_options(&self) -> IngestOptions {
-        env::set_var("AZURE_STORAGE_USE_EMULATOR", "true");
-        env::set_var("AZURE_ACCOUNT_NAME", "devstoreaccount1");
-        env::set_var("AZURE_ACCESS_KEY", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==");
-        env::set_var("AZURE_STORAGE_CONTAINER_NAME", "tests");
-        env::set_var("AZURE_STORAGE_ALLOW_HTTP", "1");
-        env::set_var("AZURITE_BLOB_STORAGE_URL", "http://127.0.0.1:10000");
-        env::set_var(
-            "AZURE_STORAGE_CONNECTION_STRING", 
-            "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;");
-
-        let mut additional_kafka_settings = HashMap::new();
-        additional_kafka_settings.insert("auto.offset.reset".to_string(), "earliest".to_string());
-        let additional_kafka_settings = Some(additional_kafka_settings);
-
-        let allowed_latency = 2;
-        let max_messages_per_batch = 10;
-        let min_bytes_per_file = 370;
-
-        let mut transforms = HashMap::new();
-        transforms.insert("date".to_string(), "substr(timestamp,`0`,`10`)".to_string());
-        transforms.insert("_kafka_offset".to_string(), "kafka.offset".to_string());
-
-        IngestOptions {
-            transforms,
-            kafka_brokers: helpers::test_broker(),
-            consumer_group_id: TEST_CONSUMER_GROUP_ID.to_string(),
-            app_id: TEST_APP_ID.to_string(),
-            additional_kafka_settings,
-            allowed_latency,
-            max_messages_per_batch,
-            min_bytes_per_file,
-            write_checkpoints: true,
-            ..Default::default()
-        }
-    }
-
-    async fn send_messages(&self, amount: i32) {
-        let producer = helpers::create_producer();
-        let now: DateTime<Utc> = Utc::now();
-
-        println!("Sending {} messages to {}", amount, &self.topic);
-        for n in 0..amount {
-            let json = &json!({
-                "id": n.to_string(),
-                "sender": format!("sender-{}@example.com", n),
-                "recipient": format!("recipient-{}@example.com", n),
-                "timestamp": (now + chrono::Duration::seconds(1)).to_rfc3339_opts(SecondsFormat::Secs, true),
-            });
-            helpers::send_json(&producer, &self.topic, &json).await;
-        }
-        println!("All messages are sent");
-    }
-
-    async fn wait_on_total_offset(&self, apps: Vec<String>, offset: i32) {
-        let mut table = deltalake::open_table(&self.table).await.unwrap();
-        let expected_total = offset - TEST_PARTITIONS;
-        loop {
-            table.update().await.unwrap();
-            let mut total = 0;
-            for key in apps.iter() {
-                total += table
-                    .get_app_transaction_version()
-                    .get(key)
-                    .map(|x| *x)
-                    .unwrap_or(0);
-            }
-
-            if total >= expected_total as i64 {
-                self.workers_token.cancel();
-                println!("All messages are in delta");
-                return;
-            }
-
-            println!("Expecting offsets in delta {}/{}...", total, expected_total);
-            tokio::time::sleep(Duration::from_secs(1)).await;
-        }
-    }
-
-    async fn validate_data(&self) {
-        let table = deltalake::open_table(&self.table).await.unwrap();
-        let result = helpers::read_files_from_store(&table).await;
-        let r: Vec<i32> = (0..TEST_TOTAL_MESSAGES).collect();
-        println!("Got messages {}/{}", result.len(), TEST_TOTAL_MESSAGES);
-
-        if result.len() != TEST_TOTAL_MESSAGES as usize {
-            helpers::inspect_table(&self.table).await;
-        }
-
-        assert_eq!(result, r);
-    }
-}
-
+#[cfg(feature = "azure")]
 async fn prepare_table(topic: &str) -> String {
     let container_client = azure_storage_blobs::prelude::ClientBuilder::emulator()
         .container_client(helpers::test_s3_bucket());
@@ -257,7 +105,7 @@ async fn prepare_table(topic: &str) -> String {
 fn create_partitions_app_ids(num_p: i32) -> Vec<String> {
     let mut vector = Vec::new();
     for n in 0..num_p {
-        vector.push(format!("{}-{}", TEST_APP_ID, n));
+        vector.push(format!("{}-{}", helpers::TEST_APP_ID, n));
     }
     vector
 }
diff --git a/tests/emails_s3_tests.rs b/tests/emails_s3_tests.rs
index e9ae38d..b35572a 100644
--- a/tests/emails_s3_tests.rs
+++ b/tests/emails_s3_tests.rs
@@ -3,30 +3,18 @@ mod helpers;
 
 use std::collections::HashMap;
 use std::env;
-use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
 
-use tokio::runtime::Runtime;
-
 use chrono::prelude::*;
-use serde_json::json;
 use serial_test::serial;
-use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
 
 use kafka_delta_ingest::{start_ingest, IngestOptions};
 use rusoto_core::Region;
 use rusoto_s3::{CopyObjectRequest, S3};
-use tokio::task::JoinHandle;
-
-const TEST_APP_ID: &str = "emails_test";
-const TEST_CONSUMER_GROUP_ID: &str = "kafka_delta_ingest_emails";
-const TEST_PARTITIONS: i32 = 4;
-const TEST_TOTAL_MESSAGES: i32 = 200;
 
-const WORKER_1: &str = "WORKER-1";
-const WORKER_2: &str = "WORKER-2";
+use helpers::*;
 
 #[tokio::test(flavor = "multi_thread")]
 #[serial]
@@ -40,16 +28,12 @@ async fn when_rebalance_happens() {
     run_emails_s3_tests(true).await;
 }
 
-struct TestScope {
-    pub topic: String,
-    pub table: String,
-    pub workers_token: Arc<CancellationToken>,
-    pub runtime: HashMap<&'static str, Runtime>,
-}
-
 async fn run_emails_s3_tests(initiate_rebalance: bool) {
     helpers::init_logger();
-    let scope = TestScope::new().await;
+    let topic = format!("emails_s3-{}", Uuid::new_v4());
+    let table = prepare_table(&topic).await;
+    let options = create_options(helpers::WORKER_1);
+    let scope = TestScope::new(&topic, &table, options).await;
 
     let w1 = scope.create_and_start(WORKER_1).await;
 
@@ -89,138 +73,40 @@ async fn run_emails_s3_tests(initiate_rebalance: bool) {
     scope.shutdown();
 }
 
-impl TestScope {
-    async fn new() -> Self {
-        let topic = format!("emails-{}", Uuid::new_v4());
-        let table = prepare_table(&topic).await;
-        let workers_token = Arc::new(CancellationToken::new());
-        let mut runtime = HashMap::new();
-        runtime.insert(WORKER_1, helpers::create_runtime(WORKER_1));
-        runtime.insert(WORKER_2, helpers::create_runtime(WORKER_2));
-
-        println!("Topic: {}", &topic);
-        println!("Table: {}", &table);
-        helpers::create_topic(topic.as_str(), TEST_PARTITIONS).await;
-
-        Self {
-            topic,
-            table,
-            workers_token,
-            runtime,
-        }
-    }
-
-    fn shutdown(self) {
-        for (_, rt) in self.runtime {
-            rt.shutdown_background()
-        }
-    }
-
-    async fn create_and_start(&self, name: &str) -> JoinHandle<()> {
-        let rt = self.runtime.get(name).unwrap();
-        let topic = self.topic.clone();
-        let table = self.table.clone();
-        let options = self.create_options(name);
-        let token = self.workers_token.clone();
-        rt.spawn(async move {
-            let res = start_ingest(topic, table, options, token.clone()).await;
-            res.unwrap_or_else(|e| println!("AN ERROR OCCURED: {}", e));
-            println!("Ingest process exited");
-
-            token.cancel();
-        })
-    }
-
-    fn create_options(&self, name: &str) -> IngestOptions {
-        env::set_var("AWS_S3_LOCKING_PROVIDER", "dynamodb");
-        env::set_var("AWS_REGION", "us-east-2");
-        env::set_var("AWS_STORAGE_ALLOW_HTTP", "true");
-        env::set_var("DYNAMO_LOCK_TABLE_NAME", "locks");
-        env::set_var("DYNAMO_LOCK_OWNER_NAME", name);
-        env::set_var("DYNAMO_LOCK_PARTITION_KEY_VALUE", "emails_s3_tests");
-        env::set_var("DYNAMO_LOCK_REFRESH_PERIOD_MILLIS", "100");
-        env::set_var("DYNAMO_LOCK_ADDITIONAL_TIME_TO_WAIT_MILLIS", "100");
-        env::set_var("DYNAMO_LOCK_LEASE_DURATION", "2");
-
-        let mut additional_kafka_settings = HashMap::new();
-        additional_kafka_settings.insert("auto.offset.reset".to_string(), "earliest".to_string());
-        let additional_kafka_settings = Some(additional_kafka_settings);
-
-        let allowed_latency = 2;
-        let max_messages_per_batch = 10;
-        let min_bytes_per_file = 370;
-
-        let mut transforms = HashMap::new();
-        transforms.insert("date".to_string(), "substr(timestamp,`0`,`10`)".to_string());
-        transforms.insert("_kafka_offset".to_string(), "kafka.offset".to_string());
-
-        IngestOptions {
-            transforms,
-            kafka_brokers: helpers::test_broker(),
-            consumer_group_id: TEST_CONSUMER_GROUP_ID.to_string(),
-            app_id: TEST_APP_ID.to_string(),
-            additional_kafka_settings,
-            allowed_latency,
-            max_messages_per_batch,
-            min_bytes_per_file,
-            write_checkpoints: true,
-            ..Default::default()
-        }
-    }
-
-    async fn send_messages(&self, amount: i32) {
-        let producer = helpers::create_producer();
-        let now: DateTime<Utc> = Utc::now();
-
-        println!("Sending {} messages to {}", amount, &self.topic);
-        for n in 0..amount {
-            let json = &json!({
-                "id": n.to_string(),
-                "sender": format!("sender-{}@example.com", n),
-                "recipient": format!("recipient-{}@example.com", n),
-                "timestamp": (now + chrono::Duration::seconds(1)).to_rfc3339_opts(SecondsFormat::Secs, true),
-            });
-            helpers::send_json(&producer, &self.topic, &json).await;
-        }
-        println!("All messages are sent");
-    }
-
-    async fn wait_on_total_offset(&self, apps: Vec<String>, offset: i32) {
-        let mut table = deltalake::open_table(&self.table).await.unwrap();
-        let expected_total = offset - TEST_PARTITIONS;
-        loop {
-            table.update().await.unwrap();
-            let mut total = 0;
-            for key in apps.iter() {
-                total += table
-                    .get_app_transaction_version()
-                    .get(key)
-                    .map(|x| *x)
-                    .unwrap_or(0);
-            }
-
-            if total >= expected_total as i64 {
-                self.workers_token.cancel();
-                println!("All messages are in delta");
-                return;
-            }
-
-            println!("Expecting offsets in delta {}/{}...", total, expected_total);
-            tokio::time::sleep(Duration::from_secs(1)).await;
-        }
-    }
-
-    async fn validate_data(&self) {
-        let table = deltalake::open_table(&self.table).await.unwrap();
-        let result = helpers::read_files_from_store(&table).await;
-        let r: Vec<i32> = (0..TEST_TOTAL_MESSAGES).collect();
-        println!("Got messages {}/{}", result.len(), TEST_TOTAL_MESSAGES);
-
-        if result.len() != TEST_TOTAL_MESSAGES as usize {
-            helpers::inspect_table(&self.table).await;
-        }
-
-        assert_eq!(result, r);
+fn create_options(name: &str) -> IngestOptions {
+    env::set_var("AWS_S3_LOCKING_PROVIDER", "dynamodb");
+    env::set_var("AWS_REGION", "us-east-2");
+    env::set_var("AWS_STORAGE_ALLOW_HTTP", "true");
+    env::set_var("DYNAMO_LOCK_TABLE_NAME", "locks");
+    env::set_var("DYNAMO_LOCK_OWNER_NAME", name);
+    env::set_var("DYNAMO_LOCK_PARTITION_KEY_VALUE", "emails_s3_tests");
+    env::set_var("DYNAMO_LOCK_REFRESH_PERIOD_MILLIS", "100");
+    env::set_var("DYNAMO_LOCK_ADDITIONAL_TIME_TO_WAIT_MILLIS", "100");
+    env::set_var("DYNAMO_LOCK_LEASE_DURATION", "2");
+
+    let mut additional_kafka_settings = HashMap::new();
+    additional_kafka_settings.insert("auto.offset.reset".to_string(), "earliest".to_string());
+    let additional_kafka_settings = Some(additional_kafka_settings);
+
+    let allowed_latency = 2;
+    let max_messages_per_batch = 10;
+    let min_bytes_per_file = 370;
+
+    let mut transforms = HashMap::new();
+    transforms.insert("date".to_string(), "substr(timestamp,`0`,`10`)".to_string());
+    transforms.insert("_kafka_offset".to_string(), "kafka.offset".to_string());
+
+    IngestOptions {
+        transforms,
+        kafka_brokers: helpers::test_broker(),
+        consumer_group_id: helpers::TEST_CONSUMER_GROUP_ID.to_string(),
+        app_id: helpers::TEST_APP_ID.to_string(),
+        additional_kafka_settings,
+        allowed_latency,
+        max_messages_per_batch,
+        min_bytes_per_file,
+        write_checkpoints: true,
+        ..Default::default()
     }
 }
 
diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index 2c148d6..ee47051 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -1,5 +1,14 @@
+use std::collections::HashMap;
+use std::env;
+use std::fs::{File, OpenOptions};
+use std::io::prelude::*;
+use std::io::{BufReader, Cursor};
+use std::path::Path as FilePath;
+use std::sync::Arc;
+use std::time::Duration;
+
 use bytes::Buf;
-use chrono::Local;
+use chrono::prelude::*;
 use deltalake::action::{Action, Add, MetaData, Protocol, Remove, Txn};
 use deltalake::parquet::{
     file::reader::{FileReader, SerializedFileReader},
@@ -16,12 +25,6 @@ use rdkafka::util::{DefaultRuntime, Timeout};
 use rdkafka::ClientConfig;
 use serde::de::DeserializeOwned;
 use serde_json::{json, Value};
-use std::env;
-use std::fs::{File, OpenOptions};
-use std::io::prelude::*;
-use std::io::{BufReader, Cursor};
-use std::path::Path as FilePath;
-use std::sync::Arc;
 use tokio::runtime::Runtime;
 use tokio::task::JoinHandle;
 use tokio_util::sync::CancellationToken;
@@ -210,6 +213,7 @@ pub async fn create_and_run_kdi(
     Runtime,
 ) {
     init_logger();
+    println!("OPTS!: {:?}", opts);
     let topic = format!("{}-{}", app_id, Uuid::new_v4());
     let table = create_local_table(schema, delta_partitions, &topic);
     create_topic(&topic, kafka_num_partitions).await;
@@ -542,3 +546,116 @@ fn parse_json_field<T: DeserializeOwned>(value: &Value, key: &str) -> Option<T>
         .and_then(|v| v.get(key))
         .and_then(|v| serde_json::from_value::<T>(v.clone()).ok())
 }
+
+pub const TEST_APP_ID: &str = "emails_test";
+pub const TEST_CONSUMER_GROUP_ID: &str = "kafka_delta_ingest_emails";
+pub const TEST_PARTITIONS: i32 = 4;
+pub const TEST_TOTAL_MESSAGES: i32 = 200;
+
+pub const WORKER_1: &str = "WORKER-1";
+pub const WORKER_2: &str = "WORKER-2";
+
+pub struct TestScope {
+    pub topic: String,
+    pub table: String,
+    pub workers_token: Arc<CancellationToken>,
+    pub runtime: HashMap<&'static str, Runtime>,
+    options: IngestOptions,
+}
+
+impl TestScope {
+    pub async fn new(topic: &str, table: &str, options: IngestOptions) -> Self {
+        let workers_token = Arc::new(CancellationToken::new());
+        let mut runtime = HashMap::new();
+        runtime.insert(WORKER_1, create_runtime(WORKER_1));
+        runtime.insert(WORKER_2, create_runtime(WORKER_2));
+
+        println!("Topic: {}", &topic);
+        println!("Table: {}", &table);
+        create_topic(topic, TEST_PARTITIONS).await;
+
+        Self {
+            topic: topic.into(),
+            table: table.into(),
+            workers_token,
+            runtime,
+            options,
+        }
+    }
+
+    pub fn shutdown(self) {
+        for (_, rt) in self.runtime {
+            rt.shutdown_background()
+        }
+    }
+
+    pub async fn create_and_start(&self, name: &str) -> JoinHandle<()> {
+        let rt = self.runtime.get(name).unwrap();
+        let topic = self.topic.clone();
+        let table = self.table.clone();
+        let token = self.workers_token.clone();
+        let options = self.options.clone();
+        rt.spawn(async move {
+            let res = start_ingest(topic, table, options, token.clone()).await;
+            res.unwrap_or_else(|e| println!("AN ERROR OCCURED: {}", e));
+            println!("Ingest process exited");
+
+            token.cancel();
+        })
+    }
+
+    pub async fn send_messages(&self, amount: i32) {
+        let producer = create_producer();
+        let now: DateTime<Utc> = Utc::now();
+
+        println!("Sending {} messages to {}", amount, &self.topic);
+        for n in 0..amount {
+            let json = &json!({
+                "id": n.to_string(),
+                "sender": format!("sender-{}@example.com", n),
+                "recipient": format!("recipient-{}@example.com", n),
+                "timestamp": (now + chrono::Duration::seconds(1)).to_rfc3339_opts(SecondsFormat::Secs, true),
+            });
+            send_json(&producer, &self.topic, &json).await;
+        }
+        println!("All messages are sent");
+    }
+
+    pub async fn wait_on_total_offset(&self, apps: Vec<String>, offset: i32) {
+        let mut table = deltalake::open_table(&self.table).await.unwrap();
+        let expected_total = offset - TEST_PARTITIONS;
+        loop {
+            table.update().await.unwrap();
+            let mut total = 0;
+            for key in apps.iter() {
+                total += table
+                    .get_app_transaction_version()
+                    .get(key)
+                    .map(|x| *x)
+                    .unwrap_or(0);
+            }
+
+            if total >= expected_total as i64 {
+                self.workers_token.cancel();
+                println!("All messages are in delta");
+                return;
+            }
+
+            println!("Expecting offsets in delta {}/{}...", total, expected_total);
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    }
+
+    pub async fn validate_data(&self) {
+        let table = deltalake::open_table(&self.table).await.unwrap();
+        let result = read_files_from_store(&table).await;
+        let r: Vec<i32> = (0..TEST_TOTAL_MESSAGES).collect();
+        println!("Got messages {}/{}", result.len(), TEST_TOTAL_MESSAGES);
+
+        if result.len() != TEST_TOTAL_MESSAGES as usize {
+            inspect_table(&self.table).await;
+        }
+
+        assert_eq!(result, r);
+    }
+}

From 903d053349897835a9c974384f4f3bbaa001673c Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Sat, 27 May 2023 15:15:15 -0700
Subject: [PATCH 07/18] Properly run clippy with feature flags

---
 .github/workflows/build.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 370bc22..8fe7c69 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -30,8 +30,10 @@ jobs:
         profile: default
         toolchain: stable
         override: true
-    - name: build and lint with clippy
-      run: cargo clippy
+    - name: Build and lint for S3
+      run: cargo clippy --features s3
+    - name: Build and lint for Azure
+      run: cargo clippy --features azure
 
   test:
     runs-on: ubuntu-latest

From 216f763191107b05c7970f763db8a4ff7384723b Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Wed, 14 Jun 2023 07:44:56 -0700
Subject: [PATCH 08/18] Copy test fixtures in Rust rather than relying on shell
 scripts

---
 build.rs                 |  2 +-
 tests/emails_s3_tests.rs | 33 ++++++++++++++++++++++++++++-----
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/build.rs b/build.rs
index 73e7796..d74f281 100644
--- a/build.rs
+++ b/build.rs
@@ -1,5 +1,5 @@
 #[cfg(not(any(feature = "s3", feature = "azure")))]
 compile_error!(
-    "Either the \"s3\" or the \"azure\" feature must be built to compile kafka-delta-ingest"
+    "Either the \"s3\" or the \"azure\" feature must be enabled to compile kafka-delta-ingest"
 );
 fn main() {}
diff --git a/tests/emails_s3_tests.rs b/tests/emails_s3_tests.rs
index b35572a..ceb0820 100644
--- a/tests/emails_s3_tests.rs
+++ b/tests/emails_s3_tests.rs
@@ -3,16 +3,16 @@ mod helpers;
 
 use std::collections::HashMap;
 use std::env;
+use std::io::Read;
 use std::thread;
 use std::time::Duration;
 
-use chrono::prelude::*;
 use serial_test::serial;
 use uuid::Uuid;
 
 use kafka_delta_ingest::{start_ingest, IngestOptions};
 use rusoto_core::Region;
-use rusoto_s3::{CopyObjectRequest, S3};
+use rusoto_s3::{CopyObjectRequest, PutObjectRequest, S3};
 
 use helpers::*;
 
@@ -111,15 +111,38 @@ fn create_options(name: &str) -> IngestOptions {
 }
 
 async fn prepare_table(topic: &str) -> String {
-    env::set_var("AWS_ENDPOINT_URL", helpers::test_aws_endpoint());
-    env::set_var("AWS_ACCESS_KEY_ID", "test");
-    env::set_var("AWS_SECRET_ACCESS_KEY", "test");
+    match env::var("AWS_ACCESS_KEY_ID") {
+        Err(_) => env::set_var("AWS_ACCESS_KEY_ID", "test"),
+        Ok(_) => {}
+    }
+    match env::var("AWS_SECRET_ACCESS_KEY") {
+        Err(_) => env::set_var("AWS_SECRET_ACCESS_KEY", "test"),
+        Ok(_) => {}
+    }
 
     let s3 = rusoto_s3::S3Client::new(Region::Custom {
         name: "custom".to_string(),
         endpoint: helpers::test_aws_endpoint(),
     });
 
+    /*
+     * Copy the local fixture to create a simple delta table in storage.
+     */
+    let mut buf = vec![];
+    let original_log =
+        std::fs::File::open("tests/data/emails/_delta_log/00000000000000000000.json")
+            .unwrap()
+            .read_to_end(&mut buf);
+
+    s3.put_object(PutObjectRequest {
+        bucket: helpers::test_s3_bucket(),
+        body: Some(buf.into()),
+        key: "emails/_delta_log/00000000000000000000.json".into(),
+        ..Default::default()
+    })
+    .await
+    .unwrap();
+
     s3.copy_object(CopyObjectRequest {
         bucket: helpers::test_s3_bucket(),
         key: format!("{}/_delta_log/00000000000000000000.json", topic),

From 67269cc0e4e2afc9a4a7fc7d67f529a5dc4f8714 Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Wed, 14 Jun 2023 10:55:35 -0700
Subject: [PATCH 09/18] Exclude DynamoDB locking mechanisms for `azure` builds

---
 src/dead_letters.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/dead_letters.rs b/src/dead_letters.rs
index 9443dbf..ae6c240 100644
--- a/src/dead_letters.rs
+++ b/src/dead_letters.rs
@@ -4,8 +4,10 @@ use chrono::prelude::*;
 use core::fmt::Debug;
 use deltalake::parquet::errors::ParquetError;
 use deltalake::{DeltaTable, DeltaTableError};
+#[cfg(feature = "s3")]
 use dynamodb_lock::dynamo_lock_options;
 use log::{error, info, warn};
+#[cfg(feature = "s3")]
 use maplit::hashmap;
 use rdkafka::message::BorrowedMessage;
 use serde::{Deserialize, Serialize};
@@ -15,6 +17,7 @@ use std::collections::HashMap;
 use crate::{transforms::TransformError, writer::*};
 use deltalake::checkpoints::CheckpointError;
 
+#[cfg(feature = "s3")]
 mod env_vars {
     pub(crate) const DEAD_LETTER_DYNAMO_LOCK_PARTITION_KEY_VALUE: &str =
         "DEAD_LETTER_DYNAMO_LOCK_PARTITION_KEY_VALUE";
@@ -246,10 +249,14 @@ impl DeltaSinkDeadLetterQueue {
     ) -> Result<Self, DeadLetterQueueError> {
         match &options.delta_table_uri {
             Some(table_uri) => {
+                #[cfg(feature = "s3")]
                 let opts = hashmap! {
                     dynamo_lock_options::DYNAMO_LOCK_PARTITION_KEY_VALUE.to_string() => std::env::var(env_vars::DEAD_LETTER_DYNAMO_LOCK_PARTITION_KEY_VALUE)
                     .unwrap_or_else(|_| "kafka_delta_ingest-dead_letters".to_string()),
                 };
+                #[cfg(feature = "azure")]
+                let opts = HashMap::default();
+
                 let table = crate::delta_helpers::load_table(table_uri, opts.clone()).await?;
                 let delta_writer = DataWriter::for_table(&table, opts)?;
 

From d6380c2ef7279a865f24dde065937e05aa7c2175 Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Thu, 15 Jun 2023 06:38:22 -0700
Subject: [PATCH 10/18] Correct the port for the localstack endpoint

This looks like it was just as a copy/paste error
---
 tests/helpers/mod.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index ee47051..f10ca8b 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -38,7 +38,7 @@ pub fn test_broker() -> String {
 }
 
 pub fn test_aws_endpoint() -> String {
-    env::var("AWS_ENDPOINT_URL").unwrap_or("http://0.0.0.0:4506".into())
+    env::var("AWS_ENDPOINT_URL").unwrap_or("http://0.0.0.0:4566".into())
 }
 
 pub fn test_s3_bucket() -> String {

From 3bb36c205730bd279eb5915a8eec3814611b4c4b Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Thu, 15 Jun 2023 07:01:10 -0700
Subject: [PATCH 11/18] Force a region to be set whenever kafka-delta-ingest is
 launched in tests

There's a test failure I'm seeing in GitHub Actions that I'm hoping this
resolves:

    hread 'when_both_workers_started_simultaneously' panicked at 'called `Result::unwrap()` on an `Err` value: LoadCheckpoint { source: Storage { source: Generic { store: "S3", source: GetRequest { source: Error { retries: 0, message: "Received redirect without LOCATION, this normally indicates an incorrectly configured region", source: None }, path: "emails_s3-33758618-2217-4969-9a91-dbd6b973ba04/_delta_log/_last_checkpoint" } } } }', tests/helpers/mod.rs:602:66
---
 tests/helpers/mod.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index f10ca8b..9d71168 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -268,6 +268,7 @@ pub fn create_kdi_with(
     let worker_name = worker_name.unwrap_or(app_id.clone());
 
     env::set_var("AWS_S3_LOCKING_PROVIDER", "dynamodb");
+    env::set_var("AWS_REGION", "us-east-2");
     env::set_var("DYNAMO_LOCK_TABLE_NAME", "locks");
     env::set_var("DYNAMO_LOCK_OWNER_NAME", Uuid::new_v4().to_string());
     env::set_var("DYNAMO_LOCK_PARTITION_KEY_VALUE", app_id.clone());

From 188373548c703cf81fee9811991bddcdfda8e1e3 Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Wed, 6 Dec 2023 22:38:43 -0800
Subject: [PATCH 12/18] Add more Debug traits

---
 src/lib.rs                        |  4 ++--
 src/main.rs                       | 10 +++++-----
 src/transforms.rs                 | 15 +++++----------
 src/writer.rs                     |  8 ++++----
 tests/buffer_flush_tests.rs       |  1 -
 tests/dead_letter_tests.rs        |  2 +-
 tests/delta_partitions_tests.rs   | 18 +++++++++---------
 tests/deserialization_tests.rs    |  2 +-
 tests/emails_azure_blob_tests.rs  |  8 --------
 tests/emails_s3_tests.rs          |  4 ++--
 tests/end_at_last_offsets_test.rs |  2 +-
 tests/feed_s3_tests.rs            |  6 +++---
 tests/helpers/mod.rs              | 24 ++++++++++--------------
 tests/schema_update_tests.rs      |  2 +-
 14 files changed, 44 insertions(+), 62 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index ae70fe0..8bef8d8 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -214,7 +214,7 @@ pub enum IngestError {
 }
 
 /// Formats for message parsing
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub enum MessageFormat {
     /// Parses messages as json and uses the inferred schema
     DefaultJson,
@@ -227,7 +227,7 @@ pub enum MessageFormat {
 }
 
 /// Source for schema
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub enum SchemaSource {
     /// Use default behavior
     None,
diff --git a/src/main.rs b/src/main.rs
index 0ee6b89..b9fdcb8 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -534,18 +534,18 @@ mod test {
         let values = ["-e", "--ends_at_latest_offsets"];
         for value in values {
             let subcommand = get_subcommand_matches_raw(vec![value]);
-            assert_eq!(true, subcommand.contains_id("end"));
-            assert_eq!(true, subcommand.get_flag("end"));
+            assert!(subcommand.contains_id("end"));
+            assert!(subcommand.get_flag("end"));
         }
 
         let subcommand = get_subcommand_matches_raw(vec![]);
-        assert_eq!(true, subcommand.contains_id("end"));
-        assert_eq!(false, subcommand.get_flag("end"));
+        assert!(subcommand.contains_id("end"));
+        assert!(!subcommand.get_flag("end"));
     }
 
     fn get_subcommand_matches(args: Vec<&str>) -> Result<MessageFormat, SchemaSourceError> {
         let arg_matches = get_subcommand_matches_raw(args);
-        return convert_matches_to_message_format(&arg_matches);
+        convert_matches_to_message_format(&arg_matches)
     }
 
     fn get_subcommand_matches_raw(args: Vec<&str>) -> ArgMatches {
diff --git a/src/transforms.rs b/src/transforms.rs
index e9da554..b4803b6 100644
--- a/src/transforms.rs
+++ b/src/transforms.rs
@@ -495,12 +495,7 @@ mod tests {
         let new_value_path = ValuePath::from_str("name.first");
         let new_value = Value::String("John".to_string());
 
-        set_value(
-            &mut val.as_object_mut().unwrap(),
-            &new_value_path,
-            0,
-            new_value,
-        );
+        set_value(val.as_object_mut().unwrap(), &new_value_path, 0, new_value);
 
         assert_eq!(
             json!({
@@ -539,7 +534,7 @@ mod tests {
 
         let transformer = Transformer::from_transforms(&transforms).unwrap();
 
-        let _ = transformer
+        transformer
             .transform(&mut test_value, Some(&test_message))
             .unwrap();
 
@@ -610,7 +605,7 @@ mod tests {
 
         let transformer = Transformer::from_transforms(&transforms).unwrap();
 
-        let _ = transformer
+        transformer
             .transform(&mut test_value, Some(&test_message))
             .unwrap();
 
@@ -677,9 +672,9 @@ mod tests {
             "kafka.timestamp_type".to_string(),
         );
 
-        let transformer = Transformer::from_transforms(&&transforms).unwrap();
+        let transformer = Transformer::from_transforms(&transforms).unwrap();
 
-        let _ = transformer
+        transformer
             .transform(&mut test_value, Some(&test_message))
             .unwrap();
 
diff --git a/src/writer.rs b/src/writer.rs
index edc884b..cccfb61 100644
--- a/src/writer.rs
+++ b/src/writer.rs
@@ -1198,7 +1198,7 @@ mod tests {
                     assert_eq!("2021-06-22", timestamp.as_str().unwrap());
                 }
                 ("some_int", ColumnValueStat::Value(v)) => assert_eq!(302, v.as_i64().unwrap()),
-                ("some_bool", ColumnValueStat::Value(v)) => assert_eq!(false, v.as_bool().unwrap()),
+                ("some_bool", ColumnValueStat::Value(v)) => assert!(!v.as_bool().unwrap()),
                 ("some_string", ColumnValueStat::Value(v)) => {
                     assert_eq!("GET", v.as_str().unwrap())
                 }
@@ -1226,7 +1226,7 @@ mod tests {
                     assert_eq!("2021-06-22", timestamp.as_str().unwrap());
                 }
                 ("some_int", ColumnValueStat::Value(v)) => assert_eq!(400, v.as_i64().unwrap()),
-                ("some_bool", ColumnValueStat::Value(v)) => assert_eq!(true, v.as_bool().unwrap()),
+                ("some_bool", ColumnValueStat::Value(v)) => assert!(v.as_bool().unwrap()),
                 ("some_string", ColumnValueStat::Value(v)) => {
                     assert_eq!("PUT", v.as_str().unwrap())
                 }
@@ -1267,8 +1267,8 @@ mod tests {
     fn create_temp_table(table_path: &Path) {
         let log_path = table_path.join("_delta_log");
 
-        let _ = std::fs::create_dir(log_path.as_path()).unwrap();
-        let _ = std::fs::write(
+        std::fs::create_dir(log_path.as_path()).unwrap();
+        std::fs::write(
             log_path.join("00000000000000000000.json"),
             V0_COMMIT.as_str(),
         )
diff --git a/tests/buffer_flush_tests.rs b/tests/buffer_flush_tests.rs
index cbdeb6d..88049d2 100644
--- a/tests/buffer_flush_tests.rs
+++ b/tests/buffer_flush_tests.rs
@@ -1,7 +1,6 @@
 #[allow(dead_code)]
 mod helpers;
 
-use deltalake;
 use log::info;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
diff --git a/tests/dead_letter_tests.rs b/tests/dead_letter_tests.rs
index d05ca07..0b9082a 100644
--- a/tests/dead_letter_tests.rs
+++ b/tests/dead_letter_tests.rs
@@ -1,7 +1,7 @@
 use kafka_delta_ingest::IngestOptions;
 use log::info;
 use serde::{Deserialize, Serialize};
-use serde_json;
+
 use serde_json::json;
 use serde_json::Value;
 use uuid::Uuid;
diff --git a/tests/delta_partitions_tests.rs b/tests/delta_partitions_tests.rs
index cb3b8bf..d354e1d 100644
--- a/tests/delta_partitions_tests.rs
+++ b/tests/delta_partitions_tests.rs
@@ -73,21 +73,21 @@ async fn test_delta_partitions() {
         {
             Some("red") => {
                 assert!(add.path.starts_with("color=red"));
-                assert_eq!(&get_stats_value(&add, "numRecords"), "4");
-                assert_eq!(msg(get_stats_value(&add, "minValues")).id, 1);
-                assert_eq!(msg(get_stats_value(&add, "maxValues")).id, 6);
+                assert_eq!(&get_stats_value(add, "numRecords"), "4");
+                assert_eq!(msg(get_stats_value(add, "minValues")).id, 1);
+                assert_eq!(msg(get_stats_value(add, "maxValues")).id, 6);
             }
             Some("blue") => {
                 assert!(add.path.starts_with("color=blue"));
-                assert_eq!(&get_stats_value(&add, "numRecords"), "4");
-                assert_eq!(msg(get_stats_value(&add, "minValues")).id, 3);
-                assert_eq!(msg(get_stats_value(&add, "maxValues")).id, 8);
+                assert_eq!(&get_stats_value(add, "numRecords"), "4");
+                assert_eq!(msg(get_stats_value(add, "minValues")).id, 3);
+                assert_eq!(msg(get_stats_value(add, "maxValues")).id, 8);
             }
             None => {
                 assert!(add.path.starts_with("color=__HIVE_DEFAULT_PARTITION__"));
-                assert_eq!(&get_stats_value(&add, "numRecords"), "1");
-                assert_eq!(msg(get_stats_value(&add, "minValues")).id, 9);
-                assert_eq!(msg(get_stats_value(&add, "maxValues")).id, 9);
+                assert_eq!(&get_stats_value(add, "numRecords"), "1");
+                assert_eq!(msg(get_stats_value(add, "minValues")).id, 9);
+                assert_eq!(msg(get_stats_value(add, "maxValues")).id, 9);
             }
             other => {
                 panic!("{:?}", other);
diff --git a/tests/deserialization_tests.rs b/tests/deserialization_tests.rs
index 48efe2b..b1cead2 100644
--- a/tests/deserialization_tests.rs
+++ b/tests/deserialization_tests.rs
@@ -299,7 +299,7 @@ struct TestMsg {
 }
 
 fn default_settings() -> SrSettings {
-    SrSettings::new(String::from(String::from(SCHEMA_REGISTRY_ADDRESS)))
+    SrSettings::new(String::from(SCHEMA_REGISTRY_ADDRESS))
 }
 
 async fn avro_encode(item: impl Serialize, topic: String) -> Result<Vec<u8>, SRCError> {
diff --git a/tests/emails_azure_blob_tests.rs b/tests/emails_azure_blob_tests.rs
index e96b053..a0393c2 100644
--- a/tests/emails_azure_blob_tests.rs
+++ b/tests/emails_azure_blob_tests.rs
@@ -1,16 +1,8 @@
 #[allow(dead_code)]
 mod helpers;
 
-use std::thread;
-use std::time::Duration;
-
 #[cfg(feature = "azure")]
 use azure_storage::{prelude::BlobSasPermissions, shared_access_signature::SasProtocol};
-use kafka_delta_ingest::{start_ingest, IngestOptions};
-use serial_test::serial;
-use uuid::Uuid;
-
-use helpers::*;
 
 #[cfg(feature = "azure")]
 #[tokio::test(flavor = "multi_thread")]
diff --git a/tests/emails_s3_tests.rs b/tests/emails_s3_tests.rs
index ceb0820..6fd32b5 100644
--- a/tests/emails_s3_tests.rs
+++ b/tests/emails_s3_tests.rs
@@ -10,7 +10,7 @@ use std::time::Duration;
 use serial_test::serial;
 use uuid::Uuid;
 
-use kafka_delta_ingest::{start_ingest, IngestOptions};
+use kafka_delta_ingest::IngestOptions;
 use rusoto_core::Region;
 use rusoto_s3::{CopyObjectRequest, PutObjectRequest, S3};
 
@@ -129,7 +129,7 @@ async fn prepare_table(topic: &str) -> String {
      * Copy the local fixture to create a simple delta table in storage.
      */
     let mut buf = vec![];
-    let original_log =
+    let _original_log =
         std::fs::File::open("tests/data/emails/_delta_log/00000000000000000000.json")
             .unwrap()
             .read_to_end(&mut buf);
diff --git a/tests/end_at_last_offsets_test.rs b/tests/end_at_last_offsets_test.rs
index f9573de..ec73e3c 100644
--- a/tests/end_at_last_offsets_test.rs
+++ b/tests/end_at_last_offsets_test.rs
@@ -63,7 +63,7 @@ async fn end_at_initial_offsets() {
         },
     );
 
-    helpers::wait_until_version_created(&table, 1);
+    helpers::wait_until_version_created(table, 1);
 
     {
         // check that there's 3 records in table
diff --git a/tests/feed_s3_tests.rs b/tests/feed_s3_tests.rs
index 477be89..e1c7e4b 100644
--- a/tests/feed_s3_tests.rs
+++ b/tests/feed_s3_tests.rs
@@ -91,7 +91,7 @@ async fn feed_load_test() {
         .collect();
     ids.sort();
 
-    let id_count = ids.iter().count();
+    let id_count = ids.len();
     let expected = (0..TOTAL_MESSAGES_RECEIVED).count();
 
     if id_count != expected {
@@ -138,8 +138,8 @@ fn spawn_worker(
         .collect();
 
     helpers::create_kdi_with(
-        &topic,
-        &table,
+        topic,
+        table,
         Some(format!("WORKER-{}", id)),
         IngestOptions {
             app_id: "feed".to_string(),
diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index 9d71168..10c8afb 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -103,7 +103,7 @@ pub async fn send_kv_json(
 }
 
 pub async fn send_bytes(producer: &FutureProducer, topic: &str, bytes: &Vec<u8>) {
-    let record: FutureRecord<String, Vec<u8>> = FutureRecord::to(topic).payload(&bytes);
+    let record: FutureRecord<String, Vec<u8>> = FutureRecord::to(topic).payload(bytes);
     let _ = producer.send(record, Timeout::Never).await;
 }
 
@@ -133,9 +133,9 @@ pub async fn read_files_from_store(table: &DeltaTable) -> Vec<i32> {
 
         let reader = SerializedFileReader::new(File::open(&tmp).unwrap()).unwrap();
 
-        let mut row_iter = reader.get_row_iter(None).unwrap();
+        let row_iter = reader.get_row_iter(None).unwrap();
 
-        while let Some(record) = row_iter.next() {
+        for record in row_iter {
             list.push(record.get_string(0).unwrap().parse::<i32>().unwrap());
         }
     }
@@ -151,14 +151,14 @@ fn parse_type(schema: &Value) -> Value {
         Value::String(_) => schema.clone(),
         Value::Object(_) => json!({
             "type": "struct",
-            "fields": parse_fields(&schema),
+            "fields": parse_fields(schema),
         }),
         Value::Array(v) if v.len() == 1 => json!({
             "type": "array",
             "elementType": parse_type(v.first().unwrap()),
             "containsNull": true,
         }),
-        _ => panic!("Unsupported type {}", schema.to_string()),
+        _ => panic!("Unsupported type {}", schema),
     }
 }
 
@@ -416,19 +416,15 @@ async fn json_listify_table_content(table: DeltaTable, store: DeltaObjectStore)
     let tmp = format!(".test-{}.tmp", Uuid::new_v4());
     let mut list = Vec::new();
     for file in table.get_files() {
-        let get_result = store
-            .storage_backend()
-            .get(&Path::from(file))
-            .await
-            .unwrap();
+        let get_result = store.storage_backend().get(&file).await.unwrap();
         let bytes = get_result.bytes().await.unwrap();
         let mut file = File::create(&tmp).unwrap();
         file.write_all(bytes.chunk()).unwrap();
         drop(file);
         let reader = SerializedFileReader::new(File::open(&tmp).unwrap()).unwrap();
-        let mut row_iter = reader.get_row_iter(None).unwrap();
+        let row_iter = reader.get_row_iter(None).unwrap();
 
-        while let Some(record) = row_iter.next() {
+        for record in row_iter {
             list.push(record.to_json_value());
         }
     }
@@ -617,7 +613,7 @@ impl TestScope {
                 "recipient": format!("recipient-{}@example.com", n),
                 "timestamp": (now + chrono::Duration::seconds(1)).to_rfc3339_opts(SecondsFormat::Secs, true),
             });
-            send_json(&producer, &self.topic, &json).await;
+            send_json(&producer, &self.topic, json).await;
         }
         println!("All messages are sent");
     }
@@ -632,7 +628,7 @@ impl TestScope {
                 total += table
                     .get_app_transaction_version()
                     .get(key)
-                    .map(|x| *x)
+                    .copied()
                     .unwrap_or(0);
             }
 
diff --git a/tests/schema_update_tests.rs b/tests/schema_update_tests.rs
index bf2e6a9..6403e39 100644
--- a/tests/schema_update_tests.rs
+++ b/tests/schema_update_tests.rs
@@ -96,7 +96,7 @@ async fn schema_update_test() {
     // convert msg v1 to v2
     let expected = vec![
         MsgV2 {
-            id: msg_v1.id.clone(),
+            id: msg_v1.id,
             color: None,
             date: msg_v1.date.clone(),
         },

From 77e62b777d7adf8bdb85466ef97752c4128844de Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Fri, 8 Dec 2023 05:33:40 +0000
Subject: [PATCH 13/18] Use the localstack endpoint when running S3 tests

---
 Cargo.toml                       |  3 ++
 tests/emails_azure_blob_tests.rs | 62 ++++++++++++++++++++++++++++----
 tests/emails_s3_tests.rs         |  3 ++
 tests/helpers/mod.rs             |  1 +
 4 files changed, 63 insertions(+), 6 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index ce97fc2..e5ee7a5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -54,6 +54,9 @@ sentry-ext = ["sentry"]
 dynamic-linking = [ "rdkafka/dynamic-linking" ]
 azure = [
     "deltalake/azure",
+    "azure_core",
+    "azure_storage",
+    "azure_storage_blobs"
 ]
 s3 = [
     "deltalake/s3",
diff --git a/tests/emails_azure_blob_tests.rs b/tests/emails_azure_blob_tests.rs
index a0393c2..4f5aab2 100644
--- a/tests/emails_azure_blob_tests.rs
+++ b/tests/emails_azure_blob_tests.rs
@@ -1,29 +1,40 @@
+#![cfg(feature = "azure")]
 #[allow(dead_code)]
 mod helpers;
 
-#[cfg(feature = "azure")]
+use std::collections::HashMap;
+use std::env;
+use std::thread;
+use std::time::Duration;
+use time::OffsetDateTime;
+
+use serial_test::serial;
+use uuid::Uuid;
+
+use kafka_delta_ingest::IngestOptions;
+
+use helpers::*;
+
 use azure_storage::{prelude::BlobSasPermissions, shared_access_signature::SasProtocol};
 
-#[cfg(feature = "azure")]
 #[tokio::test(flavor = "multi_thread")]
 #[serial]
 async fn when_both_workers_started_simultaneously_azure() {
     run_emails_s3_tests(false).await;
 }
 
-#[cfg(feature = "azure")]
 #[tokio::test(flavor = "multi_thread")]
 #[serial]
 async fn when_rebalance_happens_azure() {
     run_emails_s3_tests(true).await;
 }
 
-#[cfg(feature = "azure")]
 async fn run_emails_s3_tests(initiate_rebalance: bool) {
     helpers::init_logger();
     let topic = format!("emails_azure-{}", Uuid::new_v4());
     let table = prepare_table(&topic).await;
-    let scope = TestScope::new(&topic, &table).await;
+    let options = create_options();
+    let scope = TestScope::new(&topic, &table, options).await;
 
     let w1 = scope.create_and_start(helpers::WORKER_1).await;
 
@@ -63,7 +74,6 @@ async fn run_emails_s3_tests(initiate_rebalance: bool) {
     scope.shutdown();
 }
 
-#[cfg(feature = "azure")]
 async fn prepare_table(topic: &str) -> String {
     let container_client = azure_storage_blobs::prelude::ClientBuilder::emulator()
         .container_client(helpers::test_s3_bucket());
@@ -101,3 +111,43 @@ fn create_partitions_app_ids(num_p: i32) -> Vec<String> {
     }
     vector
 }
+
+fn create_options() -> IngestOptions {
+    env::set_var("AZURE_STORAGE_USE_EMULATOR", "true");
+    env::set_var("AZURE_ACCOUNT_NAME", "devstoreaccount1");
+    env::set_var(
+        "AZURE_ACCESS_KEY",
+        "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
+    );
+    env::set_var("AZURE_STORAGE_CONTAINER_NAME", "tests");
+    env::set_var("AZURE_STORAGE_ALLOW_HTTP", "1");
+    env::set_var("AZURITE_BLOB_STORAGE_URL", "http://127.0.0.1:10000");
+    env::set_var(
+            "AZURE_STORAGE_CONNECTION_STRING", 
+            "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;");
+
+    let mut additional_kafka_settings = HashMap::new();
+    additional_kafka_settings.insert("auto.offset.reset".to_string(), "earliest".to_string());
+    let additional_kafka_settings = Some(additional_kafka_settings);
+
+    let allowed_latency = 2;
+    let max_messages_per_batch = 10;
+    let min_bytes_per_file = 370;
+
+    let mut transforms = HashMap::new();
+    transforms.insert("date".to_string(), "substr(timestamp,`0`,`10`)".to_string());
+    transforms.insert("_kafka_offset".to_string(), "kafka.offset".to_string());
+
+    IngestOptions {
+        transforms,
+        kafka_brokers: helpers::test_broker(),
+        consumer_group_id: TEST_CONSUMER_GROUP_ID.to_string(),
+        app_id: TEST_APP_ID.to_string(),
+        additional_kafka_settings,
+        allowed_latency,
+        max_messages_per_batch,
+        min_bytes_per_file,
+        write_checkpoints: true,
+        ..Default::default()
+    }
+}
diff --git a/tests/emails_s3_tests.rs b/tests/emails_s3_tests.rs
index 6fd32b5..1c22191 100644
--- a/tests/emails_s3_tests.rs
+++ b/tests/emails_s3_tests.rs
@@ -1,3 +1,5 @@
+#![cfg(feature = "s3")]
+
 #[allow(dead_code)]
 mod helpers;
 
@@ -74,6 +76,7 @@ async fn run_emails_s3_tests(initiate_rebalance: bool) {
 }
 
 fn create_options(name: &str) -> IngestOptions {
+    env::set_var("AWS_ENDPOINT_URL", helpers::test_aws_endpoint());
     env::set_var("AWS_S3_LOCKING_PROVIDER", "dynamodb");
     env::set_var("AWS_REGION", "us-east-2");
     env::set_var("AWS_STORAGE_ALLOW_HTTP", "true");
diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index 10c8afb..67416ec 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -267,6 +267,7 @@ pub fn create_kdi_with(
     let app_id = options.app_id.to_string();
     let worker_name = worker_name.unwrap_or(app_id.clone());
 
+    env::set_var("AWS_ENDPOINT_URL", test_aws_endpoint());
     env::set_var("AWS_S3_LOCKING_PROVIDER", "dynamodb");
     env::set_var("AWS_REGION", "us-east-2");
     env::set_var("DYNAMO_LOCK_TABLE_NAME", "locks");

From df097c61d4dcf8779da759c24391007bc073c653 Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Thu, 7 Dec 2023 23:08:55 -0800
Subject: [PATCH 14/18] Upgrade to deltalake 0.12

---
 Cargo.lock           | 77 +++++++++++++++++++++-----------------------
 Cargo.toml           |  2 +-
 src/delta_helpers.rs |  6 ++--
 src/lib.rs           |  6 ++--
 src/offsets.rs       |  4 +--
 src/writer.rs        |  7 ++--
 6 files changed, 49 insertions(+), 53 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 59d58ea..a559bd2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -168,9 +168,9 @@ dependencies = [
 
 [[package]]
 name = "arrow"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c107a57b5913d852da9d5a40e280e4695f2258b5b87733c13b770c63a7117287"
+checksum = "218ca81dd088b102c0fd6687c72e73fad1ba93d2ef7b3cf9a1043b04b2c39dbf"
 dependencies = [
  "ahash",
  "arrow-arith",
@@ -190,9 +190,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ace6aa3d5617c5d03041a05e01c6819428a8ddf49dd0b055df9b40fef9d96094"
+checksum = "d49309fa2299ec34a709cfc9f487c41ecaead96d1ab70e21857466346bbbd690"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -205,9 +205,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "104a04520692cc674e6afd7682f213ca41f9b13ff1873f63a5a2857a590b87b3"
+checksum = "e7a27466d897d99654357a6d95dc0a26931d9e4306e60c14fc31a894edb86579"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -221,9 +221,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72c875bcb9530ec403998fb0b2dc6d180a7c64563ca4bc22b90eafb84b113143"
+checksum = "9405b78106a9d767c7b97c78a70ee1b23ee51a74f5188a821a716d9a85d1af2b"
 dependencies = [
  "half",
  "num 0.4.0",
@@ -231,9 +231,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6d6e18281636c8fc0b93be59834da6bf9a72bb70fd0c98ddfdaf124da466c28"
+checksum = "be0ec5a79a87783dc828b7ff8f89f62880b3f553bc5f5b932a82f4a1035024b4"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -247,9 +247,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-csv"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3197dab0963a236ff8e7c82e2272535745955ac1321eb740c29f2f88b353f54e"
+checksum = "350d8e55c3b2d602a0a04389bcc1da40167657143a9922a7103190603e7b7692"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -266,9 +266,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb68113d6ecdbe8bba48b2c4042c151bf9e1c61244e45072a50250a6fc59bafe"
+checksum = "c6f710d98964d2c069b8baf566130045e79e11baa105623f038a6c942f805681"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -278,9 +278,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eab4bbf2dd3078facb5ce0a9641316a64f42bfd8cf357e6775c8a5e6708e3a8d"
+checksum = "9c99787cb8fabc187285da9e7182d22f2b80ecfac61ca0a42c4299e9eecdf903"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -292,9 +292,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-json"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48c5b650d23746a494665d914a7fa3d21d939153cff9d53bdebe39bffa88f263"
+checksum = "91c95a58ce63f60d80d7a3a1222d65df0bc060b71d31353c34a8118c2a6eae7b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -312,9 +312,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ord"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68c6fce28e5011e30acc7466b5efcb8ed0197c396240bd2b10e167f275a3c208"
+checksum = "4141e6488610cc144e841da3de5f5371488f3cf5bc6bc7b3e752c64e7639c31b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -327,9 +327,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f20a421f19799d8b93eb8edde5217e910fa1e2d6ceb3c529f000e57b6db144c0"
+checksum = "940191a3c636c111c41e816325b0941484bf904c46de72cd9553acd1afd24d33"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -342,15 +342,15 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc85923d8d6662cc66ac6602c7d1876872e671002d60993dfdf492a6badeae92"
+checksum = "18c41d058b2895a12f46dfafc306ee3529ad9660406be0ab8a7967d5e27c417e"
 
 [[package]]
 name = "arrow-select"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6ab6613ce65b61d85a3410241744e84e48fbab0fe06e1251b4429d21b3470fd"
+checksum = "9fcbdda2772b7e712e77444f3a71f4ee517095aceb993b35de71de41c70d9b4f"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -361,9 +361,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-string"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3008641239e884aefba66d8b8532da6af40d14296349fcc85935de4ba67b89e"
+checksum = "7081c34f4b534ad320a03db79d58e38972041bb7c65686b98bbcc2f9a67a9cee"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -371,7 +371,7 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "regex",
- "regex-syntax 0.6.29",
+ "regex-syntax",
 ]
 
 [[package]]
@@ -1157,14 +1157,17 @@ dependencies = [
 
 [[package]]
 name = "deltalake"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e73bb429a72cd494c1cf3212418dfff9975c4617b408cc25a3a1fc387574c370"
+checksum = "51500e2a5f31c9d64435297f9f8a227dac90a04814596e8243a5f401bd8aee33"
 dependencies = [
  "arrow",
  "arrow-array",
  "arrow-cast",
+ "arrow-ord",
+ "arrow-row",
  "arrow-schema",
+ "arrow-select",
  "async-trait",
  "bytes",
  "cfg-if 1.0.0",
@@ -2616,9 +2619,9 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "38.0.0"
+version = "39.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cbd51311f8d9ff3d2697b1522b18a588782e097d313a1a278b0faf2ccf2d3f6"
+checksum = "b0a1e6fa27f09ebddba280f5966ef435f3ac4d74cfc3ffe370fd3fd59c2e004d"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -2962,15 +2965,9 @@ checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.7.1",
+ "regex-syntax",
 ]
 
-[[package]]
-name = "regex-syntax"
-version = "0.6.29"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
-
 [[package]]
 name = "regex-syntax"
 version = "0.7.1"
diff --git a/Cargo.toml b/Cargo.toml
index e5ee7a5..2fae52c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,7 +32,7 @@ tokio-util = "0.6.3"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 url = "2.3"
 
-deltalake = { version = "0.11.0", features = ["json"], optional = true }
+deltalake = { version = "0.12.0", features = ["json"], optional = true }
 
 # s3 feature enabled
 dynamodb_lock = { version = "^0.4.3", optional = true }
diff --git a/src/delta_helpers.rs b/src/delta_helpers.rs
index 9862fcd..1a84776 100644
--- a/src/delta_helpers.rs
+++ b/src/delta_helpers.rs
@@ -1,7 +1,7 @@
 use crate::{DataTypeOffset, DataTypePartition};
 use deltalake::action::{Action, Add, Txn};
 use deltalake::checkpoints::CheckpointError;
-use deltalake::{DeltaDataTypeVersion, DeltaTable, DeltaTableError};
+use deltalake::{DeltaTable, DeltaTableError};
 use std::collections::HashMap;
 
 pub(crate) async fn load_table(
@@ -42,7 +42,7 @@ pub(crate) fn create_txn_action(txn_app_id: String, offset: DataTypeOffset) -> A
 
 pub(crate) async fn try_create_checkpoint(
     table: &mut DeltaTable,
-    version: DeltaDataTypeVersion,
+    version: i64,
 ) -> Result<(), CheckpointError> {
     if version % 10 == 0 {
         let table_version = table.version();
@@ -73,7 +73,7 @@ pub(crate) fn txn_app_id_for_partition(app_id: &str, partition: DataTypePartitio
 }
 
 /// Returns the last transaction version for the given transaction id recorded in the delta table.
-pub(crate) fn last_txn_version(table: &DeltaTable, txn_id: &str) -> Option<DeltaDataTypeVersion> {
+pub(crate) fn last_txn_version(table: &DeltaTable, txn_id: &str) -> Option<i64> {
     table
         .get_app_transaction_version()
         .get(txn_id)
diff --git a/src/lib.rs b/src/lib.rs
index 8bef8d8..de3e7fb 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -17,7 +17,7 @@ extern crate strum_macros;
 extern crate serde_json;
 
 use coercions::CoercionTree;
-use deltalake::{DeltaDataTypeVersion, DeltaTable, DeltaTableError};
+use deltalake::{DeltaTable, DeltaTableError};
 use futures::stream::StreamExt;
 use log::{debug, error, info, warn};
 use rdkafka::{
@@ -201,9 +201,9 @@ pub enum IngestError {
     #[error("Committed delta version {actual_version} does not match the version specified in the commit attempt {expected_version}")]
     UnexpectedVersionMismatch {
         /// The version specified in the commit attempt
-        expected_version: DeltaDataTypeVersion,
+        expected_version: i64,
         /// The version returned after the commit
-        actual_version: DeltaDataTypeVersion,
+        actual_version: i64,
     },
     /// Error returned if unable to construct a deserializer
     #[error("Unable to construct a message deserializer, source: {source}")]
diff --git a/src/offsets.rs b/src/offsets.rs
index 7bf5413..bd44492 100644
--- a/src/offsets.rs
+++ b/src/offsets.rs
@@ -1,7 +1,7 @@
 use crate::delta_helpers::*;
 use crate::{DataTypeOffset, DataTypePartition};
 use deltalake::action::Action;
-use deltalake::{DeltaDataTypeTimestamp, DeltaTable, DeltaTableError};
+use deltalake::{DeltaTable, DeltaTableError};
 use log::{error, info};
 
 /// Errors returned by `write_offsets_to_delta` function.
@@ -107,7 +107,7 @@ async fn commit_partition_offsets(
         .iter()
         .map(|(txn_id, offset)| create_txn_action(txn_id.to_string(), *offset))
         .collect();
-    let epoch_id: DeltaDataTypeTimestamp = std::time::SystemTime::now()
+    let epoch_id = std::time::SystemTime::now()
         .duration_since(std::time::UNIX_EPOCH)
         .expect("Time went backwards")
         .as_millis() as i64;
diff --git a/src/writer.rs b/src/writer.rs
index cccfb61..eed9d34 100644
--- a/src/writer.rs
+++ b/src/writer.rs
@@ -25,8 +25,7 @@ use deltalake::{
     action::{Action, Add, ColumnCountStat, ColumnValueStat, Stats},
     storage::DeltaObjectStore,
     time_utils::timestamp_to_delta_stats_string,
-    DeltaDataTypeLong, DeltaDataTypeVersion, DeltaResult, DeltaTable, DeltaTableError,
-    DeltaTableMetaData, ObjectStoreError, Schema,
+    DeltaResult, DeltaTable, DeltaTableError, DeltaTableMetaData, ObjectStoreError, Schema,
 };
 use log::{error, info, warn};
 use serde_json::{Number, Value};
@@ -570,7 +569,7 @@ impl DataWriter {
         &mut self,
         table: &mut DeltaTable,
         values: Vec<Value>,
-    ) -> Result<DeltaDataTypeVersion, Box<DataWriterError>> {
+    ) -> Result<i64, Box<DataWriterError>> {
         self.write(values).await?;
         let mut adds = self.write_parquet_files(&table.table_uri()).await?;
         let actions = adds.drain(..).map(Action::add).collect();
@@ -800,7 +799,7 @@ fn apply_null_counts_for_column(
 
             match col_struct {
                 ColumnCountStat::Value(n) => {
-                    let null_count = column.null_count() as DeltaDataTypeLong;
+                    let null_count = column.null_count() as i64;
                     let n = null_count + *n;
                     null_counts.insert(key, ColumnCountStat::Value(n));
                 }

From ec3175ec4fbce505c5ae3c2ee60e77c13a86eb1d Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Fri, 8 Dec 2023 21:59:35 -0800
Subject: [PATCH 15/18] Upgrade to deltalake 0.16.5

This change includes a lot of minor changes to compile against newer
deltalake libraries

Closes #156
---
 Cargo.lock           | 1801 ++++++++++++++++++------------------------
 Cargo.toml           |    4 +-
 src/coercions.rs     |   11 +-
 src/dead_letters.rs  |   26 +-
 src/delta_helpers.rs |    5 +-
 src/lib.rs           |   15 +-
 src/offsets.rs       |    8 +-
 src/writer.rs        |   28 +-
 8 files changed, 814 insertions(+), 1084 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index a559bd2..6702455 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -21,9 +21,9 @@ dependencies = [
 
 [[package]]
 name = "addr2line"
-version = "0.19.0"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
+checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
 dependencies = [
  "gimli",
 ]
@@ -42,22 +42,23 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
 
 [[package]]
 name = "ahash"
-version = "0.8.3"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
+checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
 dependencies = [
  "cfg-if 1.0.0",
  "const-random",
- "getrandom 0.2.9",
+ "getrandom 0.2.11",
  "once_cell",
  "version_check",
+ "zerocopy 0.7.29",
 ]
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.1"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04"
+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
 dependencies = [
  "memchr",
 ]
@@ -77,6 +78,12 @@ dependencies = [
  "alloc-no-stdlib",
 ]
 
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"
@@ -88,58 +95,57 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.3.2"
+version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44"
 dependencies = [
  "anstyle",
  "anstyle-parse",
  "anstyle-query",
  "anstyle-wincon",
  "colorchoice",
- "is-terminal",
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle"
-version = "1.0.0"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d"
+checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.0"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee"
+checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
 dependencies = [
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle-query"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
+checksum = "a3a318f1f38d2418400f8209655bfd825785afd25aa30bb7ba6cc792e4596748"
 dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "anstyle-wincon"
-version = "1.0.1"
+version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
+checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
 dependencies = [
  "anstyle",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "anyhow"
-version = "1.0.71"
+version = "1.0.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
+checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
 
 [[package]]
 name = "apache-avro"
@@ -148,7 +154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8cf4144857f9e4d7dd6cc4ba4c78efd2a46bad682b029bd0d91e76a021af1b2a"
 dependencies = [
  "byteorder",
- "digest 0.10.6",
+ "digest 0.10.7",
  "lazy_static",
  "libflate",
  "log",
@@ -162,15 +168,15 @@ dependencies = [
  "strum_macros 0.24.3",
  "thiserror",
  "typed-builder",
- "uuid 1.3.2",
- "zerocopy",
+ "uuid 1.6.1",
+ "zerocopy 0.6.5",
 ]
 
 [[package]]
 name = "arrow"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "218ca81dd088b102c0fd6687c72e73fad1ba93d2ef7b3cf9a1043b04b2c39dbf"
+checksum = "7fab9e93ba8ce88a37d5a30dce4b9913b75413dc1ac56cb5d72e5a840543f829"
 dependencies = [
  "ahash",
  "arrow-arith",
@@ -190,9 +196,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d49309fa2299ec34a709cfc9f487c41ecaead96d1ab70e21857466346bbbd690"
+checksum = "bc1d4e368e87ad9ee64f28b9577a3834ce10fe2703a26b28417d485bbbdff956"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -200,14 +206,14 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "half",
- "num 0.4.0",
+ "num 0.4.1",
 ]
 
 [[package]]
 name = "arrow-array"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7a27466d897d99654357a6d95dc0a26931d9e4306e60c14fc31a894edb86579"
+checksum = "d02efa7253ede102d45a4e802a129e83bcc3f49884cab795b1ac223918e4318d"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -215,25 +221,26 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "half",
- "hashbrown 0.13.2",
- "num 0.4.0",
+ "hashbrown",
+ "num 0.4.1",
 ]
 
 [[package]]
 name = "arrow-buffer"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9405b78106a9d767c7b97c78a70ee1b23ee51a74f5188a821a716d9a85d1af2b"
+checksum = "fda119225204141138cb0541c692fbfef0e875ba01bfdeaed09e9d354f9d6195"
 dependencies = [
+ "bytes",
  "half",
- "num 0.4.0",
+ "num 0.4.1",
 ]
 
 [[package]]
 name = "arrow-cast"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be0ec5a79a87783dc828b7ff8f89f62880b3f553bc5f5b932a82f4a1035024b4"
+checksum = "1d825d51b9968868d50bc5af92388754056796dbc62a4e25307d588a1fc84dee"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -241,15 +248,16 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "chrono",
+ "half",
  "lexical-core",
- "num 0.4.0",
+ "num 0.4.1",
 ]
 
 [[package]]
 name = "arrow-csv"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "350d8e55c3b2d602a0a04389bcc1da40167657143a9922a7103190603e7b7692"
+checksum = "43ef855dc6b126dc197f43e061d4de46b9d4c033aa51c2587657f7508242cef1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -266,21 +274,21 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6f710d98964d2c069b8baf566130045e79e11baa105623f038a6c942f805681"
+checksum = "475a4c3699c8b4095ca61cecf15da6f67841847a5f5aac983ccb9a377d02f73a"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
  "half",
- "num 0.4.0",
+ "num 0.4.1",
 ]
 
 [[package]]
 name = "arrow-ipc"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c99787cb8fabc187285da9e7182d22f2b80ecfac61ca0a42c4299e9eecdf903"
+checksum = "1248005c8ac549f869b7a840859d942bf62471479c1a2d82659d453eebcd166a"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -292,9 +300,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-json"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91c95a58ce63f60d80d7a3a1222d65df0bc060b71d31353c34a8118c2a6eae7b"
+checksum = "f03d7e3b04dd688ccec354fe449aed56b831679f03e44ee2c1cfc4045067b69c"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -305,16 +313,16 @@ dependencies = [
  "half",
  "indexmap",
  "lexical-core",
- "num 0.4.0",
+ "num 0.4.1",
  "serde",
  "serde_json",
 ]
 
 [[package]]
 name = "arrow-ord"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4141e6488610cc144e841da3de5f5371488f3cf5bc6bc7b3e752c64e7639c31b"
+checksum = "03b87aa408ea6a6300e49eb2eba0c032c88ed9dc19e0a9948489c55efdca71f4"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -322,14 +330,14 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "half",
- "num 0.4.0",
+ "num 0.4.1",
 ]
 
 [[package]]
 name = "arrow-row"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "940191a3c636c111c41e816325b0941484bf904c46de72cd9553acd1afd24d33"
+checksum = "114a348ab581e7c9b6908fcab23cb39ff9f060eb19e72b13f8fb8eaa37f65d22"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -337,48 +345,53 @@ dependencies = [
  "arrow-data",
  "arrow-schema",
  "half",
- "hashbrown 0.13.2",
+ "hashbrown",
 ]
 
 [[package]]
 name = "arrow-schema"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18c41d058b2895a12f46dfafc306ee3529ad9660406be0ab8a7967d5e27c417e"
+checksum = "5d1d179c117b158853e0101bfbed5615e86fe97ee356b4af901f1c5001e1ce4b"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "arrow-select"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fcbdda2772b7e712e77444f3a71f4ee517095aceb993b35de71de41c70d9b4f"
+checksum = "d5c71e003202e67e9db139e5278c79f5520bb79922261dfe140e4637ee8b6108"
 dependencies = [
+ "ahash",
  "arrow-array",
  "arrow-buffer",
  "arrow-data",
  "arrow-schema",
- "num 0.4.0",
+ "num 0.4.1",
 ]
 
 [[package]]
 name = "arrow-string"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7081c34f4b534ad320a03db79d58e38972041bb7c65686b98bbcc2f9a67a9cee"
+checksum = "c4cebbb282d6b9244895f4a9a912e55e57bce112554c7fa91fcec5459cb421ab"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-data",
  "arrow-schema",
  "arrow-select",
+ "num 0.4.1",
  "regex",
- "regex-syntax",
+ "regex-syntax 0.7.5",
 ]
 
 [[package]]
 name = "async-channel"
-version = "1.8.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833"
+checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35"
 dependencies = [
  "concurrent-queue",
  "event-listener",
@@ -387,20 +400,20 @@ dependencies = [
 
 [[package]]
 name = "async-trait"
-version = "0.1.68"
+version = "0.1.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
+checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.39",
 ]
 
 [[package]]
 name = "atomic_refcell"
-version = "0.1.10"
+version = "0.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79d6dc922a2792b006573f60b2648076355daeae5ce9cb59507e5908c9625d31"
+checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c"
 
 [[package]]
 name = "autocfg"
@@ -408,292 +421,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
-[[package]]
-name = "aws-config"
-version = "0.54.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c3d1e2a1f1ab3ac6c4b884e37413eaa03eb9d901e4fc68ee8f5c1d49721680e"
-dependencies = [
- "aws-credential-types",
- "aws-http",
- "aws-sdk-sso",
- "aws-sdk-sts",
- "aws-smithy-async",
- "aws-smithy-client",
- "aws-smithy-http",
- "aws-smithy-http-tower",
- "aws-smithy-json",
- "aws-smithy-types",
- "aws-types",
- "bytes",
- "hex",
- "http",
- "hyper",
- "ring",
- "time 0.3.21",
- "tokio",
- "tower",
- "tracing",
- "zeroize",
-]
-
-[[package]]
-name = "aws-credential-types"
-version = "0.54.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0696a0523a39a19087747e4dafda0362dc867531e3d72a3f195564c84e5e08"
-dependencies = [
- "aws-smithy-async",
- "aws-smithy-types",
- "tokio",
- "tracing",
- "zeroize",
-]
-
-[[package]]
-name = "aws-endpoint"
-version = "0.54.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80a4f935ab6a1919fbfd6102a80c4fccd9ff5f47f94ba154074afe1051903261"
-dependencies = [
- "aws-smithy-http",
- "aws-smithy-types",
- "aws-types",
- "http",
- "regex",
- "tracing",
-]
-
-[[package]]
-name = "aws-http"
-version = "0.54.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82976ca4e426ee9ca3ffcf919d9b2c8d14d0cd80d43cc02173737a8f07f28d4d"
-dependencies = [
- "aws-credential-types",
- "aws-smithy-http",
- "aws-smithy-types",
- "aws-types",
- "bytes",
- "http",
- "http-body",
- "lazy_static",
- "percent-encoding",
- "pin-project-lite",
- "tracing",
-]
-
-[[package]]
-name = "aws-sdk-sso"
-version = "0.24.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca0119bacf0c42f587506769390983223ba834e605f049babe514b2bd646dbb2"
-dependencies = [
- "aws-credential-types",
- "aws-endpoint",
- "aws-http",
- "aws-sig-auth",
- "aws-smithy-async",
- "aws-smithy-client",
- "aws-smithy-http",
- "aws-smithy-http-tower",
- "aws-smithy-json",
- "aws-smithy-types",
- "aws-types",
- "bytes",
- "http",
- "regex",
- "tokio-stream",
- "tower",
-]
-
-[[package]]
-name = "aws-sdk-sts"
-version = "0.24.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "270b6a33969ebfcb193512fbd5e8ee5306888ad6c6d5d775cdbfb2d50d94de26"
-dependencies = [
- "aws-credential-types",
- "aws-endpoint",
- "aws-http",
- "aws-sig-auth",
- "aws-smithy-async",
- "aws-smithy-client",
- "aws-smithy-http",
- "aws-smithy-http-tower",
- "aws-smithy-json",
- "aws-smithy-query",
- "aws-smithy-types",
- "aws-smithy-xml",
- "aws-types",
- "bytes",
- "http",
- "regex",
- "tower",
- "tracing",
-]
-
-[[package]]
-name = "aws-sig-auth"
-version = "0.54.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "660a02a98ab1af83bd8d714afbab2d502ba9b18c49e7e4cddd6bf8837ff778cb"
-dependencies = [
- "aws-credential-types",
- "aws-sigv4",
- "aws-smithy-http",
- "aws-types",
- "http",
- "tracing",
-]
-
-[[package]]
-name = "aws-sigv4"
-version = "0.54.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86529e7b64d902efea8fff52c1b2529368d04f90305cf632729e3713f6b57dc0"
-dependencies = [
- "aws-smithy-http",
- "form_urlencoded",
- "hex",
- "hmac 0.12.1",
- "http",
- "once_cell",
- "percent-encoding",
- "regex",
- "sha2 0.10.6",
- "time 0.3.21",
- "tracing",
-]
-
-[[package]]
-name = "aws-smithy-async"
-version = "0.54.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63c712a28a4f2f2139759235c08bf98aca99d4fdf1b13c78c5f95613df0a5db9"
-dependencies = [
- "futures-util",
- "pin-project-lite",
- "tokio",
- "tokio-stream",
-]
-
-[[package]]
-name = "aws-smithy-client"
-version = "0.54.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "104ca17f56cde00a10207169697dfe9c6810db339d52fb352707e64875b30a44"
-dependencies = [
- "aws-smithy-async",
- "aws-smithy-http",
- "aws-smithy-http-tower",
- "aws-smithy-types",
- "bytes",
- "fastrand",
- "http",
- "http-body",
- "hyper",
- "hyper-rustls 0.23.2",
- "lazy_static",
- "pin-project-lite",
- "tokio",
- "tower",
- "tracing",
-]
-
-[[package]]
-name = "aws-smithy-http"
-version = "0.54.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "873f316f1833add0d3aa54ed1b0cd252ddd88c792a0cf839886400099971e844"
-dependencies = [
- "aws-smithy-types",
- "bytes",
- "bytes-utils",
- "futures-core",
- "http",
- "http-body",
- "hyper",
- "once_cell",
- "percent-encoding",
- "pin-project-lite",
- "pin-utils",
- "tracing",
-]
-
-[[package]]
-name = "aws-smithy-http-tower"
-version = "0.54.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f38231d3f5dac9ac7976f44e12803add1385119ffca9e5f050d8e980733d164"
-dependencies = [
- "aws-smithy-http",
- "aws-smithy-types",
- "bytes",
- "http",
- "http-body",
- "pin-project-lite",
- "tower",
- "tracing",
-]
-
-[[package]]
-name = "aws-smithy-json"
-version = "0.54.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bd83ff2b79e9f729746fcc8ad798676b68fe6ea72986571569a5306a277a182"
-dependencies = [
- "aws-smithy-types",
-]
-
-[[package]]
-name = "aws-smithy-query"
-version = "0.54.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2f0445dafe9d2cd50b44339ae3c3ed46549aad8ac696c52ad660b3e7ae8682b"
-dependencies = [
- "aws-smithy-types",
- "urlencoding",
-]
-
-[[package]]
-name = "aws-smithy-types"
-version = "0.54.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8161232eda10290f5136610a1eb9de56aceaccd70c963a26a260af20ac24794f"
-dependencies = [
- "base64-simd",
- "itoa",
- "num-integer",
- "ryu",
- "time 0.3.21",
-]
-
-[[package]]
-name = "aws-smithy-xml"
-version = "0.54.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343ffe9a9bb3f542675f4df0e0d5933513d6ad038ca3907ad1767ba690a99684"
-dependencies = [
- "xmlparser",
-]
-
-[[package]]
-name = "aws-types"
-version = "0.54.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8f15b34253b68cde08e39b0627cc6101bcca64351229484b4743392c035d057"
-dependencies = [
- "aws-credential-types",
- "aws-smithy-async",
- "aws-smithy-client",
- "aws-smithy-http",
- "aws-smithy-types",
- "http",
- "rustc_version 0.4.0",
- "tracing",
-]
-
 [[package]]
 name = "azure_core"
 version = "0.10.0"
@@ -705,7 +432,7 @@ dependencies = [
  "bytes",
  "dyn-clone",
  "futures",
- "getrandom 0.2.9",
+ "getrandom 0.2.11",
  "http-types",
  "log",
  "paste",
@@ -716,9 +443,9 @@ dependencies = [
  "serde",
  "serde-xml-rs",
  "serde_json",
- "time 0.3.21",
+ "time 0.3.30",
  "url",
- "uuid 1.3.2",
+ "uuid 1.6.1",
 ]
 
 [[package]]
@@ -740,10 +467,10 @@ dependencies = [
  "serde-xml-rs",
  "serde_derive",
  "serde_json",
- "sha2 0.10.6",
- "time 0.3.21",
+ "sha2 0.10.8",
+ "time 0.3.30",
  "url",
- "uuid 1.3.2",
+ "uuid 1.6.1",
 ]
 
 [[package]]
@@ -764,22 +491,22 @@ dependencies = [
  "serde-xml-rs",
  "serde_derive",
  "serde_json",
- "time 0.3.21",
+ "time 0.3.30",
  "url",
- "uuid 1.3.2",
+ "uuid 1.6.1",
 ]
 
 [[package]]
 name = "backtrace"
-version = "0.3.67"
+version = "0.3.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"
+checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
 dependencies = [
  "addr2line",
  "cc",
  "cfg-if 1.0.0",
  "libc",
- "miniz_oxide 0.6.2",
+ "miniz_oxide",
  "object",
  "rustc-demangle",
 ]
@@ -798,25 +525,21 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "base64"
-version = "0.21.0"
+version = "0.21.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
+checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9"
 
 [[package]]
-name = "base64-simd"
-version = "0.8.0"
+name = "bitflags"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
-dependencies = [
- "outref",
- "vsimd",
-]
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "1.3.2"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
 
 [[package]]
 name = "block-buffer"
@@ -838,9 +561,9 @@ dependencies = [
 
 [[package]]
 name = "brotli"
-version = "3.3.4"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
+checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -849,9 +572,9 @@ dependencies = [
 
 [[package]]
 name = "brotli-decompressor"
-version = "2.3.4"
+version = "2.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744"
+checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -859,39 +582,30 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.12.2"
+version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c6ed94e98ecff0c12dd1b04c15ec0d7d9458ca8fe806cea6f12954efe74c63b"
+checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
 
 [[package]]
 name = "byteorder"
-version = "1.4.3"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
-
-[[package]]
-name = "bytes-utils"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9"
-dependencies = [
- "bytes",
- "either",
-]
+checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
 
 [[package]]
 name = "cc"
-version = "1.0.79"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
 dependencies = [
  "jobserver",
+ "libc",
 ]
 
 [[package]]
@@ -908,47 +622,45 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.24"
+version = "0.4.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b"
+checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
 dependencies = [
+ "android-tzdata",
  "iana-time-zone",
  "js-sys",
- "num-integer",
  "num-traits",
  "serde",
- "time 0.1.45",
  "wasm-bindgen",
- "winapi",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]
 name = "clap"
-version = "4.3.0"
+version = "4.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc"
+checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2"
 dependencies = [
  "clap_builder",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.3.0"
+version = "4.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990"
+checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb"
 dependencies = [
  "anstream",
  "anstyle",
- "bitflags",
  "clap_lex",
  "strsim",
 ]
 
 [[package]]
 name = "clap_lex"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
+checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
 
 [[package]]
 name = "cloudabi"
@@ -956,7 +668,7 @@ version = "0.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -967,32 +679,30 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
 [[package]]
 name = "concurrent-queue"
-version = "2.2.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c"
+checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363"
 dependencies = [
- "crossbeam-utils 0.8.15",
+ "crossbeam-utils 0.8.16",
 ]
 
 [[package]]
 name = "const-random"
-version = "0.1.15"
+version = "0.1.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e"
+checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a"
 dependencies = [
  "const-random-macro",
- "proc-macro-hack",
 ]
 
 [[package]]
 name = "const-random-macro"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb"
+checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
 dependencies = [
- "getrandom 0.2.9",
+ "getrandom 0.2.11",
  "once_cell",
- "proc-macro-hack",
  "tiny-keccak",
 ]
 
@@ -1004,9 +714,9 @@ checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935"
 
 [[package]]
 name = "core-foundation"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -1014,15 +724,15 @@ dependencies = [
 
 [[package]]
 name = "core-foundation-sys"
-version = "0.8.4"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
+checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.7"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58"
+checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0"
 dependencies = [
  "libc",
 ]
@@ -1059,9 +769,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.15"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
 dependencies = [
  "cfg-if 1.0.0",
 ]
@@ -1104,9 +814,9 @@ dependencies = [
 
 [[package]]
 name = "csv"
-version = "1.2.1"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad"
+checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
 dependencies = [
  "csv-core",
  "itoa",
@@ -1116,9 +826,9 @@ dependencies = [
 
 [[package]]
 name = "csv-core"
-version = "0.1.10"
+version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
+checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
 dependencies = [
  "memchr",
 ]
@@ -1134,15 +844,15 @@ dependencies = [
 
 [[package]]
 name = "dashmap"
-version = "5.4.0"
+version = "5.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"
+checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
 dependencies = [
  "cfg-if 1.0.0",
- "hashbrown 0.12.3",
- "lock_api 0.4.9",
+ "hashbrown",
+ "lock_api 0.4.11",
  "once_cell",
- "parking_lot_core 0.9.7",
+ "parking_lot_core 0.9.9",
 ]
 
 [[package]]
@@ -1157,12 +867,13 @@ dependencies = [
 
 [[package]]
 name = "deltalake"
-version = "0.12.0"
+version = "0.16.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51500e2a5f31c9d64435297f9f8a227dac90a04814596e8243a5f401bd8aee33"
+checksum = "cb694b21358cfa35ec1ccf9443269d6e21a9afba5e942dceb50019748271e811"
 dependencies = [
  "arrow",
  "arrow-array",
+ "arrow-buffer",
  "arrow-cast",
  "arrow-ord",
  "arrow-row",
@@ -1172,21 +883,22 @@ dependencies = [
  "bytes",
  "cfg-if 1.0.0",
  "chrono",
- "dynamodb_lock",
+ "dynamodb_lock 0.6.1",
  "errno",
  "futures",
- "glibc_version",
  "itertools",
  "lazy_static",
  "libc",
  "log",
  "num-bigint",
  "num-traits",
+ "num_cpus",
  "object_store",
  "once_cell",
  "parking_lot 0.12.1",
  "parquet",
  "percent-encoding",
+ "rand 0.8.5",
  "regex",
  "rusoto_core 0.47.0",
  "rusoto_credential 0.47.0",
@@ -1197,14 +909,24 @@ dependencies = [
  "thiserror",
  "tokio",
  "url",
- "uuid 1.3.2",
+ "uuid 1.6.1",
+]
+
+[[package]]
+name = "deranged"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc"
+dependencies = [
+ "powerfmt",
+ "serde",
 ]
 
 [[package]]
 name = "deunicode"
-version = "0.4.3"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "850878694b7933ca4c9569d30a34b55031b9b139ee1fc7b94a527c4ef960d690"
+checksum = "6a1abaf4d861455be59f64fd2b55606cb151fce304ede7165f410243ce96bde6"
 
 [[package]]
 name = "digest"
@@ -1217,9 +939,9 @@ dependencies = [
 
 [[package]]
 name = "digest"
-version = "0.10.6"
+version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer 0.10.4",
  "crypto-common",
@@ -1277,9 +999,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
 
 [[package]]
 name = "dyn-clone"
-version = "1.0.11"
+version = "1.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30"
+checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d"
 
 [[package]]
 name = "dynamodb_lock"
@@ -1294,29 +1016,45 @@ dependencies = [
  "rusoto_dynamodb",
  "thiserror",
  "tokio",
- "uuid 1.3.2",
+ "uuid 1.6.1",
+]
+
+[[package]]
+name = "dynamodb_lock"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c4242838952a07117a4444cdf59fdcb47f181525f2dd6ae4ca6281954d09c77"
+dependencies = [
+ "async-trait",
+ "log",
+ "maplit",
+ "rusoto_core 0.47.0",
+ "rusoto_dynamodb",
+ "thiserror",
+ "tokio",
+ "uuid 1.6.1",
 ]
 
 [[package]]
 name = "either"
-version = "1.8.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
+checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
 [[package]]
 name = "encoding_rs"
-version = "0.8.32"
+version = "0.8.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394"
+checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1"
 dependencies = [
  "cfg-if 1.0.0",
 ]
 
 [[package]]
 name = "env_logger"
-version = "0.10.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
+checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece"
 dependencies = [
  "humantime",
  "is-terminal",
@@ -1326,24 +1064,19 @@ dependencies = [
 ]
 
 [[package]]
-name = "errno"
-version = "0.3.1"
+name = "equivalent"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
-dependencies = [
- "errno-dragonfly",
- "libc",
- "windows-sys 0.48.0",
-]
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
 [[package]]
-name = "errno-dragonfly"
-version = "0.1.2"
+name = "errno"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
 dependencies = [
- "cc",
  "libc",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -1361,24 +1094,30 @@ dependencies = [
  "instant",
 ]
 
+[[package]]
+name = "fastrand"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
+
 [[package]]
 name = "flatbuffers"
-version = "23.1.21"
+version = "23.5.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77f5399c2c9c50ae9418e522842ad362f61ee48b346ac106807bd355a8a7c619"
+checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "rustc_version 0.4.0",
 ]
 
 [[package]]
 name = "flate2"
-version = "1.0.26"
+version = "1.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
+checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e"
 dependencies = [
  "crc32fast",
- "miniz_oxide 0.7.1",
+ "miniz_oxide",
 ]
 
 [[package]]
@@ -1404,18 +1143,18 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 
 [[package]]
 name = "form_urlencoded"
-version = "1.1.0"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
+checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
 dependencies = [
  "percent-encoding",
 ]
 
 [[package]]
 name = "futures"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
+checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -1428,9 +1167,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
+checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -1438,15 +1177,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
+checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
+checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -1455,9 +1194,9 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
+checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa"
 
 [[package]]
 name = "futures-lite"
@@ -1465,7 +1204,7 @@ version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce"
 dependencies = [
- "fastrand",
+ "fastrand 1.9.0",
  "futures-core",
  "futures-io",
  "memchr",
@@ -1476,32 +1215,32 @@ dependencies = [
 
 [[package]]
 name = "futures-macro"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
+checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.39",
 ]
 
 [[package]]
 name = "futures-sink"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e"
+checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817"
 
 [[package]]
 name = "futures-task"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
+checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2"
 
 [[package]]
 name = "futures-util"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
+checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -1538,9 +1277,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.9"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
+checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
 dependencies = [
  "cfg-if 1.0.0",
  "js-sys",
@@ -1551,24 +1290,15 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.27.2"
+version = "0.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4"
-
-[[package]]
-name = "glibc_version"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "803ff7635f1ab4e2c064b68a0c60da917d3d18dc8d086130f689d62ce4f1c33e"
-dependencies = [
- "regex",
-]
+checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
 
 [[package]]
 name = "h2"
-version = "0.3.19"
+version = "0.3.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782"
+checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178"
 dependencies = [
  "bytes",
  "fnv",
@@ -1579,31 +1309,26 @@ dependencies = [
  "indexmap",
  "slab",
  "tokio",
- "tokio-util 0.7.8",
+ "tokio-util 0.7.10",
  "tracing",
 ]
 
 [[package]]
 name = "half"
-version = "2.2.1"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0"
+checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872"
 dependencies = [
+ "cfg-if 1.0.0",
  "crunchy",
  "num-traits",
 ]
 
 [[package]]
 name = "hashbrown"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
-
-[[package]]
-name = "hashbrown"
-version = "0.13.2"
+version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
 
 [[package]]
 name = "heck"
@@ -1622,18 +1347,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
 
 [[package]]
 name = "hermit-abi"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "hermit-abi"
-version = "0.3.1"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
 
 [[package]]
 name = "hex"
@@ -1667,7 +1383,7 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
 dependencies = [
- "digest 0.10.6",
+ "digest 0.10.7",
 ]
 
 [[package]]
@@ -1683,9 +1399,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "0.2.9"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
+checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"
 dependencies = [
  "bytes",
  "fnv",
@@ -1731,9 +1447,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
 
 [[package]]
 name = "httpdate"
-version = "1.0.2"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "humantime"
@@ -1743,9 +1459,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "hyper"
-version = "0.14.26"
+version = "0.14.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4"
+checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -1758,7 +1474,7 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "socket2",
+ "socket2 0.4.10",
  "tokio",
  "tower-service",
  "tracing",
@@ -1776,25 +1492,24 @@ dependencies = [
  "hyper",
  "log",
  "rustls 0.19.1",
- "rustls-native-certs 0.5.0",
+ "rustls-native-certs",
  "tokio",
  "tokio-rustls 0.22.0",
- "webpki 0.21.4",
+ "webpki",
 ]
 
 [[package]]
 name = "hyper-rustls"
-version = "0.23.2"
+version = "0.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"
+checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
 dependencies = [
+ "futures-util",
  "http",
  "hyper",
- "log",
- "rustls 0.20.8",
- "rustls-native-certs 0.6.2",
+ "rustls 0.21.9",
  "tokio",
- "tokio-rustls 0.23.4",
+ "tokio-rustls 0.24.1",
 ]
 
 [[package]]
@@ -1812,16 +1527,16 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.56"
+version = "0.1.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c"
+checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
  "iana-time-zone-haiku",
  "js-sys",
  "wasm-bindgen",
- "windows",
+ "windows-core",
 ]
 
 [[package]]
@@ -1835,9 +1550,9 @@ dependencies = [
 
 [[package]]
 name = "idna"
-version = "0.3.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
+checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
 dependencies = [
  "unicode-bidi",
  "unicode-normalization",
@@ -1845,12 +1560,12 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "1.9.3"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
 dependencies = [
- "autocfg",
- "hashbrown 0.12.3",
+ "equivalent",
+ "hashbrown",
 ]
 
 [[package]]
@@ -1869,54 +1584,42 @@ dependencies = [
 ]
 
 [[package]]
-name = "integer-encoding"
-version = "3.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
-
-[[package]]
-name = "io-lifetimes"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220"
-dependencies = [
- "hermit-abi 0.3.1",
- "libc",
- "windows-sys 0.48.0",
-]
-
-[[package]]
+name = "integer-encoding"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
+
+[[package]]
 name = "ipnet"
-version = "2.7.2"
+version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f"
+checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
 
 [[package]]
 name = "is-terminal"
-version = "0.4.7"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f"
+checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
- "hermit-abi 0.3.1",
- "io-lifetimes",
+ "hermit-abi",
  "rustix",
  "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "itertools"
-version = "0.10.5"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
 dependencies = [
  "either",
 ]
 
 [[package]]
 name = "itoa"
-version = "1.0.6"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
 
 [[package]]
 name = "jmespatch"
@@ -1932,18 +1635,18 @@ dependencies = [
 
 [[package]]
 name = "jobserver"
-version = "0.1.26"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
+checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "js-sys"
-version = "0.3.62"
+version = "0.3.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68c16e1bfd491478ab155fd8b4896b86f9ede344949b641e61501e07c2b8b4d5"
+checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -1983,7 +1686,7 @@ dependencies = [
  "clap",
  "deltalake",
  "dipstick",
- "dynamodb_lock",
+ "dynamodb_lock 0.4.3",
  "env_logger",
  "futures",
  "jmespatch",
@@ -2003,7 +1706,7 @@ dependencies = [
  "strum_macros 0.20.1",
  "tempfile",
  "thiserror",
- "time 0.3.21",
+ "time 0.3.30",
  "tokio",
  "tokio-stream",
  "tokio-util 0.6.10",
@@ -2084,9 +1787,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.144"
+version = "0.2.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1"
+checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
 
 [[package]]
 name = "libflate"
@@ -2110,15 +1813,26 @@ dependencies = [
 
 [[package]]
 name = "libm"
-version = "0.2.6"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058"
+
+[[package]]
+name = "libredox"
+version = "0.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
+checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
+dependencies = [
+ "bitflags 2.4.1",
+ "libc",
+ "redox_syscall 0.4.1",
+]
 
 [[package]]
 name = "libz-sys"
-version = "1.1.9"
+version = "1.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db"
+checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b"
 dependencies = [
  "cc",
  "libc",
@@ -2128,9 +1842,9 @@ dependencies = [
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.3.7"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f"
+checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
 
 [[package]]
 name = "lock_api"
@@ -2143,9 +1857,9 @@ dependencies = [
 
 [[package]]
 name = "lock_api"
-version = "0.4.9"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
+checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -2153,12 +1867,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.17"
+version = "0.4.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
-dependencies = [
- "cfg-if 1.0.0",
-]
+checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
 
 [[package]]
 name = "lz4"
@@ -2217,9 +1928,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
 
 [[package]]
 name = "mime"
@@ -2227,15 +1938,6 @@ version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
-[[package]]
-name = "miniz_oxide"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
-dependencies = [
- "adler",
-]
-
 [[package]]
 name = "miniz_oxide"
 version = "0.7.1"
@@ -2247,23 +1949,22 @@ dependencies = [
 
 [[package]]
 name = "minreq"
-version = "2.8.0"
+version = "2.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb6c6973f78ef55d0e5fc04fdb8f9ad67c87c9e86bca0ff77b6a3102b0eb36b7"
+checksum = "cb3371dfc7b772c540da1380123674a8e20583aca99907087d990ca58cf44203"
 dependencies = [
  "log",
 ]
 
 [[package]]
 name = "mio"
-version = "0.8.6"
+version = "0.8.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
+checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09"
 dependencies = [
  "libc",
- "log",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys 0.45.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -2308,12 +2009,12 @@ dependencies = [
 
 [[package]]
 name = "num"
-version = "0.4.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606"
+checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af"
 dependencies = [
  "num-bigint",
- "num-complex 0.4.3",
+ "num-complex 0.4.4",
  "num-integer",
  "num-iter",
  "num-rational 0.4.1",
@@ -2322,9 +2023,9 @@ dependencies = [
 
 [[package]]
 name = "num-bigint"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
+checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
 dependencies = [
  "autocfg",
  "num-integer",
@@ -2343,9 +2044,9 @@ dependencies = [
 
 [[package]]
 name = "num-complex"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d"
+checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214"
 dependencies = [
  "num-traits",
 ]
@@ -2396,9 +2097,9 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.15"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
 dependencies = [
  "autocfg",
  "libm",
@@ -2406,11 +2107,11 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.15.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi 0.2.6",
+ "hermit-abi",
  "libc",
 ]
 
@@ -2446,34 +2147,33 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.30.3"
+version = "0.32.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439"
+checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "object_store"
-version = "0.5.6"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec9cd6ca25e796a49fa242876d1c4de36a24a6da5258e9f0bc062dbf5e81c53b"
+checksum = "f930c88a43b1c3f6e776dfe495b4afab89882dbc81530c632db2ed65451ebcb4"
 dependencies = [
  "async-trait",
- "aws-config",
- "aws-credential-types",
- "aws-types",
- "base64 0.21.0",
+ "base64 0.21.5",
  "bytes",
  "chrono",
  "futures",
+ "humantime",
+ "hyper",
  "itertools",
  "parking_lot 0.12.1",
  "percent-encoding",
  "quick-xml",
  "rand 0.8.5",
  "reqwest",
- "ring",
+ "ring 0.16.20",
  "serde",
  "serde_json",
  "snafu",
@@ -2485,9 +2185,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.17.1"
+version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
 [[package]]
 name = "opaque-debug"
@@ -2497,11 +2197,11 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "openssl"
-version = "0.10.52"
+version = "0.10.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56"
+checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45"
 dependencies = [
- "bitflags",
+ "bitflags 2.4.1",
  "cfg-if 1.0.0",
  "foreign-types",
  "libc",
@@ -2518,7 +2218,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -2529,18 +2229,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-src"
-version = "111.25.3+1.1.1t"
+version = "300.1.6+3.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "924757a6a226bf60da5f7dd0311a34d2b52283dd82ddeb103208ddc66362f80c"
+checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.87"
+version = "0.9.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e"
+checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b"
 dependencies = [
  "cc",
  "libc",
@@ -2551,24 +2251,18 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "2.10.0"
+version = "2.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87"
+checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c"
 dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "outref"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a"
-
 [[package]]
 name = "parking"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e"
+checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
 
 [[package]]
 name = "parking_lot"
@@ -2586,8 +2280,8 @@ version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
 dependencies = [
- "lock_api 0.4.9",
- "parking_lot_core 0.9.7",
+ "lock_api 0.4.11",
+ "parking_lot_core 0.9.9",
 ]
 
 [[package]]
@@ -2606,22 +2300,22 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.7"
+version = "0.9.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
+checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
- "redox_syscall 0.2.16",
+ "redox_syscall 0.4.1",
  "smallvec",
- "windows-sys 0.45.0",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]
 name = "parquet"
-version = "39.0.0"
+version = "47.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0a1e6fa27f09ebddba280f5966ef435f3ac4d74cfc3ffe370fd3fd59c2e004d"
+checksum = "0463cc3b256d5f50408c49a4be3a16674f4c8ceef60941709620a062b1f6bf4d"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -2631,15 +2325,15 @@ dependencies = [
  "arrow-ipc",
  "arrow-schema",
  "arrow-select",
- "base64 0.21.0",
+ "base64 0.21.5",
  "brotli",
  "bytes",
  "chrono",
  "flate2",
  "futures",
- "hashbrown 0.13.2",
+ "hashbrown",
  "lz4",
- "num 0.4.0",
+ "num 0.4.1",
  "num-bigint",
  "object_store",
  "paste",
@@ -2654,15 +2348,15 @@ dependencies = [
 
 [[package]]
 name = "paste"
-version = "1.0.12"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
+checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
 
 [[package]]
 name = "percent-encoding"
-version = "2.2.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
+checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
 [[package]]
 name = "phf"
@@ -2704,29 +2398,29 @@ dependencies = [
 
 [[package]]
 name = "pin-project"
-version = "1.0.12"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc"
+checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.0.12"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"
+checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.39",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.9"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
+checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
 
 [[package]]
 name = "pin-utils"
@@ -2740,6 +2434,12 @@ version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -2764,18 +2464,18 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.56"
+version = "1.0.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
+checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "psl"
-version = "2.1.4"
+version = "2.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a1be0afcd844b15cfce18bf8cccf2dfa887a00a6454a9ea135f122b948cee91"
+checksum = "cdc9f7a25d234ba11af714be527b60c8bc7ce1310011ef8dc032bd5ab1a7eadd"
 dependencies = [
  "psl-types",
 ]
@@ -2794,9 +2494,9 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88"
 
 [[package]]
 name = "quick-xml"
-version = "0.28.2"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1"
+checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956"
 dependencies = [
  "memchr",
  "serde",
@@ -2804,9 +2504,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.27"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
 dependencies = [
  "proc-macro2",
 ]
@@ -2871,7 +2571,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.9",
+ "getrandom 0.2.11",
 ]
 
 [[package]]
@@ -2911,9 +2611,9 @@ dependencies = [
 
 [[package]]
 name = "rdkafka-sys"
-version = "4.4.0+1.9.2"
+version = "4.7.0+2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87ac9d87c3aba1748e3112318459f2ac8bff80bfff7359e338e0463549590249"
+checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615"
 dependencies = [
  "libc",
  "libz-sys",
@@ -2930,57 +2630,66 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.16"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
-name = "redox_syscall"
-version = "0.3.5"
+name = "redox_users"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
+checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
 dependencies = [
- "bitflags",
+ "getrandom 0.2.11",
+ "libredox",
+ "thiserror",
 ]
 
 [[package]]
-name = "redox_users"
-version = "0.4.3"
+name = "regex"
+version = "1.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
+checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
 dependencies = [
- "getrandom 0.2.9",
- "redox_syscall 0.2.16",
- "thiserror",
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax 0.8.2",
 ]
 
 [[package]]
-name = "regex"
-version = "1.8.1"
+name = "regex-automata"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370"
+checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax",
+ "regex-syntax 0.8.2",
 ]
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.1"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
 
 [[package]]
 name = "reqwest"
-version = "0.11.17"
+version = "0.11.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91"
+checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b"
 dependencies = [
- "base64 0.21.0",
+ "base64 0.21.5",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -2989,7 +2698,7 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
- "hyper-rustls 0.23.2",
+ "hyper-rustls 0.24.2",
  "hyper-tls",
  "ipnet",
  "js-sys",
@@ -2999,15 +2708,16 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
- "rustls 0.20.8",
+ "rustls 0.21.9",
  "rustls-pemfile",
  "serde",
  "serde_json",
  "serde_urlencoded",
+ "system-configuration",
  "tokio",
  "tokio-native-tls",
- "tokio-rustls 0.23.4",
- "tokio-util 0.7.8",
+ "tokio-rustls 0.24.1",
+ "tokio-util 0.7.10",
  "tower-service",
  "url",
  "wasm-bindgen",
@@ -3027,12 +2737,26 @@ dependencies = [
  "cc",
  "libc",
  "once_cell",
- "spin",
- "untrusted",
+ "spin 0.5.2",
+ "untrusted 0.7.1",
  "web-sys",
  "winapi",
 ]
 
+[[package]]
+name = "ring"
+version = "0.17.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74"
+dependencies = [
+ "cc",
+ "getrandom 0.2.11",
+ "libc",
+ "spin 0.9.8",
+ "untrusted 0.9.0",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "rle-decode-fast"
 version = "1.0.3"
@@ -3120,7 +2844,7 @@ dependencies = [
  "hyper",
  "serde",
  "serde_json",
- "shlex 1.1.0",
+ "shlex 1.2.0",
  "tokio",
  "zeroize",
 ]
@@ -3239,21 +2963,20 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
 dependencies = [
- "semver 1.0.17",
+ "semver 1.0.20",
 ]
 
 [[package]]
 name = "rustix"
-version = "0.37.19"
+version = "0.38.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d"
+checksum = "bfeae074e687625746172d639330f1de242a178bf3189b51e35a7a21573513ac"
 dependencies = [
- "bitflags",
+ "bitflags 2.4.1",
  "errno",
- "io-lifetimes",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -3264,21 +2987,21 @@ checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7"
 dependencies = [
  "base64 0.13.1",
  "log",
- "ring",
+ "ring 0.16.20",
  "sct 0.6.1",
- "webpki 0.21.4",
+ "webpki",
 ]
 
 [[package]]
 name = "rustls"
-version = "0.20.8"
+version = "0.21.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f"
+checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9"
 dependencies = [
  "log",
- "ring",
- "sct 0.7.0",
- "webpki 0.22.0",
+ "ring 0.17.7",
+ "rustls-webpki",
+ "sct 0.7.1",
 ]
 
 [[package]]
@@ -3294,37 +3017,35 @@ dependencies = [
 ]
 
 [[package]]
-name = "rustls-native-certs"
-version = "0.6.2"
+name = "rustls-pemfile"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50"
+checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
 dependencies = [
- "openssl-probe",
- "rustls-pemfile",
- "schannel",
- "security-framework",
+ "base64 0.21.5",
 ]
 
 [[package]]
-name = "rustls-pemfile"
-version = "1.0.2"
+name = "rustls-webpki"
+version = "0.101.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b"
+checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
 dependencies = [
- "base64 0.21.0",
+ "ring 0.17.7",
+ "untrusted 0.9.0",
 ]
 
 [[package]]
 name = "rustversion"
-version = "1.0.12"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06"
+checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
 
 [[package]]
 name = "ryu"
-version = "1.0.13"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
 
 [[package]]
 name = "same-file"
@@ -3337,11 +3058,11 @@ dependencies = [
 
 [[package]]
 name = "schannel"
-version = "0.1.21"
+version = "0.1.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3"
+checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
 dependencies = [
- "windows-sys 0.42.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -3364,9 +3085,9 @@ dependencies = [
 
 [[package]]
 name = "scopeguard"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "sct"
@@ -3374,27 +3095,27 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
 dependencies = [
- "ring",
- "untrusted",
+ "ring 0.16.20",
+ "untrusted 0.7.1",
 ]
 
 [[package]]
 name = "sct"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
+checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
 dependencies = [
- "ring",
- "untrusted",
+ "ring 0.17.7",
+ "untrusted 0.9.0",
 ]
 
 [[package]]
 name = "security-framework"
-version = "2.9.0"
+version = "2.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca2855b3715770894e67cbfa3df957790aa0c9edc3bf06efa1a84d77fa0839d1"
+checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
  "core-foundation",
  "core-foundation-sys",
  "libc",
@@ -3403,9 +3124,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.9.0"
+version = "2.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7"
+checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -3422,9 +3143,9 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.17"
+version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed"
+checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
 
 [[package]]
 name = "semver-parser"
@@ -3515,15 +3236,15 @@ dependencies = [
 
 [[package]]
 name = "seq-macro"
-version = "0.3.3"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc"
+checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.163"
+version = "1.0.193"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2"
+checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
 dependencies = [
  "serde_derive",
 ]
@@ -3542,20 +3263,20 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.163"
+version = "1.0.193"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e"
+checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.39",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.96"
+version = "1.0.108"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
+checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
 dependencies = [
  "itoa",
  "ryu",
@@ -3607,7 +3328,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -3640,13 +3361,13 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.10.6"
+version = "0.10.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
+checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
 dependencies = [
  "cfg-if 1.0.0",
  "cpufeatures",
- "digest 0.10.6",
+ "digest 0.10.7",
 ]
 
 [[package]]
@@ -3657,9 +3378,9 @@ checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
 
 [[package]]
 name = "shlex"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
+checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380"
 
 [[package]]
 name = "signal-hook-registry"
@@ -3672,39 +3393,40 @@ dependencies = [
 
 [[package]]
 name = "siphasher"
-version = "0.3.10"
+version = "0.3.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
+checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
 
 [[package]]
 name = "slab"
-version = "0.4.8"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d"
+checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
 dependencies = [
  "autocfg",
 ]
 
 [[package]]
 name = "slug"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3bc762e6a4b6c6fcaade73e77f9ebc6991b676f88bb2358bddb56560f073373"
+checksum = "3bd94acec9c8da640005f8e135a39fc0372e74535e6b368b7a04b875f784c8c4"
 dependencies = [
  "deunicode",
+ "wasm-bindgen",
 ]
 
 [[package]]
 name = "smallvec"
-version = "1.10.0"
+version = "1.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"
 
 [[package]]
 name = "snafu"
-version = "0.7.4"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb0656e7e3ffb70f6c39b3c2a86332bb74aa3c679da781642590f3c1118c5045"
+checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6"
 dependencies = [
  "doc-comment",
  "snafu-derive",
@@ -3712,9 +3434,9 @@ dependencies = [
 
 [[package]]
 name = "snafu-derive"
-version = "0.7.4"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "475b3bbe5245c26f2d8a6f62d67c1f30eb9fffeccee721c45d162c3ebbdf81b2"
+checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf"
 dependencies = [
  "heck 0.4.1",
  "proc-macro2",
@@ -3724,26 +3446,42 @@ dependencies = [
 
 [[package]]
 name = "snap"
-version = "1.1.0"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831"
+checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
 
 [[package]]
 name = "socket2"
-version = "0.4.9"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
+checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d"
 dependencies = [
  "libc",
  "winapi",
 ]
 
+[[package]]
+name = "socket2"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9"
+dependencies = [
+ "libc",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "spin"
 version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
 [[package]]
 name = "standback"
 version = "0.2.17"
@@ -3870,55 +3608,76 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.15"
+version = "2.0.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822"
+checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a"
 dependencies = [
  "proc-macro2",
  "quote",
  "unicode-ident",
 ]
 
+[[package]]
+name = "system-configuration"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "tempfile"
-version = "3.5.0"
+version = "3.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998"
+checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"
 dependencies = [
  "cfg-if 1.0.0",
- "fastrand",
- "redox_syscall 0.3.5",
+ "fastrand 2.0.1",
+ "redox_syscall 0.4.1",
  "rustix",
- "windows-sys 0.45.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "termcolor"
-version = "1.2.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
+checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449"
 dependencies = [
  "winapi-util",
 ]
 
 [[package]]
 name = "thiserror"
-version = "1.0.40"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
+checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.40"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
+checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -3960,23 +3719,25 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.21"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc"
+checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5"
 dependencies = [
+ "deranged",
  "itoa",
  "libc",
  "num_threads",
+ "powerfmt",
  "serde",
  "time-core",
- "time-macros 0.2.9",
+ "time-macros 0.2.15",
 ]
 
 [[package]]
 name = "time-core"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
 
 [[package]]
 name = "time-macros"
@@ -3990,9 +3751,9 @@ dependencies = [
 
 [[package]]
 name = "time-macros"
-version = "0.2.9"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b"
+checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20"
 dependencies = [
  "time-core",
 ]
@@ -4036,11 +3797,11 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.28.1"
+version = "1.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105"
+checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9"
 dependencies = [
- "autocfg",
+ "backtrace",
  "bytes",
  "libc",
  "mio",
@@ -4048,20 +3809,20 @@ dependencies = [
  "parking_lot 0.12.1",
  "pin-project-lite",
  "signal-hook-registry",
- "socket2",
+ "socket2 0.5.5",
  "tokio-macros",
  "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.39",
 ]
 
 [[package]]
@@ -4082,18 +3843,17 @@ checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
 dependencies = [
  "rustls 0.19.1",
  "tokio",
- "webpki 0.21.4",
+ "webpki",
 ]
 
 [[package]]
 name = "tokio-rustls"
-version = "0.23.4"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
+checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
 dependencies = [
- "rustls 0.20.8",
+ "rustls 0.21.9",
  "tokio",
- "webpki 0.22.0",
 ]
 
 [[package]]
@@ -4123,9 +3883,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.8"
+version = "0.7.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
+checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15"
 dependencies = [
  "bytes",
  "futures-core",
@@ -4137,43 +3897,21 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.1"
+version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622"
+checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1"
 
 [[package]]
 name = "toml_edit"
-version = "0.19.8"
+version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13"
+checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
  "indexmap",
  "toml_datetime",
  "winnow",
 ]
 
-[[package]]
-name = "tower"
-version = "0.4.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
-dependencies = [
- "futures-core",
- "futures-util",
- "pin-project",
- "pin-project-lite",
- "tokio",
- "tower-layer",
- "tower-service",
- "tracing",
-]
-
-[[package]]
-name = "tower-layer"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
-
 [[package]]
 name = "tower-service"
 version = "0.3.2"
@@ -4182,12 +3920,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
 
 [[package]]
 name = "tracing"
-version = "0.1.37"
+version = "0.1.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
+checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
 dependencies = [
- "cfg-if 1.0.0",
- "log",
  "pin-project-lite",
  "tracing-attributes",
  "tracing-core",
@@ -4195,29 +3931,29 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.24"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74"
+checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.39",
 ]
 
 [[package]]
 name = "tracing-core"
-version = "0.1.31"
+version = "0.1.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
+checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
 dependencies = [
  "once_cell",
 ]
 
 [[package]]
 name = "try-lock"
-version = "0.2.4"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
 [[package]]
 name = "twox-hash"
@@ -4242,9 +3978,9 @@ dependencies = [
 
 [[package]]
 name = "typenum"
-version = "1.16.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
+checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
 
 [[package]]
 name = "uname"
@@ -4257,15 +3993,15 @@ dependencies = [
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.13"
+version = "0.3.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
+checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.8"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
 
 [[package]]
 name = "unicode-normalization"
@@ -4288,6 +4024,12 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
 
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
 [[package]]
 name = "uritemplate-next"
 version = "0.2.0"
@@ -4299,9 +4041,9 @@ dependencies = [
 
 [[package]]
 name = "url"
-version = "2.3.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"
+checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -4309,12 +4051,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "urlencoding"
-version = "2.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9"
-
 [[package]]
 name = "utf8parse"
 version = "0.2.1"
@@ -4337,17 +4073,17 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
 dependencies = [
- "getrandom 0.2.9",
+ "getrandom 0.2.11",
  "serde",
 ]
 
 [[package]]
 name = "uuid"
-version = "1.3.2"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2"
+checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560"
 dependencies = [
- "getrandom 0.2.9",
+ "getrandom 0.2.11",
  "serde",
 ]
 
@@ -4385,23 +4121,17 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
-[[package]]
-name = "vsimd"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
-
 [[package]]
 name = "waker-fn"
-version = "1.1.0"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
+checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690"
 
 [[package]]
 name = "walkdir"
-version = "2.3.3"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698"
+checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
 dependencies = [
  "same-file",
  "winapi-util",
@@ -4409,11 +4139,10 @@ dependencies = [
 
 [[package]]
 name = "want"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
 dependencies = [
- "log",
  "try-lock",
 ]
 
@@ -4437,9 +4166,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.85"
+version = "0.2.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b6cb788c4e39112fbe1822277ef6fb3c55cd86b95cb3d3c4c1c9597e4ac74b4"
+checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e"
 dependencies = [
  "cfg-if 1.0.0",
  "wasm-bindgen-macro",
@@ -4447,24 +4176,24 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.85"
+version = "0.2.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35e522ed4105a9d626d885b35d62501b30d9666283a5c8be12c14a8bdafe7822"
+checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.39",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.35"
+version = "0.4.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "083abe15c5d88556b77bdf7aef403625be9e327ad37c62c4e4129af740168163"
+checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12"
 dependencies = [
  "cfg-if 1.0.0",
  "js-sys",
@@ -4474,9 +4203,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.85"
+version = "0.2.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "358a79a0cb89d21db8120cbfb91392335913e4890665b1a7981d9e956903b434"
+checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -4484,28 +4213,28 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.85"
+version = "0.2.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4783ce29f09b9d93134d41297aded3a712b7b979e9c6f28c32cb88c973a94869"
+checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.15",
+ "syn 2.0.39",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.85"
+version = "0.2.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a901d592cafaa4d711bc324edfaff879ac700b19c3dfd60058d2b445be2691eb"
+checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f"
 
 [[package]]
 name = "wasm-streams"
-version = "0.2.3"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078"
+checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7"
 dependencies = [
  "futures-util",
  "js-sys",
@@ -4516,9 +4245,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.62"
+version = "0.3.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16b5f940c7edfdc6d12126d98c9ef4d1b3d470011c47c76a6581df47ad9ba721"
+checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -4530,28 +4259,15 @@ version = "0.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
 dependencies = [
- "ring",
- "untrusted",
-]
-
-[[package]]
-name = "webpki"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
-dependencies = [
- "ring",
- "untrusted",
+ "ring 0.16.20",
+ "untrusted 0.7.1",
 ]
 
 [[package]]
 name = "webpki-roots"
-version = "0.22.6"
+version = "0.25.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87"
-dependencies = [
- "webpki 0.22.0",
-]
+checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10"
 
 [[package]]
 name = "winapi"
@@ -4571,9 +4287,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.5"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
 dependencies = [
  "winapi",
 ]
@@ -4585,232 +4301,232 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
-name = "windows"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
-dependencies = [
- "windows-targets 0.48.0",
-]
-
-[[package]]
-name = "windows-sys"
-version = "0.42.0"
+name = "windows-core"
+version = "0.51.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
+checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
 dependencies = [
- "windows_aarch64_gnullvm 0.42.2",
- "windows_aarch64_msvc 0.42.2",
- "windows_i686_gnu 0.42.2",
- "windows_i686_msvc 0.42.2",
- "windows_x86_64_gnu 0.42.2",
- "windows_x86_64_gnullvm 0.42.2",
- "windows_x86_64_msvc 0.42.2",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]
 name = "windows-sys"
-version = "0.45.0"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
 dependencies = [
- "windows-targets 0.42.2",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]
 name = "windows-sys"
-version = "0.48.0"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
 dependencies = [
- "windows-targets 0.48.0",
+ "windows-targets 0.52.0",
 ]
 
 [[package]]
 name = "windows-targets"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
 dependencies = [
- "windows_aarch64_gnullvm 0.42.2",
- "windows_aarch64_msvc 0.42.2",
- "windows_i686_gnu 0.42.2",
- "windows_i686_msvc 0.42.2",
- "windows_x86_64_gnu 0.42.2",
- "windows_x86_64_gnullvm 0.42.2",
- "windows_x86_64_msvc 0.42.2",
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
 ]
 
 [[package]]
 name = "windows-targets"
-version = "0.48.0"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5"
+checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
 dependencies = [
- "windows_aarch64_gnullvm 0.48.0",
- "windows_aarch64_msvc 0.48.0",
- "windows_i686_gnu 0.48.0",
- "windows_i686_msvc 0.48.0",
- "windows_x86_64_gnu 0.48.0",
- "windows_x86_64_gnullvm 0.48.0",
- "windows_x86_64_msvc 0.48.0",
+ "windows_aarch64_gnullvm 0.52.0",
+ "windows_aarch64_msvc 0.52.0",
+ "windows_i686_gnu 0.52.0",
+ "windows_i686_msvc 0.52.0",
+ "windows_x86_64_gnu 0.52.0",
+ "windows_x86_64_gnullvm 0.52.0",
+ "windows_x86_64_msvc 0.52.0",
 ]
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.48.0"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.48.0"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.48.0"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.48.0"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.48.0"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.48.0"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.48.0"
+version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
 
 [[package]]
 name = "winnow"
-version = "0.4.6"
+version = "0.5.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699"
+checksum = "b67b5f0a4e7a27a64c651977932b9dc5667ca7fc31ac44b03ed37a0cf42fdfff"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "winreg"
-version = "0.10.1"
+version = "0.50.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
+checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
 dependencies = [
- "winapi",
+ "cfg-if 1.0.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
 name = "xml-rs"
-version = "0.8.10"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc95a04ea24f543cd9be5aab44f963fa35589c99e18415c38fb2b17e133bf8d2"
+checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a"
 
 [[package]]
-name = "xmlparser"
-version = "0.13.5"
+name = "zerocopy"
+version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd"
+checksum = "96f8f25c15a0edc9b07eb66e7e6e97d124c0505435c382fde1ab7ceb188aa956"
+dependencies = [
+ "byteorder",
+ "zerocopy-derive 0.6.5",
+]
 
 [[package]]
 name = "zerocopy"
-version = "0.6.1"
+version = "0.7.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "332f188cc1bcf1fe1064b8c58d150f497e697f49774aa846f2dc949d9a25f236"
+checksum = "5d075cf85bbb114e933343e087b92f2146bac0d55b534cbb8188becf0039948e"
 dependencies = [
- "byteorder",
- "zerocopy-derive",
+ "zerocopy-derive 0.7.29",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.3.2"
+version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3"
+checksum = "855e0f6af9cd72b87d8a6c586f3cb583f5cdcc62c2c80869d8cd7e96fdf7ee20"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.39",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86cd5ca076997b97ef09d3ad65efe811fa68c9e874cb636ccb211223a813b0c2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.39",
 ]
 
 [[package]]
 name = "zeroize"
-version = "1.6.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
+checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d"
 
 [[package]]
 name = "zstd"
-version = "0.12.3+zstd.1.5.2"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806"
+checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c"
 dependencies = [
  "zstd-safe",
 ]
 
 [[package]]
 name = "zstd-safe"
-version = "6.0.5+zstd.1.5.4"
+version = "6.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b"
+checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581"
 dependencies = [
  "libc",
  "zstd-sys",
@@ -4818,11 +4534,10 @@ dependencies = [
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.8+zstd.1.5.5"
+version = "2.0.9+zstd.1.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c"
+checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656"
 dependencies = [
  "cc",
- "libc",
  "pkg-config",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 2fae52c..f506ddb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,10 +32,10 @@ tokio-util = "0.6.3"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 url = "2.3"
 
-deltalake = { version = "0.12.0", features = ["json"], optional = true }
+deltalake = { version = "0.16.5", features = ["arrow", "json", "parquet"], optional = true }
 
 # s3 feature enabled
-dynamodb_lock = { version = "^0.4.3", optional = true }
+dynamodb_lock = { version = "=0.4.3", optional = true }
 rusoto_core = { version = "0.46", optional = true }
 rusoto_credential = { version = "0.46", optional = true }
 rusoto_s3 = { version = "0.46", optional = true }
diff --git a/src/coercions.rs b/src/coercions.rs
index 6334d44..020faf7 100644
--- a/src/coercions.rs
+++ b/src/coercions.rs
@@ -136,9 +136,14 @@ fn string_to_timestamp(string: &str) -> Option<Value> {
             e
         )
     }
-    parsed
-        .ok()
-        .map(|dt: DateTime<Utc>| Value::Number((dt.timestamp_nanos() / 1000).into()))
+    parsed.ok().map(|dt: DateTime<Utc>| {
+        Value::Number(
+            (dt.timestamp_nanos_opt()
+                .expect("Failed to turn timestamp nanoseconds")
+                / 1000)
+                .into(),
+        )
+    })
 }
 
 #[cfg(test)]
diff --git a/src/dead_letters.rs b/src/dead_letters.rs
index ae6c240..8e0a4df 100644
--- a/src/dead_letters.rs
+++ b/src/dead_letters.rs
@@ -15,7 +15,6 @@ use serde_json::Value;
 use std::collections::HashMap;
 
 use crate::{transforms::TransformError, writer::*};
-use deltalake::checkpoints::CheckpointError;
 
 #[cfg(feature = "s3")]
 mod env_vars {
@@ -47,7 +46,10 @@ impl DeadLetter {
             base64_bytes: Some(base64::encode(bytes)),
             json_string: None,
             error: Some(err),
-            timestamp: timestamp.timestamp_nanos() / 1000,
+            timestamp: timestamp
+                .timestamp_nanos_opt()
+                .expect("Failed to convert timezone to nanoseconds")
+                / 1000,
         }
     }
 
@@ -59,7 +61,10 @@ impl DeadLetter {
             base64_bytes: None,
             json_string: Some(value.to_string()),
             error: Some(err.to_string()),
-            timestamp: timestamp.timestamp_nanos() / 1000,
+            timestamp: timestamp
+                .timestamp_nanos_opt()
+                .expect("Failed to convert timezone to nanoseconds")
+                / 1000,
         }
     }
 
@@ -72,7 +77,10 @@ impl DeadLetter {
             base64_bytes: None,
             json_string: Some(value.to_string()),
             error: Some(err.to_string()),
-            timestamp: timestamp.timestamp_nanos() / 1000,
+            timestamp: timestamp
+                .timestamp_nanos_opt()
+                .expect("Failed to convert timezone to nanoseconds")
+                / 1000,
         }
     }
 
@@ -111,14 +119,6 @@ pub enum DeadLetterQueueError {
         source: TransformError,
     },
 
-    /// Error occurred when writing a delta log checkpoint.
-    #[error("CheckpointErrorError error: {source}")]
-    CheckpointErrorError {
-        /// The wrapped [`CheckpointError`]
-        #[from]
-        source: CheckpointError,
-    },
-
     /// DeltaTable returned an error.
     #[error("DeltaTable interaction failed: {source}")]
     DeltaTable {
@@ -151,7 +151,7 @@ pub(crate) struct DeadLetterQueueOptions {
 /// The [LoggingDeadLetterQueue] is intended for local development only
 /// and is not provided by the [dlq_from_opts] factory method.
 #[async_trait]
-pub(crate) trait DeadLetterQueue: Send + Sync {
+pub(crate) trait DeadLetterQueue: Send {
     /// Writes one [DeadLetter] to the [DeadLetterQueue].
     async fn write_dead_letter(
         &mut self,
diff --git a/src/delta_helpers.rs b/src/delta_helpers.rs
index 1a84776..b7c7437 100644
--- a/src/delta_helpers.rs
+++ b/src/delta_helpers.rs
@@ -1,6 +1,5 @@
 use crate::{DataTypeOffset, DataTypePartition};
-use deltalake::action::{Action, Add, Txn};
-use deltalake::checkpoints::CheckpointError;
+use deltalake::protocol::{Action, Add, Txn};
 use deltalake::{DeltaTable, DeltaTableError};
 use std::collections::HashMap;
 
@@ -43,7 +42,7 @@ pub(crate) fn create_txn_action(txn_app_id: String, offset: DataTypeOffset) -> A
 pub(crate) async fn try_create_checkpoint(
     table: &mut DeltaTable,
     version: i64,
-) -> Result<(), CheckpointError> {
+) -> Result<(), DeltaTableError> {
     if version % 10 == 0 {
         let table_version = table.version();
         // if there's new version right after current commit, then we need to reset
diff --git a/src/lib.rs b/src/lib.rs
index de3e7fb..d8f2729 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -17,6 +17,8 @@ extern crate strum_macros;
 extern crate serde_json;
 
 use coercions::CoercionTree;
+use deltalake::protocol::DeltaOperation;
+use deltalake::protocol::OutputMode;
 use deltalake::{DeltaTable, DeltaTableError};
 use futures::stream::StreamExt;
 use log::{debug, error, info, warn};
@@ -58,7 +60,6 @@ use crate::{
     writer::{DataWriter, DataWriterError},
 };
 use delta_helpers::*;
-use deltalake::checkpoints::CheckpointError;
 use rdkafka::message::BorrowedMessage;
 use std::ops::Add;
 
@@ -100,14 +101,6 @@ pub enum IngestError {
         source: Box<DataWriterError>,
     },
 
-    /// Error occurred when writing a delta log checkpoint.
-    #[error("CheckpointErrorError error: {source}")]
-    CheckpointErrorError {
-        /// The wrapped [`CheckpointError`]
-        #[from]
-        source: CheckpointError,
-    },
-
     /// Error from [`WriteOffsetsError`]
     #[error("WriteOffsets error: {source}")]
     WriteOffsets {
@@ -985,8 +978,8 @@ impl IngestProcessor {
             match deltalake::operations::transaction::commit(
                 (self.table.object_store().storage_backend()).as_ref(),
                 &actions,
-                deltalake::action::DeltaOperation::StreamingUpdate {
-                    output_mode: deltalake::action::OutputMode::Append,
+                DeltaOperation::StreamingUpdate {
+                    output_mode: OutputMode::Append,
                     query_id: self.opts.app_id.clone(),
                     epoch_id,
                 },
diff --git a/src/offsets.rs b/src/offsets.rs
index bd44492..e5b4239 100644
--- a/src/offsets.rs
+++ b/src/offsets.rs
@@ -1,6 +1,8 @@
 use crate::delta_helpers::*;
 use crate::{DataTypeOffset, DataTypePartition};
-use deltalake::action::Action;
+use deltalake::protocol::Action;
+use deltalake::protocol::DeltaOperation;
+use deltalake::protocol::OutputMode;
 use deltalake::{DeltaTable, DeltaTableError};
 use log::{error, info};
 
@@ -116,8 +118,8 @@ async fn commit_partition_offsets(
     match deltalake::operations::transaction::commit(
         (table.object_store().storage_backend()).as_ref(),
         &actions,
-        deltalake::action::DeltaOperation::StreamingUpdate {
-            output_mode: deltalake::action::OutputMode::Complete,
+        DeltaOperation::StreamingUpdate {
+            output_mode: OutputMode::Complete,
             query_id: app_id,
             epoch_id,
         },
diff --git a/src/writer.rs b/src/writer.rs
index eed9d34..ce02642 100644
--- a/src/writer.rs
+++ b/src/writer.rs
@@ -12,20 +12,22 @@ use deltalake::arrow::{
     json::reader::ReaderBuilder,
     record_batch::*,
 };
-
 use deltalake::parquet::format::FileMetaData;
 use deltalake::parquet::{
     arrow::ArrowWriter,
     basic::{Compression, LogicalType},
     errors::ParquetError,
     file::{metadata::RowGroupMetaData, properties::WriterProperties, statistics::Statistics},
+    format::TimeUnit,
     schema::types::{ColumnDescriptor, SchemaDescriptor},
 };
+use deltalake::protocol::DeltaOperation;
+use deltalake::protocol::SaveMode;
 use deltalake::{
-    action::{Action, Add, ColumnCountStat, ColumnValueStat, Stats},
+    protocol::{Action, Add, ColumnCountStat, ColumnValueStat, Stats},
     storage::DeltaObjectStore,
-    time_utils::timestamp_to_delta_stats_string,
-    DeltaResult, DeltaTable, DeltaTableError, DeltaTableMetaData, ObjectStoreError, Schema,
+    table::DeltaTableMetaData,
+    DeltaResult, DeltaTable, DeltaTableError, ObjectStoreError, Schema,
 };
 use log::{error, info, warn};
 use serde_json::{Number, Value};
@@ -576,8 +578,8 @@ impl DataWriter {
         let version = deltalake::operations::transaction::commit(
             (table.object_store().storage_backend()).as_ref(),
             &actions,
-            deltalake::action::DeltaOperation::Write {
-                mode: deltalake::action::SaveMode::Append,
+            DeltaOperation::Write {
+                mode: SaveMode::Append,
                 partition_by: Some(self.partition_columns.clone()),
                 predicate: None,
             },
@@ -1086,6 +1088,7 @@ fn create_add(
         stats: Some(stats_string),
         stats_parsed: None,
         tags: None,
+        ..Default::default()
     })
 }
 
@@ -1147,6 +1150,19 @@ fn stringified_partition_value(
     Ok(Some(s))
 }
 
+/// Vendored from delta-rs since it's no longer a public API
+fn timestamp_to_delta_stats_string(n: i64, time_unit: &TimeUnit) -> Option<String> {
+    use deltalake::arrow::temporal_conversions;
+
+    let dt = match time_unit {
+        TimeUnit::MILLIS(_) => temporal_conversions::timestamp_ms_to_datetime(n),
+        TimeUnit::MICROS(_) => temporal_conversions::timestamp_us_to_datetime(n),
+        TimeUnit::NANOS(_) => temporal_conversions::timestamp_ns_to_datetime(n),
+    }?;
+
+    Some(format!("{}", dt.format("%Y-%m-%dT%H:%M:%S%.3fZ")))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

From 0b9072d0c65cf7e71195ab175d1cbc69ed276847 Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Tue, 12 Dec 2023 15:20:05 +0000
Subject: [PATCH 16/18] build on 0.16.5 properly, test timeout still

---
 Cargo.lock                      | 317 +++-----------------------------
 Cargo.toml                      |  10 +-
 tests/delta_partitions_tests.rs |  20 +-
 tests/helpers/mod.rs            |  21 +--
 4 files changed, 55 insertions(+), 313 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 6702455..e6c316c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -439,7 +439,7 @@ dependencies = [
  "pin-project",
  "rand 0.8.5",
  "reqwest",
- "rustc_version 0.4.0",
+ "rustc_version",
  "serde",
  "serde-xml-rs",
  "serde_json",
@@ -511,12 +511,6 @@ dependencies = [
  "rustc-demangle",
 ]
 
-[[package]]
-name = "base-x"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270"
-
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -706,12 +700,6 @@ dependencies = [
  "tiny-keccak",
 ]
 
-[[package]]
-name = "const_fn"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935"
-
 [[package]]
 name = "core-foundation"
 version = "0.9.4"
@@ -792,16 +780,6 @@ dependencies = [
  "typenum",
 ]
 
-[[package]]
-name = "crypto-mac"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a"
-dependencies = [
- "generic-array",
- "subtle",
-]
-
 [[package]]
 name = "crypto-mac"
 version = "0.11.1"
@@ -883,7 +861,7 @@ dependencies = [
  "bytes",
  "cfg-if 1.0.0",
  "chrono",
- "dynamodb_lock 0.6.1",
+ "dynamodb_lock",
  "errno",
  "futures",
  "itertools",
@@ -900,8 +878,8 @@ dependencies = [
  "percent-encoding",
  "rand 0.8.5",
  "regex",
- "rusoto_core 0.47.0",
- "rusoto_credential 0.47.0",
+ "rusoto_core",
+ "rusoto_credential",
  "rusoto_dynamodb",
  "rusoto_sts",
  "serde",
@@ -985,12 +963,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "discard"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0"
-
 [[package]]
 name = "doc-comment"
 version = "0.3.3"
@@ -1003,22 +975,6 @@ version = "1.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d"
 
-[[package]]
-name = "dynamodb_lock"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fff628406c318f098017c78e203b5b6466e0610fc48be865ebb9ae0eb229b4c8"
-dependencies = [
- "async-trait",
- "log",
- "maplit",
- "rusoto_core 0.47.0",
- "rusoto_dynamodb",
- "thiserror",
- "tokio",
- "uuid 1.6.1",
-]
-
 [[package]]
 name = "dynamodb_lock"
 version = "0.6.1"
@@ -1028,7 +984,7 @@ dependencies = [
  "async-trait",
  "log",
  "maplit",
- "rusoto_core 0.47.0",
+ "rusoto_core",
  "rusoto_dynamodb",
  "thiserror",
  "tokio",
@@ -1107,7 +1063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640"
 dependencies = [
  "bitflags 1.3.2",
- "rustc_version 0.4.0",
+ "rustc_version",
 ]
 
 [[package]]
@@ -1357,23 +1313,13 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
-[[package]]
-name = "hmac"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15"
-dependencies = [
- "crypto-mac 0.10.1",
- "digest 0.9.0",
-]
-
 [[package]]
 name = "hmac"
 version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
 dependencies = [
- "crypto-mac 0.11.1",
+ "crypto-mac",
  "digest 0.9.0",
 ]
 
@@ -1686,7 +1632,7 @@ dependencies = [
  "clap",
  "deltalake",
  "dipstick",
- "dynamodb_lock 0.4.3",
+ "dynamodb_lock",
  "env_logger",
  "futures",
  "jmespatch",
@@ -1694,8 +1640,8 @@ dependencies = [
  "log",
  "maplit",
  "rdkafka",
- "rusoto_core 0.46.0",
- "rusoto_credential 0.46.0",
+ "rusoto_core",
+ "rusoto_credential",
  "rusoto_s3",
  "schema_registry_converter",
  "sentry",
@@ -2456,12 +2402,6 @@ dependencies = [
  "toml_edit",
 ]
 
-[[package]]
-name = "proc-macro-hack"
-version = "0.5.20+deprecated"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.70"
@@ -2763,31 +2703,6 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
 
-[[package]]
-name = "rusoto_core"
-version = "0.46.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02aff20978970d47630f08de5f0d04799497818d16cafee5aec90c4b4d0806cf"
-dependencies = [
- "async-trait",
- "base64 0.13.1",
- "bytes",
- "crc32fast",
- "futures",
- "http",
- "hyper",
- "hyper-tls",
- "lazy_static",
- "log",
- "rusoto_credential 0.46.0",
- "rusoto_signature 0.46.0",
- "rustc_version 0.2.3",
- "serde",
- "serde_json",
- "tokio",
- "xml-rs",
-]
-
 [[package]]
 name = "rusoto_core"
 version = "0.47.0"
@@ -2804,33 +2719,15 @@ dependencies = [
  "hyper-rustls 0.22.1",
  "lazy_static",
  "log",
- "rusoto_credential 0.47.0",
- "rusoto_signature 0.47.0",
- "rustc_version 0.4.0",
+ "rusoto_credential",
+ "rusoto_signature",
+ "rustc_version",
  "serde",
  "serde_json",
  "tokio",
  "xml-rs",
 ]
 
-[[package]]
-name = "rusoto_credential"
-version = "0.46.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e91e4c25ea8bfa6247684ff635299015845113baaa93ba8169b9e565701b58e"
-dependencies = [
- "async-trait",
- "chrono",
- "dirs-next",
- "futures",
- "hyper",
- "serde",
- "serde_json",
- "shlex 0.1.1",
- "tokio",
- "zeroize",
-]
-
 [[package]]
 name = "rusoto_credential"
 version = "0.47.0"
@@ -2844,7 +2741,7 @@ dependencies = [
  "hyper",
  "serde",
  "serde_json",
- "shlex 1.2.0",
+ "shlex",
  "tokio",
  "zeroize",
 ]
@@ -2858,49 +2755,24 @@ dependencies = [
  "async-trait",
  "bytes",
  "futures",
- "rusoto_core 0.47.0",
+ "rusoto_core",
  "serde",
  "serde_json",
 ]
 
 [[package]]
 name = "rusoto_s3"
-version = "0.46.0"
+version = "0.47.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abc3f56f14ccf91f880b9a9c2d0556d8523e8c155041c54db155b384a1dd1119"
+checksum = "048c2fe811a823ad5a9acc976e8bf4f1d910df719dcf44b15c3e96c5b7a51027"
 dependencies = [
  "async-trait",
  "bytes",
  "futures",
- "rusoto_core 0.46.0",
+ "rusoto_core",
  "xml-rs",
 ]
 
-[[package]]
-name = "rusoto_signature"
-version = "0.46.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5486e6b1673ab3e0ba1ded284fb444845fe1b7f41d13989a54dd60f62a7b2baa"
-dependencies = [
- "base64 0.13.1",
- "bytes",
- "futures",
- "hex",
- "hmac 0.10.1",
- "http",
- "hyper",
- "log",
- "md5",
- "percent-encoding",
- "pin-project-lite",
- "rusoto_credential 0.46.0",
- "rustc_version 0.2.3",
- "serde",
- "sha2 0.9.9",
- "time 0.2.27",
- "tokio",
-]
-
 [[package]]
 name = "rusoto_signature"
 version = "0.47.0"
@@ -2920,8 +2792,8 @@ dependencies = [
  "md-5",
  "percent-encoding",
  "pin-project-lite",
- "rusoto_credential 0.47.0",
- "rustc_version 0.4.0",
+ "rusoto_credential",
+ "rustc_version",
  "serde",
  "sha2 0.9.9",
  "tokio",
@@ -2937,7 +2809,7 @@ dependencies = [
  "bytes",
  "chrono",
  "futures",
- "rusoto_core 0.47.0",
+ "rusoto_core",
  "serde_urlencoded",
  "xml-rs",
 ]
@@ -2948,22 +2820,13 @@ version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
-[[package]]
-name = "rustc_version"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-dependencies = [
- "semver 0.9.0",
-]
-
 [[package]]
 name = "rustc_version"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
 dependencies = [
- "semver 1.0.20",
+ "semver",
 ]
 
 [[package]]
@@ -3132,27 +2995,12 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "semver"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
-dependencies = [
- "semver-parser",
-]
-
 [[package]]
 name = "semver"
 version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
 
-[[package]]
-name = "semver-parser"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-
 [[package]]
 name = "sentry"
 version = "0.23.0"
@@ -3190,7 +3038,7 @@ dependencies = [
  "lazy_static",
  "libc",
  "regex",
- "rustc_version 0.4.0",
+ "rustc_version",
  "sentry-core",
  "uname",
 ]
@@ -3331,21 +3179,6 @@ dependencies = [
  "syn 2.0.39",
 ]
 
-[[package]]
-name = "sha1"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770"
-dependencies = [
- "sha1_smol",
-]
-
-[[package]]
-name = "sha1_smol"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
-
 [[package]]
 name = "sha2"
 version = "0.9.9"
@@ -3370,12 +3203,6 @@ dependencies = [
  "digest 0.10.7",
 ]
 
-[[package]]
-name = "shlex"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
-
 [[package]]
 name = "shlex"
 version = "1.2.0"
@@ -3482,70 +3309,12 @@ version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
 
-[[package]]
-name = "standback"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff"
-dependencies = [
- "version_check",
-]
-
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
-[[package]]
-name = "stdweb"
-version = "0.4.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5"
-dependencies = [
- "discard",
- "rustc_version 0.2.3",
- "stdweb-derive",
- "stdweb-internal-macros",
- "stdweb-internal-runtime",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "stdweb-derive"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef"
-dependencies = [
- "proc-macro2",
- "quote",
- "serde",
- "serde_derive",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "stdweb-internal-macros"
-version = "0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11"
-dependencies = [
- "base-x",
- "proc-macro2",
- "quote",
- "serde",
- "serde_derive",
- "serde_json",
- "sha1",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "stdweb-internal-runtime"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
-
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -3702,21 +3471,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "time"
-version = "0.2.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242"
-dependencies = [
- "const_fn",
- "libc",
- "standback",
- "stdweb",
- "time-macros 0.1.1",
- "version_check",
- "winapi",
-]
-
 [[package]]
 name = "time"
 version = "0.3.30"
@@ -3730,7 +3484,7 @@ dependencies = [
  "powerfmt",
  "serde",
  "time-core",
- "time-macros 0.2.15",
+ "time-macros",
 ]
 
 [[package]]
@@ -3739,16 +3493,6 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
 
-[[package]]
-name = "time-macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1"
-dependencies = [
- "proc-macro-hack",
- "time-macros-impl",
-]
-
 [[package]]
 name = "time-macros"
 version = "0.2.15"
@@ -3758,19 +3502,6 @@ dependencies = [
  "time-core",
 ]
 
-[[package]]
-name = "time-macros-impl"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f"
-dependencies = [
- "proc-macro-hack",
- "proc-macro2",
- "quote",
- "standback",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "tiny-keccak"
 version = "2.0.2"
diff --git a/Cargo.toml b/Cargo.toml
index f506ddb..5c0f5ae 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,7 +10,7 @@ async-trait = "0.1"
 apache-avro = "^0.14"
 base64 = "0.13"
 bytes = "1"
-chrono = "0"
+chrono = "0.4.31"
 clap = { version = "4", features = ["color"] }
 dipstick = "0.9"
 env_logger = "0"
@@ -35,10 +35,10 @@ url = "2.3"
 deltalake = { version = "0.16.5", features = ["arrow", "json", "parquet"], optional = true }
 
 # s3 feature enabled
-dynamodb_lock = { version = "=0.4.3", optional = true }
-rusoto_core = { version = "0.46", optional = true }
-rusoto_credential = { version = "0.46", optional = true }
-rusoto_s3 = { version = "0.46", optional = true }
+dynamodb_lock = { version = "0.6.0", optional = true }
+rusoto_core = { version = "0.47", default-features = false, features = ["rustls"], optional = true }
+rusoto_credential = { version = "0.47", optional = true }
+rusoto_s3 = { version = "0.47", default-features = false, features = ["rustls"], optional = true }
 
 # sentry
 sentry = { version = "0.23.0", optional = true }
diff --git a/tests/delta_partitions_tests.rs b/tests/delta_partitions_tests.rs
index d354e1d..de08236 100644
--- a/tests/delta_partitions_tests.rs
+++ b/tests/delta_partitions_tests.rs
@@ -1,7 +1,7 @@
 #[allow(dead_code)]
 mod helpers;
 
-use deltalake::action::{Action, Add};
+use deltalake::protocol::{Action, Add, DeltaOperation, SaveMode};
 use kafka_delta_ingest::writer::*;
 use serde::{Deserialize, Serialize};
 use serde_json::{json, Value};
@@ -95,9 +95,21 @@ async fn test_delta_partitions() {
         }
     }
 
-    let mut tx = table.create_transaction(None);
-    tx.add_actions(result.iter().cloned().map(Action::add).collect());
-    let version = tx.commit(None, None).await.unwrap();
+    let operation = DeltaOperation::Write {
+        mode: SaveMode::Append,
+        partition_by: None,
+        predicate: None,
+    };
+
+    let version = deltalake::operations::transaction::commit(
+        &*table.object_store(),
+        &result.iter().cloned().map(Action::add).collect(),
+        operation,
+        &table.state,
+        None,
+    )
+    .await
+    .expect("Failed to create transaction");
 
     deltalake::checkpoints::create_checkpoint(&table)
         .await
diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index 67416ec..20c2b7b 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -9,13 +9,13 @@ use std::time::Duration;
 
 use bytes::Buf;
 use chrono::prelude::*;
-use deltalake::action::{Action, Add, MetaData, Protocol, Remove, Txn};
 use deltalake::parquet::{
     file::reader::{FileReader, SerializedFileReader},
     record::RowAccessor,
 };
+use deltalake::protocol::{Action, Add, MetaData, Protocol, Remove, Txn};
 use deltalake::storage::DeltaObjectStore;
-use deltalake::{DeltaDataTypeVersion, DeltaTable, Path};
+use deltalake::{DeltaTable, Path};
 use kafka_delta_ingest::writer::load_object_store_from_uri;
 use kafka_delta_ingest::{start_ingest, IngestOptions};
 use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};
@@ -136,7 +136,9 @@ pub async fn read_files_from_store(table: &DeltaTable) -> Vec<i32> {
         let row_iter = reader.get_row_iter(None).unwrap();
 
         for record in row_iter {
-            list.push(record.get_string(0).unwrap().parse::<i32>().unwrap());
+            if let Ok(record) = record {
+                list.push(record.get_string(0).unwrap().parse::<i32>().unwrap());
+            }
         }
     }
 
@@ -386,7 +388,7 @@ pub async fn read_table_content_as<T: DeserializeOwned>(table_uri: &str) -> Vec<
 
 pub async fn read_table_content_at_version_as<T: DeserializeOwned>(
     table_uri: &str,
-    version: DeltaDataTypeVersion,
+    version: i64,
 ) -> Vec<T> {
     read_table_content_at_version_as_jsons(table_uri, version)
         .await
@@ -401,10 +403,7 @@ pub async fn read_table_content_as_jsons(table_uri: &str) -> Vec<Value> {
     json_listify_table_content(table, store).await
 }
 
-pub async fn read_table_content_at_version_as_jsons(
-    table_uri: &str,
-    version: DeltaDataTypeVersion,
-) -> Vec<Value> {
+pub async fn read_table_content_at_version_as_jsons(table_uri: &str, version: i64) -> Vec<Value> {
     let table = deltalake::open_table_with_version(table_uri, version)
         .await
         .unwrap();
@@ -426,7 +425,7 @@ async fn json_listify_table_content(table: DeltaTable, store: DeltaObjectStore)
         let row_iter = reader.get_row_iter(None).unwrap();
 
         for record in row_iter {
-            list.push(record.to_json_value());
+            list.push(record.unwrap().to_json_value());
         }
     }
 
@@ -481,7 +480,7 @@ pub async fn inspect_table(path: &str) {
                         .unwrap();
                     let reader = SerializedFileReader::new(parquet_bytes).unwrap();
                     for record in reader.get_row_iter(None).unwrap() {
-                        println!("  - {}", record.to_json_value())
+                        println!("  - {}", record.unwrap().to_json_value())
                     }
                 }
                 _ => println!("Unknown action {:?}", action),
@@ -505,7 +504,7 @@ pub async fn inspect_table(path: &str) {
             let reader = SerializedFileReader::new(bytes).unwrap();
             let mut i = 0;
             for record in reader.get_row_iter(None).unwrap() {
-                let json = record.to_json_value();
+                let json = record.unwrap().to_json_value();
                 if let Some(m) = parse_json_field::<MetaData>(&json, "metaData") {
                     println!(" {}. metaData: {}", i, m.id);
                 }

From fc9049408fdc4b506d967481c95554fd9fc2d3da Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Sun, 17 Dec 2023 18:38:05 +0000
Subject: [PATCH 17/18] Allow hitting HTTP endpoints in tests

This is being done in init_logger() which means its typically as early
as possible for most tests
---
 tests/helpers/mod.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index 20c2b7b..9cf941d 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -316,6 +316,10 @@ pub fn create_runtime(name: &str) -> Runtime {
 }
 
 pub fn init_logger() {
+    // Any time the test_aws_endpoint() is being used the ability to hit HTTP hosts
+    // needs to be enabled
+    env::set_var("AWS_ALLOW_HTTP", "true");
+
     let _ = env_logger::Builder::new()
         .format(|buf, record| {
             let thread_name = std::thread::current()

From 91172ac3c014a3dff9a65b3464cdac400f222e11 Mon Sep 17 00:00:00 2001
From: "R. Tyler Croy" <rtyler@brokenco.de>
Date: Sun, 17 Dec 2023 21:38:25 +0000
Subject: [PATCH 18/18] Adjust the measurement of the writer progress based on
 the bytes written

Versions of the parquet crate after 38 don't perform direct writes to
the underlying writer, but instead create rowgroup writers, so the
mechanism for understanding the number of bytes "written"
---
 src/cursor.rs        | 25 -------------------------
 src/lib.rs           | 12 +++++++++---
 src/writer.rs        | 13 ++++++++++++-
 tests/helpers/mod.rs |  1 +
 4 files changed, 22 insertions(+), 29 deletions(-)

diff --git a/src/cursor.rs b/src/cursor.rs
index 2b1ab40..2858e9d 100644
--- a/src/cursor.rs
+++ b/src/cursor.rs
@@ -116,37 +116,13 @@ pub(crate) struct InMemoryWriteableCursor {
 }
 
 impl InMemoryWriteableCursor {
-    /*
-    /// Consume this instance and return the underlying buffer as long as there are no other
-    /// references to this instance.
-    pub fn into_inner(self) -> Option<Vec<u8>> {
-        Arc::try_unwrap(self.buffer)
-            .ok()
-            .and_then(|mutex| mutex.into_inner().ok())
-            .map(|cursor| cursor.into_inner())
-    }*/
-
     /// Returns a clone of the underlying buffer
     pub fn data(&self) -> Vec<u8> {
         let inner = self.buffer.lock().unwrap();
         inner.get_ref().to_vec()
     }
-
-    /// Returns a length of the underlying buffer
-    pub fn len(&self) -> usize {
-        let inner = self.buffer.lock().unwrap();
-        inner.get_ref().len()
-    }
-
-    /*
-    /// Returns true if the underlying buffer contains no elements
-    pub fn is_empty(&self) -> bool {
-        let inner = self.buffer.lock().unwrap();
-        inner.get_ref().is_empty()
-    }*/
 }
 
-#[allow(deprecated)]
 impl Write for InMemoryWriteableCursor {
     fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
         let mut inner = self.buffer.lock().unwrap();
@@ -159,7 +135,6 @@ impl Write for InMemoryWriteableCursor {
     }
 }
 
-#[allow(deprecated)]
 impl Seek for InMemoryWriteableCursor {
     fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
         let mut inner = self.buffer.lock().unwrap();
diff --git a/src/lib.rs b/src/lib.rs
index d8f2729..b165125 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -382,6 +382,7 @@ pub async fn start_ingest(
 
     // The run loop
     loop {
+        debug!("running the runloop");
         // Consume the next message from the stream.
         // Timeout if the next message is not received before the next flush interval.
         let duration = ingest_processor.consume_timeout_duration();
@@ -417,6 +418,7 @@ pub async fn start_ingest(
                 // Startup can take significant time,
                 // so re-initialize the latency timer after consuming the first message.
                 if consumed == 0 {
+                    debug!("Latency timer reset ");
                     ingest_processor.latency_timer = Instant::now();
                 }
 
@@ -442,7 +444,7 @@ pub async fn start_ingest(
                 }
             }
             Err(_) => {
-                log::debug!("Latency timer expired.");
+                log::error!("Latency timer expired.");
                 // Set the latency timer expired flag to indicate that
                 // that the latency timer should be reset after flush checks.
                 latency_timer_expired = true;
@@ -506,6 +508,7 @@ pub async fn start_ingest(
         // Reset it to now so we don't run flush checks again
         // until the next appropriate interval.
         if latency_timer_expired {
+            debug!("latency timer expired, resetting");
             ingest_processor.latency_timer = Instant::now();
         }
 
@@ -1107,10 +1110,13 @@ impl IngestProcessor {
     /// Returns a boolean indicating whether a record batch should be written based on current state.
     fn should_complete_record_batch(&self) -> bool {
         let elapsed_millis = self.latency_timer.elapsed().as_millis();
+        debug!("latency_timer {:?}", self.latency_timer);
+        debug!("elapsed_millis: {:?}", elapsed_millis);
+        debug!("Value buffers has {} items", self.value_buffers.len());
 
         let should = self.value_buffers.len() > 0
-            && (self.value_buffers.len() == self.opts.max_messages_per_batch
-                || elapsed_millis >= (self.opts.allowed_latency * 1000) as u128);
+            && (self.value_buffers.len() == self.opts.max_messages_per_batch)
+            || (elapsed_millis >= (self.opts.allowed_latency * 1000) as u128);
 
         debug!(
             "Should complete record batch - latency test: {} >= {}",
diff --git a/src/writer.rs b/src/writer.rs
index ce02642..a2ad0bf 100644
--- a/src/writer.rs
+++ b/src/writer.rs
@@ -430,7 +430,18 @@ impl DataWriter {
     /// Returns the current byte length of the in memory buffer.
     /// This may be used by the caller to decide when to finalize the file write.
     pub fn buffer_len(&self) -> usize {
-        self.arrow_writers.values().map(|w| w.cursor.len()).sum()
+        self.arrow_writers
+            .values()
+            .map(|w| {
+                let l: i64 = w
+                    .arrow_writer
+                    .flushed_row_groups()
+                    .iter()
+                    .map(|rg| rg.total_byte_size())
+                    .sum();
+                l as usize
+            })
+            .sum()
     }
 
     /// Writes the existing parquet bytes to storage and resets internal state to handle another file.
diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs
index 9cf941d..335bc42 100644
--- a/tests/helpers/mod.rs
+++ b/tests/helpers/mod.rs
@@ -352,6 +352,7 @@ pub fn wait_until_file_created(path: &FilePath) {
 
         let now = Local::now();
         let poll_time = now - start_time;
+        std::thread::sleep(Duration::from_secs(1));
 
         if poll_time > chrono::Duration::seconds(180) {
             panic!("File was not created before timeout");