From 112ed86da5d2faf15644b112a4ed1d350dff8c79 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Fri, 20 Sep 2024 02:13:13 +0800
Subject: [PATCH 01/26] Save work

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock | 313 ++++++++++++++++++++++++++++++++++++++---------------
 Cargo.toml |   6 +-
 2 files changed, 230 insertions(+), 89 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 3c9cd71ff6e6b..f6fde316fa912 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -608,7 +608,7 @@ version = "0.2.6"
 source = "git+https://github.com/zhang2014/async-backtrace.git?rev=dea4553#dea4553c47ff2946684f7efb295a4105d5627fac"
 dependencies = [
  "async-backtrace-attributes",
- "dashmap",
+ "dashmap 5.5.3",
  "futures",
  "itertools 0.10.5",
  "loom",
@@ -812,7 +812,7 @@ dependencies = [
  "futures-core",
  "futures-io",
  "futures-lite 1.13.0",
- "gloo-timers",
+ "gloo-timers 0.2.6",
  "kv-log-macro",
  "log",
  "memchr",
@@ -1364,6 +1364,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "backon"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4fa97bb310c33c811334143cf64c5bb2b7b3c06e453db6b095d7061eff8f113"
+dependencies = [
+ "fastrand 2.1.0",
+ "gloo-timers 0.3.0",
+ "tokio",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.73"
@@ -3020,6 +3031,20 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "dashmap"
+version = "6.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-utils",
+ "hashbrown 0.14.5",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core 0.9.10",
+]
+
 [[package]]
 name = "data-encoding"
 version = "2.6.0"
@@ -3045,7 +3070,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "limits-rs",
  "log",
- "opendal",
+ "opendal 0.50.0",
  "serde",
  "serde_json",
  "serfig",
@@ -3116,7 +3141,7 @@ dependencies = [
  "lz4",
  "num",
  "num-traits",
- "opendal",
+ "opendal 0.50.0",
  "ordered-float 4.2.2",
  "proptest",
  "quanta 0.11.1",
@@ -3142,7 +3167,7 @@ dependencies = [
  "enum-as-inner 0.5.1",
  "ethnum",
  "fast-float",
- "fastrace",
+ "fastrace 0.6.8",
  "goldenfile",
  "indent",
  "itertools 0.10.5",
@@ -3191,7 +3216,7 @@ dependencies = [
  "databend-common-building",
  "databend-common-exception",
  "enquote",
- "fastrace",
+ "fastrace 0.6.8",
  "futures",
  "libc",
  "log",
@@ -3249,7 +3274,7 @@ dependencies = [
  "async-backtrace",
  "async-trait",
  "chrono",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-arrow",
  "databend-common-ast",
  "databend-common-base",
@@ -3361,7 +3386,7 @@ dependencies = [
  "geos",
  "geozero 0.13.0",
  "http 1.1.0",
- "opendal",
+ "opendal 0.50.0",
  "parquet",
  "paste",
  "prost 0.12.6",
@@ -3389,7 +3414,7 @@ dependencies = [
  "chrono",
  "chrono-tz 0.8.6",
  "comfy-table 6.2.0",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-arrow",
  "databend-common-ast",
  "databend-common-base",
@@ -3630,7 +3655,7 @@ dependencies = [
  "databend-common-proto-conv",
  "databend-common-storage",
  "enumflags2",
- "fastrace",
+ "fastrace 0.6.8",
  "futures",
  "log",
  "mockall",
@@ -3656,7 +3681,7 @@ dependencies = [
  "databend-common-meta-stoerr",
  "databend-common-meta-types",
  "databend-common-proto-conv",
- "fastrace",
+ "fastrace 0.6.8",
  "futures",
  "log",
  "logcall",
@@ -3693,7 +3718,7 @@ dependencies = [
  "maplit",
  "num-derive",
  "num-traits",
- "opendal",
+ "opendal 0.50.0",
  "paste",
  "prost 0.12.6",
  "serde",
@@ -3722,7 +3747,7 @@ dependencies = [
  "databend-common-metrics",
  "databend-common-tracing",
  "derive_more",
- "fastrace",
+ "fastrace 0.6.8",
  "futures",
  "itertools 0.10.5",
  "log",
@@ -3751,7 +3776,7 @@ dependencies = [
  "databend-common-meta-stoerr",
  "databend-common-meta-types",
  "databend-common-tracing",
- "fastrace",
+ "fastrace 0.6.8",
  "log",
  "tempfile",
  "test-harness",
@@ -3764,7 +3789,7 @@ dependencies = [
  "anyhow",
  "async-trait",
  "databend-common-meta-types",
- "fastrace",
+ "fastrace 0.6.8",
  "futures-util",
  "log",
  "serde",
@@ -3811,7 +3836,7 @@ dependencies = [
  "databend-common-meta-types",
  "databend-common-tracing",
  "derive_more",
- "fastrace",
+ "fastrace 0.6.8",
  "futures",
  "futures-async-stream",
  "futures-util",
@@ -3845,7 +3870,7 @@ dependencies = [
  "databend-common-meta-stoerr",
  "databend-common-meta-types",
  "databend-common-tracing",
- "fastrace",
+ "fastrace 0.6.8",
  "log",
  "openraft",
  "pretty_assertions",
@@ -3946,7 +3971,7 @@ dependencies = [
  "flate2",
  "futures",
  "lz4",
- "opendal",
+ "opendal 0.50.0",
  "parquet-format-safe",
  "rand 0.8.5",
  "seq-macro",
@@ -3967,7 +3992,7 @@ dependencies = [
  "databend-common-base",
  "databend-common-exception",
  "databend-common-expression",
- "fastrace",
+ "fastrace 0.6.8",
  "futures",
  "log",
  "petgraph",
@@ -4040,7 +4065,7 @@ dependencies = [
  "databend-common-meta-types",
  "databend-common-protos",
  "enumflags2",
- "fastrace",
+ "fastrace 0.6.8",
  "maplit",
  "num",
  "pretty_assertions",
@@ -4069,7 +4094,7 @@ dependencies = [
  "databend-common-ast",
  "databend-common-exception",
  "derive-visitor",
- "fastrace",
+ "fastrace 0.6.8",
  "goldenfile",
  "tokio",
  "unindent",
@@ -4081,7 +4106,7 @@ version = "0.1.0"
 dependencies = [
  "async-backtrace",
  "chrono-tz 0.8.6",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-ast",
  "databend-common-base",
  "databend-common-config",
@@ -4114,7 +4139,7 @@ dependencies = [
  "cidr",
  "cron",
  "ctor 0.2.8",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-ast",
  "databend-common-base",
  "databend-common-catalog",
@@ -4140,7 +4165,7 @@ dependencies = [
  "derive-visitor",
  "educe 0.4.23",
  "enum-as-inner 0.5.1",
- "fastrace",
+ "fastrace 0.6.8",
  "globiter",
  "indexmap 2.4.0",
  "itertools 0.10.5",
@@ -4148,7 +4173,7 @@ dependencies = [
  "log",
  "num-derive",
  "num-traits",
- "opendal",
+ "opendal 0.50.0",
  "parking_lot 0.12.3",
  "percent-encoding",
  "prqlc",
@@ -4172,7 +4197,7 @@ dependencies = [
  "arrow-schema",
  "async-backtrace",
  "chrono",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-arrow",
  "databend-common-auth",
  "databend-common-base",
@@ -4183,7 +4208,7 @@ dependencies = [
  "flagset",
  "futures",
  "log",
- "opendal",
+ "opendal 0.50.0",
  "parquet",
  "prometheus-client",
  "regex",
@@ -4210,7 +4235,7 @@ dependencies = [
  "databend-common-storages-parquet",
  "databend-storages-common-table-meta",
  "deltalake",
- "fastrace",
+ "fastrace 0.6.8",
  "maplit",
  "match-template",
  "object_store_opendal",
@@ -4227,7 +4252,7 @@ name = "databend-common-storages-factory"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-catalog",
  "databend-common-config",
  "databend-common-exception",
@@ -4286,14 +4311,14 @@ dependencies = [
  "databend-storages-common-session",
  "databend-storages-common-table-meta",
  "enum-as-inner 0.5.1",
- "fastrace",
+ "fastrace 0.6.8",
  "futures",
  "futures-util",
  "indexmap 2.4.0",
  "itertools 0.10.5",
  "jsonb",
  "log",
- "opendal",
+ "opendal 0.50.0",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4334,12 +4359,12 @@ dependencies = [
  "databend-storages-common-cache",
  "databend-storages-common-index",
  "databend-storages-common-table-meta",
- "fastrace",
+ "fastrace 0.6.8",
  "faststr",
  "futures",
  "hive_metastore",
  "log",
- "opendal",
+ "opendal 0.50.0",
  "recursive",
  "serde",
  "typetag",
@@ -4368,7 +4393,7 @@ dependencies = [
  "databend-common-storages-parquet",
  "databend-storages-common-pruner",
  "databend-storages-common-table-meta",
- "fastrace",
+ "fastrace 0.6.8",
  "iceberg",
  "iceberg-catalog-hms",
  "iceberg-catalog-rest",
@@ -4440,7 +4465,7 @@ dependencies = [
  "async-trait",
  "bytes",
  "chrono",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-base",
  "databend-common-catalog",
  "databend-common-exception",
@@ -4455,7 +4480,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "futures-util",
  "log",
- "opendal",
+ "opendal 0.50.0",
  "orc-rust",
  "serde",
  "serde_json",
@@ -4491,7 +4516,7 @@ dependencies = [
  "ethnum",
  "futures",
  "log",
- "opendal",
+ "opendal 0.50.0",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4538,7 +4563,7 @@ dependencies = [
  "databend-common-storages-parquet",
  "databend-storages-common-blocks",
  "databend-storages-common-table-meta",
- "opendal",
+ "opendal 0.50.0",
  "parquet",
  "serde",
  "serde_json",
@@ -4579,7 +4604,7 @@ dependencies = [
  "enum-as-inner 0.6.0",
  "futures",
  "log",
- "opendal",
+ "opendal 0.50.0",
  "parquet",
  "serde",
  "serde_json",
@@ -4603,7 +4628,7 @@ dependencies = [
  "databend-common-sql",
  "databend-common-storages-fuse",
  "databend-storages-common-table-meta",
- "fastrace",
+ "fastrace 0.6.8",
  "futures",
  "log",
 ]
@@ -4644,7 +4669,7 @@ dependencies = [
  "jsonb",
  "log",
  "once_cell",
- "opendal",
+ "opendal 0.50.0",
  "parking_lot 0.12.3",
  "regex",
  "serde",
@@ -4675,7 +4700,7 @@ dependencies = [
  "databend-common-base",
  "databend-common-exception",
  "defer",
- "fastrace",
+ "fastrace 0.6.8",
  "fastrace-opentelemetry",
  "fern",
  "itertools 0.10.5",
@@ -4856,7 +4881,7 @@ dependencies = [
  "aws-sdk-s3",
  "chrono",
  "chrono-tz 0.8.6",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-ast",
  "databend-common-base",
  "databend-common-building",
@@ -4898,7 +4923,7 @@ dependencies = [
  "jsonb",
  "jwt-simple 0.11.9",
  "log",
- "opendal",
+ "opendal 0.50.0",
  "tantivy",
  "tempfile",
 ]
@@ -4969,7 +4994,7 @@ dependencies = [
  "anyerror",
  "anyhow",
  "async-trait",
- "backon",
+ "backon 0.4.4",
  "clap",
  "databend-common-arrow",
  "databend-common-base",
@@ -4988,7 +5013,7 @@ dependencies = [
  "deepsize",
  "derive_more",
  "env_logger",
- "fastrace",
+ "fastrace 0.6.8",
  "feature-set",
  "futures",
  "futures-async-stream",
@@ -5037,7 +5062,7 @@ dependencies = [
  "databend-common-meta-types",
  "databend-common-tracing",
  "databend-meta",
- "fastrace",
+ "fastrace 0.6.8",
  "futures",
  "log",
  "rand 0.8.5",
@@ -5067,7 +5092,7 @@ dependencies = [
  "async-stream",
  "async-trait",
  "backoff",
- "backon",
+ "backon 0.4.4",
  "base64 0.21.7",
  "bumpalo",
  "byte-unit",
@@ -5077,7 +5102,7 @@ dependencies = [
  "config",
  "criterion",
  "ctor 0.2.8",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-arrow",
  "databend-common-ast",
  "databend-common-base",
@@ -5145,7 +5170,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "derive-visitor",
  "ethnum",
- "fastrace",
+ "fastrace 0.6.8",
  "flatbuffers",
  "futures",
  "futures-util",
@@ -5170,7 +5195,7 @@ dependencies = [
  "num",
  "num_cpus",
  "once_cell",
- "opendal",
+ "opendal 0.50.0",
  "opensrv-mysql",
  "opentelemetry",
  "opentelemetry_sdk",
@@ -5338,7 +5363,7 @@ dependencies = [
  "databend-common-expression",
  "databend-common-functions",
  "databend-storages-common-table-meta",
- "fastrace",
+ "fastrace 0.6.8",
  "jsonb",
  "log",
  "match-template",
@@ -5360,9 +5385,9 @@ dependencies = [
  "databend-common-base",
  "databend-common-catalog",
  "databend-common-exception",
- "fastrace",
+ "fastrace 0.6.8",
  "log",
- "opendal",
+ "opendal 0.50.0",
 ]
 
 [[package]]
@@ -5448,7 +5473,7 @@ dependencies = [
  "bytes",
  "bzip2",
  "chrono",
- "dashmap",
+ "dashmap 5.5.3",
  "datafusion-common",
  "datafusion-common-runtime",
  "datafusion-execution",
@@ -5470,7 +5495,7 @@ dependencies = [
  "itertools 0.12.1",
  "log",
  "num_cpus",
- "object_store",
+ "object_store 0.10.2",
  "parking_lot 0.12.3",
  "parquet",
  "paste",
@@ -5503,7 +5528,7 @@ dependencies = [
  "instant",
  "libc",
  "num_cpus",
- "object_store",
+ "object_store 0.10.2",
  "parquet",
  "sqlparser 0.47.0",
 ]
@@ -5525,13 +5550,13 @@ checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1"
 dependencies = [
  "arrow",
  "chrono",
- "dashmap",
+ "dashmap 5.5.3",
  "datafusion-common",
  "datafusion-expr",
  "futures",
  "hashbrown 0.14.5",
  "log",
- "object_store",
+ "object_store 0.10.2",
  "parking_lot 0.12.3",
  "rand 0.8.5",
  "tempfile",
@@ -5809,12 +5834,13 @@ checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
 
 [[package]]
 name = "delta_kernel"
-version = "0.1.1"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1ddfe35af3696786ab5f23cd995df33a66f6cff272ac1f85e09c1a6316acd4c"
+checksum = "fa08a82239f51e6d3d249c38f0f5bf7c8a78b28587e1b466893c9eac84d252d8"
 dependencies = [
  "arrow-arith",
  "arrow-array",
+ "arrow-cast",
  "arrow-json",
  "arrow-ord",
  "arrow-schema",
@@ -5832,6 +5858,7 @@ dependencies = [
  "rustc_version",
  "serde",
  "serde_json",
+ "strum 0.26.3",
  "thiserror",
  "tracing",
  "url",
@@ -5842,9 +5869,9 @@ dependencies = [
 
 [[package]]
 name = "delta_kernel_derive"
-version = "0.1.1"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4d2127a34b12919a6bce08225f0ca6fde8a19342a32675370edfc8795e7c38a"
+checksum = "ec5c4fb5b59b1bd55ed8ebcf941f27a327d600c19a4a4103546846c358be93ff"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5853,16 +5880,18 @@ dependencies = [
 
 [[package]]
 name = "deltalake"
-version = "0.18.0"
-source = "git+https://github.com/delta-io/delta-rs?rev=57795da#57795da9d9cc86a460a5888713adfb3d0584b4cc"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb19a9a5ef642da82a32eed2c85a3b1c66dd3ecdc7cc0d986ed5e484d9d17396"
 dependencies = [
  "deltalake-core",
 ]
 
 [[package]]
 name = "deltalake-core"
-version = "0.18.0"
-source = "git+https://github.com/delta-io/delta-rs?rev=57795da#57795da9d9cc86a460a5888713adfb3d0584b4cc"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "052397afa413dfcf60b040c715a955caf25d69555c7566cc447665590f73eda7"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -5879,7 +5908,7 @@ dependencies = [
  "bytes",
  "cfg-if 1.0.0",
  "chrono",
- "dashmap",
+ "dashmap 6.1.0",
  "delta_kernel",
  "either",
  "errno",
@@ -5894,7 +5923,7 @@ dependencies = [
  "num-bigint",
  "num-traits",
  "num_cpus",
- "object_store",
+ "object_store 0.10.2",
  "once_cell",
  "parking_lot 0.12.3",
  "parquet",
@@ -5905,10 +5934,12 @@ dependencies = [
  "roaring",
  "serde",
  "serde_json",
+ "sqlparser 0.51.0",
  "thiserror",
  "tokio",
  "tracing",
  "url",
+ "urlencoding",
  "uuid",
  "z85",
 ]
@@ -6530,13 +6561,28 @@ version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dfd798348670042f0628810b10bc11b767c1f42aafbd3b1ae31f282740c549fe"
 dependencies = [
- "fastrace-macro",
+ "fastrace-macro 0.6.8",
  "minstant",
  "once_cell",
  "parking_lot 0.12.3",
  "pin-project",
  "rand 0.8.5",
- "rtrb",
+ "rtrb 0.2.3",
+]
+
+[[package]]
+name = "fastrace"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25767929385a5128ff5a58c77d5fef43779bb580656801f72d4d32f7a97fea28"
+dependencies = [
+ "fastrace-macro 0.7.2",
+ "minstant",
+ "once_cell",
+ "parking_lot 0.12.3",
+ "pin-project",
+ "rand 0.8.5",
+ "rtrb 0.3.1",
 ]
 
 [[package]]
@@ -6551,13 +6597,25 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "fastrace-macro"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d11870c0168b3d10fc7a7f0b6f9319fd7244d4caacc669ae86b3ecc5b6ca1b8c"
+dependencies = [
+ "proc-macro-error 1.0.4",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.58",
+]
+
 [[package]]
 name = "fastrace-opentelemetry"
 version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05895fbf0eba689978865a110fcead4a5c04724322dc6bd73ff55fe06e39ba66"
 dependencies = [
- "fastrace",
+ "fastrace 0.6.8",
  "futures",
  "log",
  "opentelemetry",
@@ -8020,7 +8078,7 @@ version = "14.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "006acf5a613e0b5cf095d8e4b3f48c12a60d9062aa2b2dd105afaf8344a5600c"
 dependencies = [
- "dashmap",
+ "dashmap 5.5.3",
  "gix-fs",
  "libc",
  "once_cell",
@@ -8173,6 +8231,18 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "gloo-timers"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "goldenfile"
 version = "1.7.1"
@@ -8846,7 +8916,7 @@ dependencies = [
  "itertools 0.13.0",
  "murmur3",
  "once_cell",
- "opendal",
+ "opendal 0.49.0",
  "ordered-float 4.2.2",
  "parquet",
  "reqwest 0.12.5",
@@ -9681,7 +9751,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
 dependencies = [
  "cfg-if 1.0.0",
- "windows-targets 0.48.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -10734,19 +10804,40 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "object_store"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25a0c4b3a0e31f8b66f71ad8064521efa773910196e2cde791436f13409f3b45"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "chrono",
+ "futures",
+ "humantime",
+ "itertools 0.13.0",
+ "parking_lot 0.12.3",
+ "percent-encoding",
+ "snafu 0.8.4",
+ "tokio",
+ "tracing",
+ "url",
+ "walkdir",
+]
+
 [[package]]
 name = "object_store_opendal"
-version = "0.46.0"
+version = "0.47.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7e5902fc99e9fb9e32c93f6a67dc5cc0772dc0fb348e2ef4ce258b03666d034"
+checksum = "f7a27272d45c2d9d60c780f5d4d48b50f0d692e82de25835cde38bdf0e2b2a9c"
 dependencies = [
  "async-trait",
  "bytes",
  "flagset",
  "futures",
  "futures-util",
- "object_store",
- "opendal",
+ "object_store 0.11.0",
+ "opendal 0.49.0",
  "pin-project",
  "tokio",
 ]
@@ -10785,16 +10876,46 @@ name = "opendal"
 version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39d516adf7db912c38af382c3e92c27cd62fbbc240e630920555d784c2ab1494"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "backon 0.4.4",
+ "base64 0.22.1",
+ "bytes",
+ "chrono",
+ "crc32c",
+ "flagset",
+ "futures",
+ "getrandom 0.2.15",
+ "http 1.1.0",
+ "log",
+ "md-5",
+ "once_cell",
+ "percent-encoding",
+ "quick-xml 0.36.1",
+ "reqsign",
+ "reqwest 0.12.5",
+ "serde",
+ "serde_json",
+ "tokio",
+ "uuid",
+]
+
+[[package]]
+name = "opendal"
+version = "0.50.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36e44fc43be9ffe18dad3e3ef9d61c1ae01991ee6f1c8c026978c35777a711bf"
 dependencies = [
  "anyhow",
  "async-backtrace",
  "async-trait",
- "backon",
+ "backon 1.2.0",
  "base64 0.22.1",
  "bytes",
  "chrono",
  "crc32c",
- "fastrace",
+ "fastrace 0.7.2",
  "flagset",
  "futures",
  "getrandom 0.2.15",
@@ -11024,7 +11145,7 @@ dependencies = [
  "lz4_flex",
  "lzokay-native",
  "num",
- "object_store",
+ "object_store 0.10.2",
  "prost 0.12.6",
  "snafu 0.8.4",
  "snap",
@@ -11233,7 +11354,7 @@ dependencies = [
  "lz4_flex",
  "num",
  "num-bigint",
- "object_store",
+ "object_store 0.10.2",
  "paste",
  "seq-macro",
  "snap",
@@ -13164,6 +13285,12 @@ dependencies = [
  "cache-padded",
 ]
 
+[[package]]
+name = "rtrb"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3f94e84c073f3b85d4012b44722fa8842b9986d741590d4f2636ad0a5b14143"
+
 [[package]]
 name = "rust-ini"
 version = "0.18.0"
@@ -14210,6 +14337,15 @@ dependencies = [
  "sqlparser_derive",
 ]
 
+[[package]]
+name = "sqlparser"
+version = "0.51.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7"
+dependencies = [
+ "log",
+]
+
 [[package]]
 name = "sqlparser_derive"
 version = "0.2.2"
@@ -15424,8 +15560,8 @@ version = "1.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
 dependencies = [
- "cfg-if 0.1.10",
- "rand 0.7.3",
+ "cfg-if 1.0.0",
+ "rand 0.8.5",
  "static_assertions",
 ]
 
@@ -15781,7 +15917,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71bc0fbf23ebdfc8718d40ba5f31a520376767edd6ebb9fe06c27eb977379796"
 dependencies = [
  "async-broadcast",
- "dashmap",
+ "dashmap 5.5.3",
  "faststr",
  "futures",
  "lazy_static",
@@ -17036,3 +17172,8 @@ dependencies = [
  "cc",
  "pkg-config",
 ]
+
+[[patch.unused]]
+name = "deltalake"
+version = "0.18.0"
+source = "git+https://github.com/delta-io/delta-rs?rev=57795da#57795da9d9cc86a460a5888713adfb3d0584b4cc"
diff --git a/Cargo.toml b/Cargo.toml
index cf0986912747e..90633e8389d57 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -229,7 +229,7 @@ criterion = "0.5"
 ctor = "0.2"
 dashmap = "5.4.0"
 deepsize = { version = "0.2.0" }
-deltalake = "0.18"
+deltalake = "0.20"
 derive-visitor = { version = "0.4.0", features = ["std-types-drive"] }
 derive_more = "0.99.17"
 enumflags2 = { version = "0.7.7", features = ["serde"] }
@@ -249,10 +249,10 @@ jsonb = "0.4.1"
 jwt-simple = "0.11.0"
 match-template = "0.0.1"
 mysql_async = { version = "0.34", default-features = false, features = ["native-tls-tls"] }
-object_store_opendal = "0.46"
+object_store_opendal = "0.47"
 once_cell = "1.15.0"
 openai_api_rust = "0.1"
-opendal = { version = "0.49.0", features = [
+opendal = { version = "0.50.0", features = [
     "layers-fastrace",
     "layers-prometheus-client",
     "layers-async-backtrace",

From 80d924a72a1576e3fd3537bbe590d047dd832395 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 23 Oct 2024 12:36:40 +0800
Subject: [PATCH 02/26] feat: Bump OpenDAL to 0.50.1

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock                                    | 1820 +++++++++--------
 Cargo.toml                                    |    5 +-
 src/common/base/Cargo.toml                    |    2 +
 src/common/base/src/http_client.rs            |   88 +
 src/common/base/src/lib.rs                    |    1 +
 src/common/storage/Cargo.toml                 |    1 -
 src/common/storage/src/operator.rs            |   61 +-
 src/meta/app/Cargo.toml                       |    1 +
 src/meta/app/src/storage/storage_params.rs    |   32 +-
 .../fuse/operations/vacuum_temporary_files.rs |    2 +-
 .../fuse/operations/virtual_columns.rs        |    2 +-
 src/query/storages/delta/Cargo.toml           |    1 +
 src/query/storages/delta/src/table.rs         |    1 +
 .../storages/fuse/src/pruning/bloom_pruner.rs |    2 +-
 14 files changed, 1082 insertions(+), 937 deletions(-)
 create mode 100644 src/common/base/src/http_client.rs

diff --git a/Cargo.lock b/Cargo.lock
index f6fde316fa912..6093019f47e76 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -38,7 +38,7 @@ version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "cipher",
  "cpufeatures",
 ]
@@ -60,7 +60,7 @@ version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "const-random",
  "getrandom 0.2.15",
  "once_cell",
@@ -222,7 +222,7 @@ dependencies = [
  "strum 0.26.3",
  "strum_macros 0.26.4",
  "thiserror",
- "typed-builder",
+ "typed-builder 0.19.1",
  "uuid",
 ]
 
@@ -428,7 +428,7 @@ dependencies = [
  "prost 0.12.6",
  "prost-types",
  "tokio",
- "tonic",
+ "tonic 0.11.0",
 ]
 
 [[package]]
@@ -440,7 +440,7 @@ dependencies = [
  "prost 0.12.6",
  "prost-derive 0.12.6",
  "serde",
- "tonic",
+ "tonic 0.11.0",
 ]
 
 [[package]]
@@ -455,7 +455,6 @@ dependencies = [
  "arrow-data",
  "arrow-schema",
  "flatbuffers",
- "lz4_flex",
 ]
 
 [[package]]
@@ -605,7 +604,7 @@ dependencies = [
 [[package]]
 name = "async-backtrace"
 version = "0.2.6"
-source = "git+https://github.com/zhang2014/async-backtrace.git?rev=dea4553#dea4553c47ff2946684f7efb295a4105d5627fac"
+source = "git+https://github.com/datafuse-extras/async-backtrace.git?rev=dea4553#dea4553c47ff2946684f7efb295a4105d5627fac"
 dependencies = [
  "async-backtrace-attributes",
  "dashmap 5.5.3",
@@ -621,7 +620,7 @@ dependencies = [
 [[package]]
 name = "async-backtrace-attributes"
 version = "0.2.6"
-source = "git+https://github.com/zhang2014/async-backtrace.git?rev=dea4553#dea4553c47ff2946684f7efb295a4105d5627fac"
+source = "git+https://github.com/datafuse-extras/async-backtrace.git?rev=dea4553#dea4553c47ff2946684f7efb295a4105d5627fac"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -666,7 +665,7 @@ dependencies = [
 [[package]]
 name = "async-compression"
 version = "0.4.3"
-source = "git+https://github.com/everpcpc/async-compression?rev=dc81082#dc8108229e3a0288ee9097c986b89a90788fe9f6"
+source = "git+https://github.com/datafuse-extras/async-compression?rev=dc81082#dc8108229e3a0288ee9097c986b89a90788fe9f6"
 dependencies = [
  "brotli 3.5.0",
  "bzip2",
@@ -688,14 +687,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa"
 dependencies = [
  "brotli 6.0.0",
- "bzip2",
  "flate2",
  "futures-core",
- "futures-io",
  "memchr",
  "pin-project-lite",
  "tokio",
- "xz2",
  "zstd 0.13.2",
  "zstd-safe 7.2.1",
 ]
@@ -736,7 +732,7 @@ checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af"
 dependencies = [
  "async-lock 2.8.0",
  "autocfg",
- "cfg-if 1.0.0",
+ "cfg-if",
  "concurrent-queue",
  "futures-lite 1.13.0",
  "log",
@@ -755,13 +751,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "444b0228950ee6501b3568d3c93bf1176a1fdbc3b758dcd9475046d30f4dc7e8"
 dependencies = [
  "async-lock 3.4.0",
- "cfg-if 1.0.0",
+ "cfg-if",
  "concurrent-queue",
  "futures-io",
  "futures-lite 2.3.0",
  "parking",
  "polling 3.7.3",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "slab",
  "tracing",
  "windows-sys 0.59.0",
@@ -790,7 +786,7 @@ dependencies = [
 [[package]]
 name = "async-recursion"
 version = "1.1.1"
-source = "git+https://github.com/zhang2014/async-recursion.git?rev=a353334#a353334ceb408c80a802634ff0cfeeb7a50d0ef7"
+source = "git+https://github.com/datafuse-extras/async-recursion.git?rev=a353334#a353334ceb408c80a802634ff0cfeeb7a50d0ef7"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1300,7 +1296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
 dependencies = [
  "async-trait",
- "axum-core",
+ "axum-core 0.3.4",
  "bitflags 1.3.2",
  "bytes",
  "futures-util",
@@ -1316,7 +1312,34 @@ dependencies = [
  "rustversion",
  "serde",
  "sync_wrapper 0.1.2",
- "tower",
+ "tower 0.4.13",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "axum"
+version = "0.7.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae"
+dependencies = [
+ "async-trait",
+ "axum-core 0.4.5",
+ "bytes",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "http-body-util",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustversion",
+ "serde",
+ "sync_wrapper 1.0.1",
+ "tower 0.5.1",
  "tower-layer",
  "tower-service",
 ]
@@ -1338,6 +1361,26 @@ dependencies = [
  "tower-service",
 ]
 
+[[package]]
+name = "axum-core"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "http-body-util",
+ "mime",
+ "pin-project-lite",
+ "rustversion",
+ "sync_wrapper 1.0.1",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "backoff"
 version = "0.4.0"
@@ -1382,11 +1425,12 @@ source = "git+https://github.com/rust-lang/backtrace-rs.git?rev=72265be#72265bea
 dependencies = [
  "addr2line 0.22.0",
  "cc",
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "miniz_oxide",
  "object 0.36.3",
  "rustc-demangle",
+ "serde",
 ]
 
 [[package]]
@@ -1435,6 +1479,18 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
 
+[[package]]
+name = "bb8"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8"
+dependencies = [
+ "async-trait",
+ "futures-util",
+ "parking_lot 0.12.3",
+ "tokio",
+]
+
 [[package]]
 name = "beef"
 version = "0.5.2"
@@ -1555,6 +1611,9 @@ name = "bitflags"
 version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "bitmaps"
@@ -1595,15 +1654,6 @@ dependencies = [
  "wyz",
 ]
 
-[[package]]
-name = "blake2"
-version = "0.10.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
-dependencies = [
- "digest",
-]
-
 [[package]]
 name = "blake2b_simd"
 version = "1.0.2"
@@ -1624,7 +1674,7 @@ dependencies = [
  "arrayref",
  "arrayvec 0.7.4",
  "cc",
- "cfg-if 1.0.0",
+ "cfg-if",
  "constant_time_eq",
 ]
 
@@ -1701,7 +1751,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b"
 dependencies = [
  "once_cell",
- "proc-macro-crate",
+ "proc-macro-crate 3.1.0",
  "proc-macro2",
  "quote",
  "syn 2.0.58",
@@ -1886,12 +1936,6 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fdd7a427adc0135366d99db65b36dae9237130997e560ed61118041fb72be6e8"
 
-[[package]]
-name = "cache-padded"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21"
-
 [[package]]
 name = "camino"
 version = "1.1.7"
@@ -1925,7 +1969,7 @@ dependencies = [
  "io-lifetimes 2.0.3",
  "ipnet",
  "maybe-owned",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "windows-sys 0.52.0",
  "winx",
 ]
@@ -1949,7 +1993,7 @@ dependencies = [
  "cap-primitives",
  "io-extras",
  "io-lifetimes 2.0.3",
- "rustix 0.38.34",
+ "rustix 0.38.37",
 ]
 
 [[package]]
@@ -1962,7 +2006,7 @@ dependencies = [
  "cap-primitives",
  "iana-time-zone",
  "once_cell",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "winx",
 ]
 
@@ -2085,12 +2129,6 @@ dependencies = [
  "nom",
 ]
 
-[[package]]
-name = "cfg-if"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
-
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -2321,7 +2359,7 @@ version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7fc6734af48458f72f5a3fa7b840903606427d98a710256e808f76a965047d9"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "clap",
  "is-terminal",
  "libc",
@@ -2365,16 +2403,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "color-backtrace"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "150fd80a270c0671379f388c8204deb6a746bb4eac8a6c03fe2460b2c0127ea0"
-dependencies = [
- "backtrace",
- "termcolor",
-]
-
 [[package]]
 name = "color-eyre"
 version = "0.6.2"
@@ -2416,6 +2444,16 @@ dependencies = [
  "colorchoice",
 ]
 
+[[package]]
+name = "colored"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8"
+dependencies = [
+ "lazy_static",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -2594,7 +2632,7 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -2715,6 +2753,21 @@ dependencies = [
  "wasmtime-types",
 ]
 
+[[package]]
+name = "crc"
+version = "3.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636"
+dependencies = [
+ "crc-catalog",
+]
+
+[[package]]
+name = "crc-catalog"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
+
 [[package]]
 name = "crc16"
 version = "0.4.0"
@@ -2736,7 +2789,7 @@ version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -3023,7 +3076,7 @@ version = "5.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "hashbrown 0.14.5",
  "lock_api",
  "once_cell",
@@ -3037,7 +3090,7 @@ version = "6.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "crossbeam-utils",
  "hashbrown 0.14.5",
  "lock_api",
@@ -3070,7 +3123,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "limits-rs",
  "log",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "serde",
  "serde_json",
  "serfig",
@@ -3086,7 +3139,7 @@ dependencies = [
  "log",
  "once_cell",
  "percent-encoding",
- "reqwest 0.12.5",
+ "reqwest",
  "serde",
  "serde_json",
  "tokio",
@@ -3141,7 +3194,7 @@ dependencies = [
  "lz4",
  "num",
  "num-traits",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "ordered-float 4.2.2",
  "proptest",
  "quanta 0.11.1",
@@ -3167,7 +3220,7 @@ dependencies = [
  "enum-as-inner 0.5.1",
  "ethnum",
  "fast-float",
- "fastrace 0.6.8",
+ "fastrace",
  "goldenfile",
  "indent",
  "itertools 0.10.5",
@@ -3175,7 +3228,7 @@ dependencies = [
  "nom",
  "nom-rule",
  "ordered-float 4.2.2",
- "pratt 0.4.0",
+ "pratt",
  "pretty",
  "pretty_assertions",
  "recursive",
@@ -3216,7 +3269,7 @@ dependencies = [
  "databend-common-building",
  "databend-common-exception",
  "enquote",
- "fastrace 0.6.8",
+ "fastrace",
  "futures",
  "libc",
  "log",
@@ -3234,6 +3287,9 @@ dependencies = [
  "rand 0.8.5",
  "regex",
  "replace_with",
+ "reqwest",
+ "reqwest-hickory-resolver",
+ "rustix 0.38.37",
  "semver",
  "serde",
  "serde_json",
@@ -3319,9 +3375,9 @@ dependencies = [
  "prost-build",
  "semver",
  "serde",
- "tonic",
+ "tonic 0.11.0",
  "tonic-build",
- "tower",
+ "tower 0.4.13",
 ]
 
 [[package]]
@@ -3386,16 +3442,17 @@ dependencies = [
  "geos",
  "geozero 0.13.0",
  "http 1.1.0",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "parquet",
  "paste",
  "prost 0.12.6",
- "reqwest 0.12.5",
+ "reqwest",
  "serde",
  "serde_json",
+ "sqlx",
  "tantivy",
  "thiserror",
- "tonic",
+ "tonic 0.11.0",
 ]
 
 [[package]]
@@ -3451,7 +3508,7 @@ dependencies = [
  "simdutf8",
  "strength_reduce",
  "terminal_size 0.2.6",
- "tonic",
+ "tonic 0.11.0",
  "typetag",
  "unicode-segmentation",
 ]
@@ -3533,7 +3590,7 @@ dependencies = [
  "libm",
  "match-template",
  "md-5",
- "memchr",
+ "multiversion",
  "naive-cityhash",
  "num-traits",
  "once_cell",
@@ -3566,7 +3623,7 @@ dependencies = [
  "log",
  "serde",
  "thiserror",
- "tonic",
+ "tonic 0.11.0",
 ]
 
 [[package]]
@@ -3575,7 +3632,7 @@ version = "0.1.0"
 dependencies = [
  "ahash 0.8.11",
  "bumpalo",
- "cfg-if 1.0.0",
+ "cfg-if",
  "databend-common-arrow",
  "databend-common-base",
  "ethnum",
@@ -3611,6 +3668,8 @@ dependencies = [
  "chrono-tz 0.8.6",
  "databend-common-base",
  "databend-common-exception",
+ "enquote",
+ "enumflags2",
  "ethnum",
  "geo",
  "geos",
@@ -3655,7 +3714,7 @@ dependencies = [
  "databend-common-proto-conv",
  "databend-common-storage",
  "enumflags2",
- "fastrace 0.6.8",
+ "fastrace",
  "futures",
  "log",
  "mockall",
@@ -3681,8 +3740,9 @@ dependencies = [
  "databend-common-meta-stoerr",
  "databend-common-meta-types",
  "databend-common-proto-conv",
- "fastrace 0.6.8",
+ "fastrace",
  "futures",
+ "itertools 0.10.5",
  "log",
  "logcall",
  "maplit",
@@ -3692,7 +3752,7 @@ dependencies = [
  "serde_json",
  "thiserror",
  "tokio",
- "tonic",
+ "tonic 0.11.0",
 ]
 
 [[package]]
@@ -3705,6 +3765,7 @@ dependencies = [
  "chrono-tz 0.8.6",
  "cron",
  "databend-common-ast",
+ "databend-common-base",
  "databend-common-building",
  "databend-common-exception",
  "databend-common-expression",
@@ -3718,9 +3779,10 @@ dependencies = [
  "maplit",
  "num-derive",
  "num-traits",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "paste",
  "prost 0.12.6",
+ "reqwest",
  "serde",
  "serde_json",
  "sha1",
@@ -3747,7 +3809,7 @@ dependencies = [
  "databend-common-metrics",
  "databend-common-tracing",
  "derive_more",
- "fastrace 0.6.8",
+ "fastrace",
  "futures",
  "itertools 0.10.5",
  "log",
@@ -3759,7 +3821,7 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
- "tonic",
+ "tonic 0.11.0",
 ]
 
 [[package]]
@@ -3776,7 +3838,7 @@ dependencies = [
  "databend-common-meta-stoerr",
  "databend-common-meta-types",
  "databend-common-tracing",
- "fastrace 0.6.8",
+ "fastrace",
  "log",
  "tempfile",
  "test-harness",
@@ -3789,7 +3851,7 @@ dependencies = [
  "anyhow",
  "async-trait",
  "databend-common-meta-types",
- "fastrace 0.6.8",
+ "fastrace",
  "futures-util",
  "log",
  "serde",
@@ -3836,7 +3898,7 @@ dependencies = [
  "databend-common-meta-types",
  "databend-common-tracing",
  "derive_more",
- "fastrace 0.6.8",
+ "fastrace",
  "futures",
  "futures-async-stream",
  "futures-util",
@@ -3870,7 +3932,7 @@ dependencies = [
  "databend-common-meta-stoerr",
  "databend-common-meta-types",
  "databend-common-tracing",
- "fastrace 0.6.8",
+ "fastrace",
  "log",
  "openraft",
  "pretty_assertions",
@@ -3935,7 +3997,7 @@ dependencies = [
  "serde_json",
  "thiserror",
  "tokio",
- "tonic",
+ "tonic 0.11.0",
  "tonic-build",
 ]
 
@@ -3971,7 +4033,7 @@ dependencies = [
  "flate2",
  "futures",
  "lz4",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "parquet-format-safe",
  "rand 0.8.5",
  "seq-macro",
@@ -3992,7 +4054,7 @@ dependencies = [
  "databend-common-base",
  "databend-common-exception",
  "databend-common-expression",
- "fastrace 0.6.8",
+ "fastrace",
  "futures",
  "log",
  "petgraph",
@@ -4065,7 +4127,7 @@ dependencies = [
  "databend-common-meta-types",
  "databend-common-protos",
  "enumflags2",
- "fastrace 0.6.8",
+ "fastrace",
  "maplit",
  "num",
  "pretty_assertions",
@@ -4083,7 +4145,7 @@ dependencies = [
  "prost 0.12.6",
  "prost-build",
  "semver",
- "tonic",
+ "tonic 0.11.0",
  "tonic-build",
 ]
 
@@ -4094,7 +4156,7 @@ dependencies = [
  "databend-common-ast",
  "databend-common-exception",
  "derive-visitor",
- "fastrace 0.6.8",
+ "fastrace",
  "goldenfile",
  "tokio",
  "unindent",
@@ -4165,7 +4227,7 @@ dependencies = [
  "derive-visitor",
  "educe 0.4.23",
  "enum-as-inner 0.5.1",
- "fastrace 0.6.8",
+ "fastrace",
  "globiter",
  "indexmap 2.4.0",
  "itertools 0.10.5",
@@ -4173,7 +4235,7 @@ dependencies = [
  "log",
  "num-derive",
  "num-traits",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "parking_lot 0.12.3",
  "percent-encoding",
  "prqlc",
@@ -4208,12 +4270,11 @@ dependencies = [
  "flagset",
  "futures",
  "log",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "parquet",
  "prometheus-client",
  "regex",
- "reqwest 0.12.5",
- "reqwest-hickory-resolver",
+ "reqwest",
  "serde",
  "thiserror",
 ]
@@ -4235,10 +4296,11 @@ dependencies = [
  "databend-common-storages-parquet",
  "databend-storages-common-table-meta",
  "deltalake",
- "fastrace 0.6.8",
+ "fastrace",
  "maplit",
  "match-template",
  "object_store_opendal",
+ "opendal_compat",
  "parquet",
  "serde",
  "serde_json",
@@ -4276,6 +4338,7 @@ dependencies = [
  "ahash 0.8.11",
  "arrow",
  "arrow-array",
+ "arrow-ipc",
  "async-backtrace",
  "async-channel 1.9.0",
  "async-trait",
@@ -4311,14 +4374,14 @@ dependencies = [
  "databend-storages-common-session",
  "databend-storages-common-table-meta",
  "enum-as-inner 0.5.1",
- "fastrace 0.6.8",
+ "fastrace",
  "futures",
  "futures-util",
  "indexmap 2.4.0",
  "itertools 0.10.5",
  "jsonb",
  "log",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4327,6 +4390,7 @@ dependencies = [
  "siphasher",
  "sys-info",
  "tantivy",
+ "tantivy-fst",
  "tantivy-jieba",
  "thrift",
  "typetag",
@@ -4359,12 +4423,12 @@ dependencies = [
  "databend-storages-common-cache",
  "databend-storages-common-index",
  "databend-storages-common-table-meta",
- "fastrace 0.6.8",
+ "fastrace",
  "faststr",
  "futures",
  "hive_metastore",
  "log",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "recursive",
  "serde",
  "typetag",
@@ -4384,22 +4448,19 @@ dependencies = [
  "databend-common-config",
  "databend-common-exception",
  "databend-common-expression",
- "databend-common-functions",
  "databend-common-meta-app",
  "databend-common-meta-store",
  "databend-common-meta-types",
  "databend-common-pipeline-core",
- "databend-common-storage",
  "databend-common-storages-parquet",
- "databend-storages-common-pruner",
  "databend-storages-common-table-meta",
- "fastrace 0.6.8",
+ "fastrace",
+ "futures",
  "iceberg",
  "iceberg-catalog-hms",
  "iceberg-catalog-rest",
  "match-template",
  "ordered-float 4.2.2",
- "parquet",
  "serde",
  "tokio",
  "typetag",
@@ -4480,7 +4541,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "futures-util",
  "log",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "orc-rust",
  "serde",
  "serde_json",
@@ -4516,7 +4577,7 @@ dependencies = [
  "ethnum",
  "futures",
  "log",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4563,7 +4624,7 @@ dependencies = [
  "databend-common-storages-parquet",
  "databend-storages-common-blocks",
  "databend-storages-common-table-meta",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "parquet",
  "serde",
  "serde_json",
@@ -4604,7 +4665,7 @@ dependencies = [
  "enum-as-inner 0.6.0",
  "futures",
  "log",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "parquet",
  "serde",
  "serde_json",
@@ -4628,7 +4689,7 @@ dependencies = [
  "databend-common-sql",
  "databend-common-storages-fuse",
  "databend-storages-common-table-meta",
- "fastrace 0.6.8",
+ "fastrace",
  "futures",
  "log",
 ]
@@ -4669,7 +4730,7 @@ dependencies = [
  "jsonb",
  "log",
  "once_cell",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "parking_lot 0.12.3",
  "regex",
  "serde",
@@ -4694,26 +4755,25 @@ dependencies = [
 name = "databend-common-tracing"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
  "backtrace",
+ "bincode 2.0.0-rc.3",
  "chrono",
- "color-backtrace",
  "databend-common-base",
  "databend-common-exception",
  "defer",
- "fastrace 0.6.8",
+ "fastrace",
  "fastrace-opentelemetry",
- "fern",
  "itertools 0.10.5",
  "libc",
  "log",
+ "logforth",
  "opentelemetry",
  "opentelemetry-otlp",
  "opentelemetry_sdk",
  "serde",
  "serde_json",
- "strip-ansi-escapes",
- "tonic",
- "tracing-appender",
+ "tonic 0.11.0",
 ]
 
 [[package]]
@@ -4744,7 +4804,7 @@ dependencies = [
  "parking_lot 0.12.3",
  "passwords",
  "pretty_assertions",
- "reqwest 0.12.5",
+ "reqwest",
  "serde",
  "serde_json",
  "wiremock 0.5.22",
@@ -4915,6 +4975,7 @@ dependencies = [
  "databend-enterprise-virtual-column",
  "databend-query",
  "databend-storages-common-cache",
+ "databend-storages-common-io",
  "databend-storages-common-pruner",
  "databend-storages-common-table-meta",
  "derive-visitor",
@@ -4923,7 +4984,7 @@ dependencies = [
  "jsonb",
  "jwt-simple 0.11.9",
  "log",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "tantivy",
  "tempfile",
 ]
@@ -5013,7 +5074,7 @@ dependencies = [
  "deepsize",
  "derive_more",
  "env_logger",
- "fastrace 0.6.8",
+ "fastrace",
  "feature-set",
  "futures",
  "futures-async-stream",
@@ -5027,7 +5088,7 @@ dependencies = [
  "prometheus-client",
  "prost 0.12.6",
  "regex",
- "reqwest 0.12.5",
+ "reqwest",
  "semver",
  "serde",
  "serde_json",
@@ -5037,7 +5098,7 @@ dependencies = [
  "tempfile",
  "test-harness",
  "tokio-stream",
- "tonic",
+ "tonic 0.11.0",
  "tonic-reflection",
 ]
 
@@ -5062,11 +5123,11 @@ dependencies = [
  "databend-common-meta-types",
  "databend-common-tracing",
  "databend-meta",
- "fastrace 0.6.8",
+ "fastrace",
  "futures",
  "log",
  "rand 0.8.5",
- "reqwest 0.12.5",
+ "reqwest",
  "serde",
  "serde_json",
  "tokio",
@@ -5170,7 +5231,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "derive-visitor",
  "ethnum",
- "fastrace 0.6.8",
+ "fastrace",
  "flatbuffers",
  "futures",
  "futures-util",
@@ -5195,7 +5256,7 @@ dependencies = [
  "num",
  "num_cpus",
  "once_cell",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "opensrv-mysql",
  "opentelemetry",
  "opentelemetry_sdk",
@@ -5212,7 +5273,7 @@ dependencies = [
  "rand 0.8.5",
  "recursive",
  "regex",
- "reqwest 0.12.5",
+ "reqwest",
  "rmp-serde",
  "rustls 0.22.4",
  "rustls-pemfile 2.1.3",
@@ -5224,6 +5285,7 @@ dependencies = [
  "serde_urlencoded",
  "sha2",
  "socket2 0.5.7",
+ "sqlx",
  "strength_reduce",
  "sysinfo",
  "temp-env",
@@ -5232,8 +5294,8 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "toml 0.8.19",
- "tonic",
- "tower",
+ "tonic 0.11.0",
+ "tower 0.4.13",
  "typetag",
  "url",
  "uuid",
@@ -5274,13 +5336,16 @@ dependencies = [
  "databend-common-exception",
  "env_logger",
  "futures-util",
+ "msql-srv",
  "mysql_async",
+ "mysql_common 0.32.4",
  "rand 0.8.5",
  "regex",
- "reqwest 0.12.5",
+ "reqwest",
  "serde",
  "serde_json",
  "sqllogictest",
+ "sqlparser 0.50.0",
  "thiserror",
  "tokio",
  "walkdir",
@@ -5345,6 +5410,7 @@ dependencies = [
  "log",
  "parking_lot 0.12.3",
  "rayon",
+ "rustix 0.38.37",
  "siphasher",
  "tempfile",
 ]
@@ -5355,7 +5421,6 @@ version = "0.1.0"
 dependencies = [
  "anyerror",
  "cbordata",
- "crc32fast",
  "criterion",
  "databend-common-arrow",
  "databend-common-ast",
@@ -5363,16 +5428,18 @@ dependencies = [
  "databend-common-expression",
  "databend-common-functions",
  "databend-storages-common-table-meta",
- "fastrace 0.6.8",
+ "fastrace",
  "jsonb",
+ "levenshtein_automata",
  "log",
  "match-template",
  "parquet",
  "rand 0.8.5",
+ "roaring",
  "serde",
- "serde_json",
  "tantivy",
  "tantivy-common",
+ "tantivy-fst",
  "thiserror",
  "xorfilter-rs",
 ]
@@ -5382,12 +5449,16 @@ name = "databend-storages-common-io"
 version = "0.1.0"
 dependencies = [
  "async-backtrace",
+ "bytes",
  "databend-common-base",
  "databend-common-catalog",
  "databend-common-exception",
- "fastrace 0.6.8",
+ "databend-common-expression",
+ "databend-common-metrics",
+ "fastrace",
+ "futures",
  "log",
- "opendal 0.50.0",
+ "opendal 0.50.1",
 ]
 
 [[package]]
@@ -5458,390 +5529,87 @@ dependencies = [
 ]
 
 [[package]]
-name = "datafusion"
-version = "39.0.0"
+name = "deadpool"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068"
+checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e"
 dependencies = [
- "ahash 0.8.11",
- "arrow",
- "arrow-array",
- "arrow-ipc",
- "arrow-schema",
- "async-compression 0.4.12",
  "async-trait",
- "bytes",
- "bzip2",
- "chrono",
- "dashmap 5.5.3",
- "datafusion-common",
- "datafusion-common-runtime",
- "datafusion-execution",
- "datafusion-expr",
- "datafusion-functions",
- "datafusion-functions-aggregate",
- "datafusion-functions-array",
- "datafusion-optimizer",
- "datafusion-physical-expr",
- "datafusion-physical-expr-common",
- "datafusion-physical-plan",
- "datafusion-sql",
- "flate2",
- "futures",
- "glob",
- "half",
- "hashbrown 0.14.5",
- "indexmap 2.4.0",
- "itertools 0.12.1",
- "log",
+ "deadpool-runtime",
  "num_cpus",
- "object_store 0.10.2",
- "parking_lot 0.12.3",
- "parquet",
- "paste",
- "pin-project-lite",
- "rand 0.8.5",
- "sqlparser 0.47.0",
- "tempfile",
+ "retain_mut",
  "tokio",
- "tokio-util",
- "url",
- "uuid",
- "xz2",
- "zstd 0.13.2",
 ]
 
 [[package]]
-name = "datafusion-common"
-version = "39.0.0"
+name = "deadpool"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971"
+checksum = "fb84100978c1c7b37f09ed3ce3e5f843af02c2a2c431bae5b19230dad2c1b490"
 dependencies = [
- "ahash 0.8.11",
- "arrow",
- "arrow-array",
- "arrow-buffer",
- "arrow-schema",
- "chrono",
- "half",
- "hashbrown 0.14.5",
- "instant",
- "libc",
+ "async-trait",
+ "deadpool-runtime",
  "num_cpus",
- "object_store 0.10.2",
- "parquet",
- "sqlparser 0.47.0",
+ "tokio",
 ]
 
 [[package]]
-name = "datafusion-common-runtime"
-version = "39.0.0"
+name = "deadpool-runtime"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0091318129dad1359f08e4c6c71f855163c35bba05d1dbf983196f727857894"
-dependencies = [
- "tokio",
-]
+checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
 
 [[package]]
-name = "datafusion-execution"
-version = "39.0.0"
+name = "debugid"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1"
+checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
 dependencies = [
- "arrow",
- "chrono",
- "dashmap 5.5.3",
- "datafusion-common",
- "datafusion-expr",
- "futures",
- "hashbrown 0.14.5",
- "log",
- "object_store 0.10.2",
- "parking_lot 0.12.3",
- "rand 0.8.5",
- "tempfile",
- "url",
+ "uuid",
 ]
 
 [[package]]
-name = "datafusion-expr"
-version = "39.0.0"
+name = "deepsize"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3"
+checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c"
 dependencies = [
- "ahash 0.8.11",
- "arrow",
- "arrow-array",
- "arrow-buffer",
- "chrono",
- "datafusion-common",
- "paste",
- "serde_json",
- "sqlparser 0.47.0",
- "strum 0.26.3",
- "strum_macros 0.26.4",
+ "deepsize_derive",
 ]
 
 [[package]]
-name = "datafusion-functions"
-version = "39.0.0"
+name = "deepsize_derive"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b"
+checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796"
 dependencies = [
- "arrow",
- "base64 0.22.1",
- "blake2",
- "blake3",
- "chrono",
- "datafusion-common",
- "datafusion-execution",
- "datafusion-expr",
- "datafusion-physical-expr",
- "hashbrown 0.14.5",
- "hex",
- "itertools 0.12.1",
- "log",
- "md-5",
- "rand 0.8.5",
- "regex",
- "sha2",
- "unicode-segmentation",
- "uuid",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
 ]
 
 [[package]]
-name = "datafusion-functions-aggregate"
-version = "39.0.0"
+name = "defer"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78"
-dependencies = [
- "ahash 0.8.11",
- "arrow",
- "arrow-schema",
- "datafusion-common",
- "datafusion-execution",
- "datafusion-expr",
- "datafusion-physical-expr-common",
- "log",
- "paste",
- "sqlparser 0.47.0",
-]
+checksum = "930c7171c8df9fb1782bdf9b918ed9ed2d33d1d22300abb754f9085bc48bf8e8"
+
+[[package]]
+name = "deflate64"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
 
 [[package]]
-name = "datafusion-functions-array"
-version = "39.0.0"
+name = "delta_kernel"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a"
+checksum = "fa08a82239f51e6d3d249c38f0f5bf7c8a78b28587e1b466893c9eac84d252d8"
 dependencies = [
- "arrow",
+ "arrow-arith",
  "arrow-array",
- "arrow-buffer",
- "arrow-ord",
- "arrow-schema",
- "datafusion-common",
- "datafusion-execution",
- "datafusion-expr",
- "datafusion-functions",
- "itertools 0.12.1",
- "log",
- "paste",
-]
-
-[[package]]
-name = "datafusion-optimizer"
-version = "39.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76"
-dependencies = [
- "arrow",
- "async-trait",
- "chrono",
- "datafusion-common",
- "datafusion-expr",
- "datafusion-physical-expr",
- "hashbrown 0.14.5",
- "indexmap 2.4.0",
- "itertools 0.12.1",
- "log",
- "regex-syntax 0.8.4",
-]
-
-[[package]]
-name = "datafusion-physical-expr"
-version = "39.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e"
-dependencies = [
- "ahash 0.8.11",
- "arrow",
- "arrow-array",
- "arrow-buffer",
- "arrow-ord",
- "arrow-schema",
- "arrow-string",
- "base64 0.22.1",
- "chrono",
- "datafusion-common",
- "datafusion-execution",
- "datafusion-expr",
- "datafusion-functions-aggregate",
- "datafusion-physical-expr-common",
- "half",
- "hashbrown 0.14.5",
- "hex",
- "indexmap 2.4.0",
- "itertools 0.12.1",
- "log",
- "paste",
- "petgraph",
- "regex",
-]
-
-[[package]]
-name = "datafusion-physical-expr-common"
-version = "39.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77"
-dependencies = [
- "arrow",
- "datafusion-common",
- "datafusion-expr",
- "rand 0.8.5",
-]
-
-[[package]]
-name = "datafusion-physical-plan"
-version = "39.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff"
-dependencies = [
- "ahash 0.8.11",
- "arrow",
- "arrow-array",
- "arrow-buffer",
- "arrow-ord",
- "arrow-schema",
- "async-trait",
- "chrono",
- "datafusion-common",
- "datafusion-common-runtime",
- "datafusion-execution",
- "datafusion-expr",
- "datafusion-functions-aggregate",
- "datafusion-physical-expr",
- "datafusion-physical-expr-common",
- "futures",
- "half",
- "hashbrown 0.14.5",
- "indexmap 2.4.0",
- "itertools 0.12.1",
- "log",
- "once_cell",
- "parking_lot 0.12.3",
- "pin-project-lite",
- "rand 0.8.5",
- "tokio",
-]
-
-[[package]]
-name = "datafusion-sql"
-version = "39.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e"
-dependencies = [
- "arrow",
- "arrow-array",
- "arrow-schema",
- "datafusion-common",
- "datafusion-expr",
- "log",
- "regex",
- "sqlparser 0.47.0",
- "strum 0.26.3",
-]
-
-[[package]]
-name = "deadpool"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e"
-dependencies = [
- "async-trait",
- "deadpool-runtime",
- "num_cpus",
- "retain_mut",
- "tokio",
-]
-
-[[package]]
-name = "deadpool"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb84100978c1c7b37f09ed3ce3e5f843af02c2a2c431bae5b19230dad2c1b490"
-dependencies = [
- "async-trait",
- "deadpool-runtime",
- "num_cpus",
- "tokio",
-]
-
-[[package]]
-name = "deadpool-runtime"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
-
-[[package]]
-name = "debugid"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
-dependencies = [
- "uuid",
-]
-
-[[package]]
-name = "deepsize"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c"
-dependencies = [
- "deepsize_derive",
-]
-
-[[package]]
-name = "deepsize_derive"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "defer"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "930c7171c8df9fb1782bdf9b918ed9ed2d33d1d22300abb754f9085bc48bf8e8"
-
-[[package]]
-name = "deflate64"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
-
-[[package]]
-name = "delta_kernel"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa08a82239f51e6d3d249c38f0f5bf7c8a78b28587e1b466893c9eac84d252d8"
-dependencies = [
- "arrow-arith",
- "arrow-array",
- "arrow-cast",
- "arrow-json",
+ "arrow-cast",
+ "arrow-json",
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
@@ -5906,7 +5674,7 @@ dependencies = [
  "arrow-select",
  "async-trait",
  "bytes",
- "cfg-if 1.0.0",
+ "cfg-if",
  "chrono",
  "dashmap 6.1.0",
  "delta_kernel",
@@ -5923,7 +5691,7 @@ dependencies = [
  "num-bigint",
  "num-traits",
  "num_cpus",
- "object_store 0.10.2",
+ "object_store",
  "once_cell",
  "parking_lot 0.12.3",
  "parquet",
@@ -6094,7 +5862,7 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "dirs-sys-next",
 ]
 
@@ -6150,6 +5918,12 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
 
+[[package]]
+name = "dotenvy"
+version = "0.15.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
+
 [[package]]
 name = "downcast"
 version = "0.11.0"
@@ -6171,7 +5945,7 @@ checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"
 [[package]]
 name = "dtparse"
 version = "2.0.0"
-source = "git+https://github.com/TCeason/dtparse.git?rev=de0a15b#de0a15b5db5cf3f2c99f38a6535f669bc0ee7ea1"
+source = "git+https://github.com/datafuse-extras/dtparse.git?rev=de0a15b#de0a15b5db5cf3f2c99f38a6535f669bc0ee7ea1"
 dependencies = [
  "chrono",
  "lazy_static",
@@ -6266,6 +6040,9 @@ name = "either"
 version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "elliptic-curve"
@@ -6320,7 +6097,7 @@ version = "0.8.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -6479,10 +6256,21 @@ version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6"
 
+[[package]]
+name = "etcetera"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943"
+dependencies = [
+ "cfg-if",
+ "home",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "ethnum"
 version = "1.5.0"
-source = "git+https://github.com/ariesdevil/ethnum-rs?rev=4cb05f1#4cb05f1e407f76b193d81eef71b5dd0b73216856"
+source = "git+https://github.com/datafuse-extras/ethnum-rs?rev=4cb05f1#4cb05f1e407f76b193d81eef71b5dd0b73216856"
 dependencies = [
  "borsh",
  "serde",
@@ -6555,46 +6343,19 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2a2b11eda1d40935b26cf18f6833c526845ae8c41e58d09af6adeb6f0269183"
 
-[[package]]
-name = "fastrace"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfd798348670042f0628810b10bc11b767c1f42aafbd3b1ae31f282740c549fe"
-dependencies = [
- "fastrace-macro 0.6.8",
- "minstant",
- "once_cell",
- "parking_lot 0.12.3",
- "pin-project",
- "rand 0.8.5",
- "rtrb 0.2.3",
-]
-
 [[package]]
 name = "fastrace"
 version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "25767929385a5128ff5a58c77d5fef43779bb580656801f72d4d32f7a97fea28"
 dependencies = [
- "fastrace-macro 0.7.2",
+ "fastrace-macro",
  "minstant",
  "once_cell",
  "parking_lot 0.12.3",
  "pin-project",
  "rand 0.8.5",
- "rtrb 0.3.1",
-]
-
-[[package]]
-name = "fastrace-macro"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eea27de901dc6b14aa952aeefcc41144da4dc97e4db3eb3593c19f3b80406786"
-dependencies = [
- "proc-macro-error 1.0.4",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
+ "rtrb",
 ]
 
 [[package]]
@@ -6611,11 +6372,11 @@ dependencies = [
 
 [[package]]
 name = "fastrace-opentelemetry"
-version = "0.6.8"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05895fbf0eba689978865a110fcead4a5c04724322dc6bd73ff55fe06e39ba66"
+checksum = "8f33f99bae33fb7b952a933500e77cb9976b7672bb5d9def8e464663dfdfc107"
 dependencies = [
- "fastrace 0.6.8",
+ "fastrace",
  "futures",
  "log",
  "opentelemetry",
@@ -6654,8 +6415,8 @@ version = "4.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7e5768da2206272c81ef0b5e951a41862938a6070da63bcea197899942d3b947"
 dependencies = [
- "cfg-if 1.0.0",
- "rustix 0.38.34",
+ "cfg-if",
+ "rustix 0.38.37",
  "windows-sys 0.52.0",
 ]
 
@@ -6669,15 +6430,6 @@ dependencies = [
  "semver",
 ]
 
-[[package]]
-name = "fern"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9f0c14694cbd524c8720dd69b0e3179344f04ebb5f90f2e4a440c6ea3b2f1ee"
-dependencies = [
- "log",
-]
-
 [[package]]
 name = "ff"
 version = "0.12.1"
@@ -6704,7 +6456,7 @@ version = "0.2.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "libredox",
  "windows-sys 0.59.0",
@@ -6797,6 +6549,17 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8"
 
+[[package]]
+name = "flume"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "spin",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -6934,7 +6697,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "033b337d725b97690d86893f9de22b67b80dcc4e9ad815f348254c38119db8fb"
 dependencies = [
  "io-lifetimes 2.0.3",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "windows-sys 0.52.0",
 ]
 
@@ -6954,7 +6717,7 @@ version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
 dependencies = [
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "windows-sys 0.52.0",
 ]
 
@@ -7053,6 +6816,17 @@ dependencies = [
  "futures-util",
 ]
 
+[[package]]
+name = "futures-intrusive"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f"
+dependencies = [
+ "futures-core",
+ "lock_api",
+ "parking_lot 0.12.3",
+]
+
 [[package]]
 name = "futures-io"
 version = "0.3.30"
@@ -7353,7 +7127,7 @@ version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "wasi 0.9.0+wasi-snapshot-preview1",
 ]
@@ -7364,7 +7138,7 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "js-sys",
  "libc",
  "wasi 0.11.0+wasi-snapshot-preview1",
@@ -7779,7 +7553,7 @@ dependencies = [
  "itoa",
  "libc",
  "memmap2 0.9.4",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "smallvec",
  "thiserror",
 ]
@@ -7941,7 +7715,7 @@ dependencies = [
  "gix-command",
  "gix-config-value",
  "parking_lot 0.12.3",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "thiserror",
 ]
 
@@ -8334,7 +8108,7 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "crunchy",
  "num-traits",
 ]
@@ -8393,6 +8167,15 @@ dependencies = [
  "hashbrown 0.14.5",
 ]
 
+[[package]]
+name = "hashlink"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
+dependencies = [
+ "hashbrown 0.14.5",
+]
+
 [[package]]
 name = "hdfs-sys"
 version = "0.3.0"
@@ -8509,7 +8292,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512"
 dependencies = [
  "async-trait",
- "cfg-if 1.0.0",
+ "cfg-if",
  "data-encoding",
  "enum-as-inner 0.6.0",
  "futures-channel",
@@ -8532,7 +8315,7 @@ version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "futures-util",
  "hickory-proto",
  "ipconfig",
@@ -8831,6 +8614,19 @@ dependencies = [
  "tokio-io-timeout",
 ]
 
+[[package]]
+name = "hyper-timeout"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793"
+dependencies = [
+ "hyper 1.4.1",
+ "hyper-util",
+ "pin-project-lite",
+ "tokio",
+ "tower-service",
+]
+
 [[package]]
 name = "hyper-tls"
 version = "0.6.0"
@@ -8862,7 +8658,7 @@ dependencies = [
  "pin-project-lite",
  "socket2 0.5.7",
  "tokio",
- "tower",
+ "tower 0.4.13",
  "tower-service",
  "tracing",
 ]
@@ -8893,8 +8689,7 @@ dependencies = [
 [[package]]
 name = "iceberg"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "651dfca7c429918e164607a549287cfdd1e7814d2e4cb577d0d6dc57fe19b785"
+source = "git+https://github.com/Xuanwo/iceberg-rust/?rev=a0e74b1412ae7ce4b857c251bcbb3179b7713fb6#a0e74b1412ae7ce4b857c251bcbb3179b7713fb6"
 dependencies = [
  "anyhow",
  "apache-avro",
@@ -8914,12 +8709,15 @@ dependencies = [
  "fnv",
  "futures",
  "itertools 0.13.0",
+ "moka",
  "murmur3",
+ "num-bigint",
  "once_cell",
- "opendal 0.49.0",
+ "opendal 0.50.1",
  "ordered-float 4.2.2",
  "parquet",
- "reqwest 0.12.5",
+ "paste",
+ "reqwest",
  "rust_decimal",
  "serde",
  "serde_bytes",
@@ -8928,7 +8726,7 @@ dependencies = [
  "serde_repr",
  "serde_with",
  "tokio",
- "typed-builder",
+ "typed-builder 0.20.0",
  "url",
  "uuid",
 ]
@@ -8936,8 +8734,7 @@ dependencies = [
 [[package]]
 name = "iceberg-catalog-hms"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a04f1f48f0f15e42c910fafb57edb0dd309c936cd285b1089fa3d4a07d1ef09"
+source = "git+https://github.com/Xuanwo/iceberg-rust/?rev=a0e74b1412ae7ce4b857c251bcbb3179b7713fb6#a0e74b1412ae7ce4b857c251bcbb3179b7713fb6"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -8948,7 +8745,7 @@ dependencies = [
  "pilota",
  "serde_json",
  "tokio",
- "typed-builder",
+ "typed-builder 0.20.0",
  "uuid",
  "volo-thrift",
 ]
@@ -8956,8 +8753,7 @@ dependencies = [
 [[package]]
 name = "iceberg-catalog-rest"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f351c7b964fa6f3b4f976f8de3f16f1bf84eea8478606aaebdfd6a871d6b082c"
+source = "git+https://github.com/Xuanwo/iceberg-rust/?rev=a0e74b1412ae7ce4b857c251bcbb3179b7713fb6#a0e74b1412ae7ce4b857c251bcbb3179b7713fb6"
 dependencies = [
  "async-trait",
  "chrono",
@@ -8965,12 +8761,12 @@ dependencies = [
  "iceberg",
  "itertools 0.13.0",
  "log",
- "reqwest 0.12.5",
+ "reqwest",
  "serde",
  "serde_derive",
  "serde_json",
  "tokio",
- "typed-builder",
+ "typed-builder 0.20.0",
  "uuid",
 ]
 
@@ -9143,7 +8939,7 @@ version = "0.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "js-sys",
  "wasm-bindgen",
  "web-sys",
@@ -9390,6 +9186,32 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "jiff"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a45489186a6123c128fdf6016183fcfab7113e1820eb813127e036e287233fb"
+dependencies = [
+ "jiff-tzdb-platform",
+ "serde",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "jiff-tzdb"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91335e575850c5c4c673b9bd467b0e025f164ca59d0564f69d0c2ee0ffad4653"
+
+[[package]]
+name = "jiff-tzdb-platform"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9835f0060a626fe59f160437bc725491a6af23133ea906500027d1bd2f8f4329"
+dependencies = [
+ "jiff-tzdb",
+]
+
 [[package]]
 name = "jobserver"
 version = "0.1.32"
@@ -9421,9 +9243,8 @@ dependencies = [
 
 [[package]]
 name = "jsonb"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f98d1dd2023aa6a1b74f23432e2db5a9122bf53ae98053efc650951d31f59ffb"
+version = "0.4.3"
+source = "git+https://github.com/databendlabs/jsonb?rev=c7525d9#c7525d93ad7b1e800df982e49a6e0e6454bd308e"
 dependencies = [
  "byteorder",
  "fast-float",
@@ -9519,7 +9340,7 @@ version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "956ff9b67e26e1a6a866cb758f12c6f8746208489e3e4a4b5580802f2f0a587b"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "ecdsa 0.16.9",
  "elliptic-curve 0.13.8",
  "once_cell",
@@ -9716,9 +9537,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.159"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
 
 [[package]]
 name = "libflate"
@@ -9750,7 +9571,7 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "windows-targets 0.52.6",
 ]
 
@@ -9771,6 +9592,17 @@ dependencies = [
  "redox_syscall 0.5.3",
 ]
 
+[[package]]
+name = "libsqlite3-sys"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
+dependencies = [
+ "cc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "libtest-mimic"
 version = "0.7.3"
@@ -9889,6 +9721,27 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "logforth"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "633080680671612565f637d1e33c5bcb7d58fb12c7d658baa166a03487265e80"
+dependencies = [
+ "anyhow",
+ "colored",
+ "crossbeam-channel",
+ "fastrace",
+ "jiff",
+ "log",
+ "opentelemetry",
+ "opentelemetry-otlp",
+ "opentelemetry_sdk",
+ "parking_lot 0.12.3",
+ "paste",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "logos"
 version = "0.12.1"
@@ -9918,7 +9771,7 @@ version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "generator",
  "scoped-tls",
  "serde",
@@ -10073,7 +9926,7 @@ version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "digest",
 ]
 
@@ -10099,7 +9952,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64"
 dependencies = [
- "rustix 0.38.34",
+ "rustix 0.38.37",
 ]
 
 [[package]]
@@ -10129,12 +9982,6 @@ dependencies = [
  "autocfg",
 ]
 
-[[package]]
-name = "memory_units"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3"
-
 [[package]]
 name = "metainfo"
 version = "0.7.12"
@@ -10248,7 +10095,7 @@ version = "0.11.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "downcast",
  "fragile",
  "lazy_static",
@@ -10263,7 +10110,7 @@ version = "0.11.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "proc-macro2",
  "quote",
  "syn 1.0.109",
@@ -10316,6 +10163,19 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "msql-srv"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b821d09e9a4ed6b61015a889597446b3b6c7721544d0f4b617bcfdacf6ee7877"
+dependencies = [
+ "byteorder",
+ "chrono",
+ "mysql_common 0.31.0",
+ "nom",
+ "rustls 0.22.4",
+]
+
 [[package]]
 name = "multer"
 version = "3.1.0"
@@ -10340,6 +10200,28 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03"
 
+[[package]]
+name = "multiversion"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4851161a11d3ad0bf9402d90ffc3967bf231768bfd7aeb61755ad06dbf1a142"
+dependencies = [
+ "multiversion-macros",
+ "target-features",
+]
+
+[[package]]
+name = "multiversion-macros"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79a74ddee9e0c27d2578323c13905793e91622148f138ba29738f9dddb835e90"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "target-features",
+]
+
 [[package]]
 name = "mur3"
 version = "0.1.0"
@@ -10358,6 +10240,24 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
 
+[[package]]
+name = "mysql-common-derive"
+version = "0.30.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56b0d8a0db9bf6d2213e11f2c701cb91387b0614361625ab7b9743b41aa4938f"
+dependencies = [
+ "darling",
+ "heck 0.4.1",
+ "num-bigint",
+ "proc-macro-crate 1.3.1",
+ "proc-macro-error 1.0.4",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.58",
+ "termcolor",
+ "thiserror",
+]
+
 [[package]]
 name = "mysql-common-derive"
 version = "0.31.1"
@@ -10367,7 +10267,7 @@ dependencies = [
  "darling",
  "heck 0.4.1",
  "num-bigint",
- "proc-macro-crate",
+ "proc-macro-crate 3.1.0",
  "proc-macro-error 1.0.4",
  "proc-macro2",
  "quote",
@@ -10390,7 +10290,7 @@ dependencies = [
  "futures-util",
  "keyed_priority_queue",
  "lru",
- "mysql_common",
+ "mysql_common 0.32.4",
  "native-tls",
  "pem",
  "percent-encoding",
@@ -10407,6 +10307,46 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "mysql_common"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef"
+dependencies = [
+ "base64 0.21.7",
+ "bigdecimal",
+ "bindgen 0.69.4",
+ "bitflags 2.6.0",
+ "bitvec",
+ "btoi",
+ "byteorder",
+ "bytes",
+ "cc",
+ "chrono",
+ "cmake",
+ "crc32fast",
+ "flate2",
+ "frunk",
+ "lazy_static",
+ "mysql-common-derive 0.30.2",
+ "num-bigint",
+ "num-traits",
+ "rand 0.8.5",
+ "regex",
+ "rust_decimal",
+ "saturating",
+ "serde",
+ "serde_json",
+ "sha1",
+ "sha2",
+ "smallvec",
+ "subprocess",
+ "thiserror",
+ "time",
+ "uuid",
+ "zstd 0.12.4",
+]
+
 [[package]]
 name = "mysql_common"
 version = "0.32.4"
@@ -10428,7 +10368,7 @@ dependencies = [
  "flate2",
  "frunk",
  "lazy_static",
- "mysql-common-derive",
+ "mysql-common-derive 0.31.1",
  "num-bigint",
  "num-traits",
  "rand 0.8.5",
@@ -10499,7 +10439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
 dependencies = [
  "bitflags 1.3.2",
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
 ]
 
@@ -10510,7 +10450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
 dependencies = [
  "bitflags 2.6.0",
- "cfg-if 1.0.0",
+ "cfg-if",
  "cfg_aliases 0.1.1",
  "libc",
  "memoffset",
@@ -10523,7 +10463,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
 dependencies = [
  "bitflags 2.6.0",
- "cfg-if 1.0.0",
+ "cfg-if",
  "cfg_aliases 0.2.1",
  "libc",
 ]
@@ -10540,16 +10480,16 @@ dependencies = [
 
 [[package]]
 name = "nom-rule"
-version = "0.3.1"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f0f33cd048b8e0f5cd2fc6fd26f4feda8e33b4781673e59fe1406b137600e9d"
+checksum = "c72951bd83c76b88d820f03b38b124f981dc2520070d62170da7012d1ce964ab"
 dependencies = [
  "nom",
- "pratt 0.3.0",
+ "pratt",
  "proc-macro-error 1.0.4",
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.58",
 ]
 
 [[package]]
@@ -10741,7 +10681,7 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56"
 dependencies = [
- "proc-macro-crate",
+ "proc-macro-crate 3.1.0",
  "proc-macro2",
  "quote",
  "syn 2.0.58",
@@ -10804,39 +10744,18 @@ dependencies = [
  "walkdir",
 ]
 
-[[package]]
-name = "object_store"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25a0c4b3a0e31f8b66f71ad8064521efa773910196e2cde791436f13409f3b45"
-dependencies = [
- "async-trait",
- "bytes",
- "chrono",
- "futures",
- "humantime",
- "itertools 0.13.0",
- "parking_lot 0.12.3",
- "percent-encoding",
- "snafu 0.8.4",
- "tokio",
- "tracing",
- "url",
- "walkdir",
-]
-
 [[package]]
 name = "object_store_opendal"
-version = "0.47.0"
+version = "0.46.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7a27272d45c2d9d60c780f5d4d48b50f0d692e82de25835cde38bdf0e2b2a9c"
+checksum = "4493c44c90a25391782b52886365d00f6be7d0f75aaecab4365279203438ebec"
 dependencies = [
  "async-trait",
  "bytes",
  "flagset",
  "futures",
  "futures-util",
- "object_store 0.11.0",
+ "object_store",
  "opendal 0.49.0",
  "pin-project",
  "tokio",
@@ -10883,7 +10802,6 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "chrono",
- "crc32c",
  "flagset",
  "futures",
  "getrandom 0.2.15",
@@ -10893,8 +10811,7 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "quick-xml 0.36.1",
- "reqsign",
- "reqwest 0.12.5",
+ "reqwest",
  "serde",
  "serde_json",
  "tokio",
@@ -10903,19 +10820,20 @@ dependencies = [
 
 [[package]]
 name = "opendal"
-version = "0.50.0"
+version = "0.50.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36e44fc43be9ffe18dad3e3ef9d61c1ae01991ee6f1c8c026978c35777a711bf"
+checksum = "213222b6c86949314d8f51acb26d8241e7c8dd0879b016a79471d49f21ee592f"
 dependencies = [
  "anyhow",
  "async-backtrace",
  "async-trait",
  "backon 1.2.0",
  "base64 0.22.1",
+ "bb8",
  "bytes",
  "chrono",
  "crc32c",
- "fastrace 0.7.2",
+ "fastrace",
  "flagset",
  "futures",
  "getrandom 0.2.15",
@@ -10931,7 +10849,7 @@ dependencies = [
  "quick-xml 0.36.1",
  "redis",
  "reqsign",
- "reqwest 0.12.5",
+ "reqwest",
  "serde",
  "serde_json",
  "sha2",
@@ -10939,6 +10857,17 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "opendal_compat"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "091a08a86c52bd69d7b4be44202d03edf6d0358256805561be2287b5c308fcdf"
+dependencies = [
+ "async-trait",
+ "opendal 0.49.0",
+ "opendal 0.50.1",
+]
+
 [[package]]
 name = "openraft"
 version = "0.10.0"
@@ -10982,7 +10911,7 @@ dependencies = [
  "async-trait",
  "byteorder",
  "chrono",
- "mysql_common",
+ "mysql_common 0.32.4",
  "nom",
  "pin-project-lite",
  "tokio",
@@ -10996,7 +10925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
 dependencies = [
  "bitflags 2.6.0",
- "cfg-if 1.0.0",
+ "cfg-if",
  "foreign-types 0.3.2",
  "libc",
  "once_cell",
@@ -11045,9 +10974,9 @@ dependencies = [
 
 [[package]]
 name = "opentelemetry"
-version = "0.23.0"
+version = "0.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b69a91d4893e713e06f724597ad630f1fa76057a5e1026c0ca67054a9032a76"
+checksum = "4c365a63eec4f55b7efeceb724f1336f26a9cf3427b70e59e2cd2a5b947fba96"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -11059,64 +10988,65 @@ dependencies = [
 
 [[package]]
 name = "opentelemetry-http"
-version = "0.12.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0ba633e55c5ea6f431875ba55e71664f2fa5d3a90bd34ec9302eecc41c865dd"
+checksum = "ad31e9de44ee3538fb9d64fe3376c1362f406162434609e79aea2a41a0af78ab"
 dependencies = [
  "async-trait",
  "bytes",
- "http 0.2.12",
+ "http 1.1.0",
  "opentelemetry",
- "reqwest 0.11.27",
+ "reqwest",
 ]
 
 [[package]]
 name = "opentelemetry-otlp"
-version = "0.16.0"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a94c69209c05319cdf7460c6d4c055ed102be242a0a6245835d7bc42c6ec7f54"
+checksum = "6b925a602ffb916fb7421276b86756027b37ee708f9dce2dbdcc51739f07e727"
 dependencies = [
  "async-trait",
  "futures-core",
- "http 0.2.12",
+ "http 1.1.0",
  "opentelemetry",
  "opentelemetry-http",
  "opentelemetry-proto",
  "opentelemetry_sdk",
- "prost 0.12.6",
- "reqwest 0.11.27",
+ "prost 0.13.1",
+ "reqwest",
+ "serde_json",
  "thiserror",
  "tokio",
- "tonic",
+ "tonic 0.12.3",
 ]
 
 [[package]]
 name = "opentelemetry-proto"
-version = "0.6.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "984806e6cf27f2b49282e2a05e288f30594f3dbc74eb7a6e99422bc48ed78162"
+checksum = "30ee9f20bff9c984511a02f082dc8ede839e4a9bf15cc2487c8d6fea5ad850d9"
 dependencies = [
+ "hex",
  "opentelemetry",
  "opentelemetry_sdk",
- "prost 0.12.6",
- "tonic",
+ "prost 0.13.1",
+ "serde",
+ "tonic 0.12.3",
 ]
 
 [[package]]
 name = "opentelemetry_sdk"
-version = "0.23.0"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae312d58eaa90a82d2e627fd86e075cf5230b3f11794e2ed74199ebbe572d4fd"
+checksum = "692eac490ec80f24a17828d49b40b60f5aeaccdfe6a503f939713afd22bc28df"
 dependencies = [
  "async-trait",
  "futures-channel",
  "futures-executor",
  "futures-util",
  "glob",
- "lazy_static",
  "once_cell",
  "opentelemetry",
- "ordered-float 4.2.2",
  "percent-encoding",
  "rand 0.8.5",
  "serde_json",
@@ -11128,16 +11058,13 @@ dependencies = [
 [[package]]
 name = "orc-rust"
 version = "0.3.1"
-source = "git+https://github.com/youngsofun/datafusion-orc?rev=1745375#174537521e65bdf195e5fd4eabb59a1c2e724314"
+source = "git+https://github.com/datafuse-extras/datafusion-orc?rev=03372b97#03372b9744825a629edf76071b3315dcd99c2013"
 dependencies = [
  "arrow",
  "async-trait",
  "bytes",
  "chrono",
  "chrono-tz 0.9.0",
- "datafusion",
- "datafusion-expr",
- "datafusion-physical-expr",
  "fallible-streaming-iterator",
  "flate2",
  "futures",
@@ -11145,7 +11072,6 @@ dependencies = [
  "lz4_flex",
  "lzokay-native",
  "num",
- "object_store 0.10.2",
  "prost 0.12.6",
  "snafu 0.8.4",
  "snap",
@@ -11218,8 +11144,7 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 [[package]]
 name = "ownedbytes"
 version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558"
+source = "git+https://github.com/datafuse-extras/tantivy?rev=7502370#7502370b68e6822a687ee071660e350b67808533"
 dependencies = [
  "stable_deref_trait",
 ]
@@ -11308,7 +11233,7 @@ version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "instant",
  "libc",
  "redox_syscall 0.2.16",
@@ -11322,7 +11247,7 @@ version = "0.9.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "redox_syscall 0.5.3",
  "smallvec",
@@ -11354,7 +11279,7 @@ dependencies = [
  "lz4_flex",
  "num",
  "num-bigint",
- "object_store 0.10.2",
+ "object_store",
  "paste",
  "seq-macro",
  "snap",
@@ -11762,7 +11687,7 @@ version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a62fea1692d80a000126f9b28d865012a160b80000abb53ccf152b428222c155"
 dependencies = [
- "proc-macro-crate",
+ "proc-macro-crate 3.1.0",
  "proc-macro2",
  "quote",
  "syn 2.0.58",
@@ -11776,7 +11701,7 @@ checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce"
 dependencies = [
  "autocfg",
  "bitflags 1.3.2",
- "cfg-if 1.0.0",
+ "cfg-if",
  "concurrent-queue",
  "libc",
  "log",
@@ -11790,11 +11715,11 @@ version = "3.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cc2790cd301dec6cd3b7a025e4815cf825724a51c98dccfe6a3e55f05ffb6511"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "concurrent-queue",
  "hermit-abi 0.4.0",
  "pin-project-lite",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "tracing",
  "windows-sys 0.59.0",
 ]
@@ -11832,7 +11757,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059"
 dependencies = [
  "backtrace",
- "cfg-if 1.0.0",
+ "cfg-if",
  "findshlibs",
  "inferno",
  "libc",
@@ -11857,12 +11782,6 @@ dependencies = [
  "zerocopy",
 ]
 
-[[package]]
-name = "pratt"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e31bbc12f7936a7b195790dd6d9b982b66c54f45ff6766decf25c44cac302dce"
-
 [[package]]
 name = "pratt"
 version = "0.4.0"
@@ -11940,6 +11859,16 @@ dependencies = [
  "elliptic-curve 0.13.8",
 ]
 
+[[package]]
+name = "proc-macro-crate"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919"
+dependencies = [
+ "once_cell",
+ "toml_edit 0.19.15",
+]
+
 [[package]]
 name = "proc-macro-crate"
 version = "3.1.0"
@@ -12041,7 +11970,7 @@ dependencies = [
  "hex",
  "lazy_static",
  "procfs-core",
- "rustix 0.38.34",
+ "rustix 0.38.37",
 ]
 
 [[package]]
@@ -12067,9 +11996,9 @@ dependencies = [
 
 [[package]]
 name = "proj4rs"
-version = "0.1.3"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5847217d30c155a90885e9a2ad25aa8f572c5be36664e114182b4c317b2eb5bb"
+checksum = "fed6bd261b7a9ec9308845b375e7d8bcf19394b30d2fcc150bd9c04b2fd14788"
 dependencies = [
  "console_log",
  "crs-definitions",
@@ -12079,7 +12008,6 @@ dependencies = [
  "thiserror",
  "wasm-bindgen",
  "web-sys",
- "wee_alloc",
 ]
 
 [[package]]
@@ -12334,7 +12262,7 @@ version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "indoc",
  "libc",
  "memoffset",
@@ -12715,7 +12643,7 @@ dependencies = [
 [[package]]
 name = "recursive"
 version = "0.1.1"
-source = "git+https://github.com/zhang2014/recursive.git?rev=6af35a1#6af35a1e59e7050f86ee19fbd0a79535d016c87d"
+source = "git+https://github.com/datafuse-extras/recursive.git?rev=6af35a1#6af35a1e59e7050f86ee19fbd0a79535d016c87d"
 dependencies = [
  "recursive-proc-macro-impl",
  "stacker",
@@ -12724,7 +12652,7 @@ dependencies = [
 [[package]]
 name = "recursive-proc-macro-impl"
 version = "0.1.1"
-source = "git+https://github.com/zhang2014/recursive.git?rev=6af35a1#6af35a1e59e7050f86ee19fbd0a79535d016c87d"
+source = "git+https://github.com/datafuse-extras/recursive.git?rev=6af35a1#6af35a1e59e7050f86ee19fbd0a79535d016c87d"
 dependencies = [
  "quote",
  "syn 2.0.58",
@@ -12732,9 +12660,9 @@ dependencies = [
 
 [[package]]
 name = "redis"
-version = "0.26.1"
+version = "0.27.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e902a69d09078829137b4a5d9d082e0490393537badd7c91a3d69d14639e115f"
+checksum = "81cccf17a692ce51b86564334614d72dcae1def0fd5ecebc9f02956da74352b5"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -12757,7 +12685,7 @@ dependencies = [
  "sha1_smol",
  "socket2 0.5.7",
  "tokio",
- "tokio-retry",
+ "tokio-retry2",
  "tokio-rustls 0.26.0",
  "tokio-util",
  "url",
@@ -12892,7 +12820,7 @@ dependencies = [
  "percent-encoding",
  "quick-xml 0.35.0",
  "rand 0.8.5",
- "reqwest 0.12.5",
+ "reqwest",
  "rsa 0.9.6",
  "rust-ini 0.21.1",
  "serde",
@@ -12901,42 +12829,6 @@ dependencies = [
  "sha2",
 ]
 
-[[package]]
-name = "reqwest"
-version = "0.11.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
-dependencies = [
- "base64 0.21.7",
- "bytes",
- "encoding_rs",
- "futures-core",
- "futures-util",
- "h2 0.3.26",
- "http 0.2.12",
- "http-body 0.4.6",
- "hyper 0.14.30",
- "ipnet",
- "js-sys",
- "log",
- "mime",
- "once_cell",
- "percent-encoding",
- "pin-project-lite",
- "serde",
- "serde_json",
- "serde_urlencoded",
- "sync_wrapper 0.1.2",
- "system-configuration",
- "tokio",
- "tower-service",
- "url",
- "wasm-bindgen",
- "wasm-bindgen-futures",
- "web-sys",
- "winreg 0.50.0",
-]
-
 [[package]]
 name = "reqwest"
 version = "0.12.5"
@@ -12945,6 +12837,7 @@ checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37"
 dependencies = [
  "base64 0.22.1",
  "bytes",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "h2 0.4.5",
@@ -12994,7 +12887,7 @@ checksum = "1a1c6019977b5e51744bff6ac8ef399d8ab52bca845623e76e8e058219e5e97e"
 dependencies = [
  "hickory-resolver",
  "once_cell",
- "reqwest 0.12.5",
+ "reqwest",
 ]
 
 [[package]]
@@ -13059,7 +12952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
 dependencies = [
  "cc",
- "cfg-if 1.0.0",
+ "cfg-if",
  "getrandom 0.2.15",
  "libc",
  "spin",
@@ -13276,15 +13169,6 @@ dependencies = [
  "smallvec",
 ]
 
-[[package]]
-name = "rtrb"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99e704dd104faf2326a320140f70f0b736d607c1caa1b1748a6c568a79819109"
-dependencies = [
- "cache-padded",
-]
-
 [[package]]
 name = "rtrb"
 version = "0.3.1"
@@ -13297,7 +13181,7 @@ version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "ordered-multimap 0.4.3",
 ]
 
@@ -13307,7 +13191,7 @@ version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "ordered-multimap 0.7.3",
  "trim-in-place",
 ]
@@ -13398,9 +13282,9 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.38.34"
+version = "0.38.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
+checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
 dependencies = [
  "bitflags 2.6.0",
  "errno",
@@ -13536,7 +13420,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63"
 dependencies = [
  "bitflags 2.6.0",
- "cfg-if 1.0.0",
+ "cfg-if",
  "clipboard-win",
  "fd-lock",
  "home",
@@ -13906,7 +13790,7 @@ version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "cpufeatures",
  "digest",
 ]
@@ -13923,7 +13807,7 @@ version = "0.10.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "cpufeatures",
  "digest",
 ]
@@ -14126,6 +14010,9 @@ name = "smallvec"
 version = "1.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "snafu"
@@ -14215,7 +14102,7 @@ dependencies = [
  "arrayref",
  "bumpalo",
  "bytes",
- "cfg-if 1.0.0",
+ "cfg-if",
  "faststr",
  "itoa",
  "page_size",
@@ -14329,12 +14216,11 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.47.0"
+version = "0.50.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25"
+checksum = "b2e5b515a2bd5168426033e9efbfd05500114833916f1d5c268f938b4ee130ac"
 dependencies = [
  "log",
- "sqlparser_derive",
 ]
 
 [[package]]
@@ -14347,30 +14233,213 @@ dependencies = [
 ]
 
 [[package]]
-name = "sqlparser_derive"
-version = "0.2.2"
+name = "sqlx"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554"
+checksum = "93334716a037193fac19df402f8571269c84a00852f6a7066b5d2616dcd64d3e"
 dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.58",
+ "sqlx-core",
+ "sqlx-macros",
+ "sqlx-mysql",
+ "sqlx-postgres",
+ "sqlx-sqlite",
 ]
 
 [[package]]
-name = "stable_deref_trait"
-version = "1.2.0"
+name = "sqlx-core"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
-
-[[package]]
-name = "stacker"
+checksum = "d4d8060b456358185f7d50c55d9b5066ad956956fddec42ee2e8567134a8936e"
+dependencies = [
+ "atoi",
+ "byteorder",
+ "bytes",
+ "crc",
+ "crossbeam-queue",
+ "either",
+ "event-listener 5.3.1",
+ "futures-channel",
+ "futures-core",
+ "futures-intrusive",
+ "futures-io",
+ "futures-util",
+ "hashbrown 0.14.5",
+ "hashlink 0.9.1",
+ "hex",
+ "indexmap 2.4.0",
+ "log",
+ "memchr",
+ "once_cell",
+ "paste",
+ "percent-encoding",
+ "serde",
+ "serde_json",
+ "sha2",
+ "smallvec",
+ "sqlformat",
+ "thiserror",
+ "tokio",
+ "tokio-stream",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "sqlx-macros"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cac0692bcc9de3b073e8d747391827297e075c7710ff6276d9f7a1f3d58c6657"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "sqlx-core",
+ "sqlx-macros-core",
+ "syn 2.0.58",
+]
+
+[[package]]
+name = "sqlx-macros-core"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1804e8a7c7865599c9c79be146dc8a9fd8cc86935fa641d3ea58e5f0688abaa5"
+dependencies = [
+ "dotenvy",
+ "either",
+ "heck 0.5.0",
+ "hex",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "sha2",
+ "sqlx-core",
+ "sqlx-mysql",
+ "sqlx-postgres",
+ "sqlx-sqlite",
+ "syn 2.0.58",
+ "tempfile",
+ "tokio",
+ "url",
+]
+
+[[package]]
+name = "sqlx-mysql"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64bb4714269afa44aef2755150a0fc19d756fb580a67db8885608cf02f47d06a"
+dependencies = [
+ "atoi",
+ "base64 0.22.1",
+ "bitflags 2.6.0",
+ "byteorder",
+ "bytes",
+ "crc",
+ "digest",
+ "dotenvy",
+ "either",
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-util",
+ "generic-array",
+ "hex",
+ "hkdf",
+ "hmac",
+ "itoa",
+ "log",
+ "md-5",
+ "memchr",
+ "once_cell",
+ "percent-encoding",
+ "rand 0.8.5",
+ "rsa 0.9.6",
+ "serde",
+ "sha1",
+ "sha2",
+ "smallvec",
+ "sqlx-core",
+ "stringprep",
+ "thiserror",
+ "tracing",
+ "whoami",
+]
+
+[[package]]
+name = "sqlx-postgres"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fa91a732d854c5d7726349bb4bb879bb9478993ceb764247660aee25f67c2f8"
+dependencies = [
+ "atoi",
+ "base64 0.22.1",
+ "bitflags 2.6.0",
+ "byteorder",
+ "crc",
+ "dotenvy",
+ "etcetera",
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-util",
+ "hex",
+ "hkdf",
+ "hmac",
+ "home",
+ "itoa",
+ "log",
+ "md-5",
+ "memchr",
+ "once_cell",
+ "rand 0.8.5",
+ "serde",
+ "serde_json",
+ "sha2",
+ "smallvec",
+ "sqlx-core",
+ "stringprep",
+ "thiserror",
+ "tracing",
+ "whoami",
+]
+
+[[package]]
+name = "sqlx-sqlite"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5b2cf34a45953bfd3daaf3db0f7a7878ab9b7a6b91b422d24a7a9e4c857b680"
+dependencies = [
+ "atoi",
+ "flume",
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-intrusive",
+ "futures-util",
+ "libsqlite3-sys",
+ "log",
+ "percent-encoding",
+ "serde",
+ "serde_urlencoded",
+ "sqlx-core",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
+[[package]]
+name = "stacker"
 version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce"
 dependencies = [
  "cc",
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "psm",
  "winapi",
@@ -14431,19 +14500,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"
 
 [[package]]
-name = "stringslice"
-version = "0.2.0"
+name = "stringprep"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8dad614e9c8a0773603982e0c1d3192f2bce1c0ee193caf0526b8ae3c6c3e38c"
+checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+ "unicode-properties",
+]
 
 [[package]]
-name = "strip-ansi-escapes"
+name = "stringslice"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa"
-dependencies = [
- "vte",
-]
+checksum = "8dad614e9c8a0773603982e0c1d3192f2bce1c0ee193caf0526b8ae3c6c3e38c"
 
 [[package]]
 name = "strsim"
@@ -14714,7 +14785,7 @@ version = "0.30.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "core-foundation-sys",
  "libc",
  "ntapi",
@@ -14723,27 +14794,6 @@ dependencies = [
  "windows 0.52.0",
 ]
 
-[[package]]
-name = "system-configuration"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
-dependencies = [
- "bitflags 1.3.2",
- "core-foundation",
- "system-configuration-sys",
-]
-
-[[package]]
-name = "system-configuration-sys"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
-dependencies = [
- "core-foundation-sys",
- "libc",
-]
-
 [[package]]
 name = "system-interface"
 version = "0.27.2"
@@ -14755,7 +14805,7 @@ dependencies = [
  "cap-std",
  "fd-lock",
  "io-lifetimes 2.0.3",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "windows-sys 0.52.0",
  "winx",
 ]
@@ -14769,8 +14819,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
 [[package]]
 name = "tantivy"
 version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8d0582f186c0a6d55655d24543f15e43607299425c5ad8352c242b914b31856"
+source = "git+https://github.com/datafuse-extras/tantivy?rev=7502370#7502370b68e6822a687ee071660e350b67808533"
 dependencies = [
  "aho-corasick",
  "arc-swap",
@@ -14820,8 +14869,7 @@ dependencies = [
 [[package]]
 name = "tantivy-bitpacker"
 version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df"
+source = "git+https://github.com/datafuse-extras/tantivy?rev=7502370#7502370b68e6822a687ee071660e350b67808533"
 dependencies = [
  "bitpacking 0.9.2",
 ]
@@ -14829,8 +14877,7 @@ dependencies = [
 [[package]]
 name = "tantivy-columnar"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e"
+source = "git+https://github.com/datafuse-extras/tantivy?rev=7502370#7502370b68e6822a687ee071660e350b67808533"
 dependencies = [
  "downcast-rs",
  "fastdivide",
@@ -14845,8 +14892,7 @@ dependencies = [
 [[package]]
 name = "tantivy-common"
 version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4"
+source = "git+https://github.com/datafuse-extras/tantivy?rev=7502370#7502370b68e6822a687ee071660e350b67808533"
 dependencies = [
  "async-trait",
  "byteorder",
@@ -14869,8 +14915,7 @@ dependencies = [
 [[package]]
 name = "tantivy-jieba"
 version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f2fe65c125f0d76d06f0f2ce9fbb9287b53f0dafb51a6270d984a840e2f16c1"
+source = "git+https://github.com/datafuse-extras/tantivy-jieba?rev=0e300e9#0e300e9085651b7e6659dfcc7b0ea0fa9cab09c2"
 dependencies = [
  "jieba-rs",
  "lazy_static",
@@ -14880,8 +14925,7 @@ dependencies = [
 [[package]]
 name = "tantivy-query-grammar"
 version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82"
+source = "git+https://github.com/datafuse-extras/tantivy?rev=7502370#7502370b68e6822a687ee071660e350b67808533"
 dependencies = [
  "nom",
 ]
@@ -14889,8 +14933,7 @@ dependencies = [
 [[package]]
 name = "tantivy-sstable"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e"
+source = "git+https://github.com/datafuse-extras/tantivy?rev=7502370#7502370b68e6822a687ee071660e350b67808533"
 dependencies = [
  "tantivy-bitpacker",
  "tantivy-common",
@@ -14901,8 +14944,7 @@ dependencies = [
 [[package]]
 name = "tantivy-stacker"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8"
+source = "git+https://github.com/datafuse-extras/tantivy?rev=7502370#7502370b68e6822a687ee071660e350b67808533"
 dependencies = [
  "murmurhash32",
  "rand_distr",
@@ -14912,8 +14954,7 @@ dependencies = [
 [[package]]
 name = "tantivy-tokenizer-api"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04"
+source = "git+https://github.com/datafuse-extras/tantivy?rev=7502370#7502370b68e6822a687ee071660e350b67808533"
 dependencies = [
  "serde",
 ]
@@ -14924,6 +14965,12 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
+[[package]]
+name = "target-features"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5"
+
 [[package]]
 name = "target-lexicon"
 version = "0.12.16"
@@ -14945,10 +14992,10 @@ version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "fastrand 2.1.0",
  "once_cell",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "windows-sys 0.59.0",
 ]
 
@@ -14977,7 +15024,7 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7"
 dependencies = [
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "windows-sys 0.48.0",
 ]
 
@@ -15024,7 +15071,7 @@ version = "1.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "once_cell",
 ]
 
@@ -15140,9 +15187,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.39.2"
+version = "1.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"
+checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb"
 dependencies = [
  "backtrace",
  "bytes",
@@ -15211,6 +15258,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-retry2"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "903934dba1c4c2f2e9cb460ef10b5695e0b0ecad3bf9ee7c8675e540c5e8b2d1"
+dependencies = [
+ "pin-project",
+ "rand 0.8.5",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.24.1"
@@ -15245,9 +15303,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
+checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -15355,14 +15413,14 @@ checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13"
 dependencies = [
  "async-stream",
  "async-trait",
- "axum",
+ "axum 0.6.20",
  "base64 0.21.7",
  "bytes",
  "h2 0.3.26",
  "http 0.2.12",
  "http-body 0.4.6",
  "hyper 0.14.30",
- "hyper-timeout",
+ "hyper-timeout 0.4.1",
  "percent-encoding",
  "pin-project",
  "prost 0.12.6",
@@ -15372,7 +15430,37 @@ dependencies = [
  "tokio",
  "tokio-rustls 0.25.0",
  "tokio-stream",
- "tower",
+ "tower 0.4.13",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tonic"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "axum 0.7.7",
+ "base64 0.22.1",
+ "bytes",
+ "h2 0.4.5",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "http-body-util",
+ "hyper 1.4.1",
+ "hyper-timeout 0.5.1",
+ "hyper-util",
+ "percent-encoding",
+ "pin-project",
+ "prost 0.13.1",
+ "socket2 0.5.7",
+ "tokio",
+ "tokio-stream",
+ "tower 0.4.13",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -15401,7 +15489,7 @@ dependencies = [
  "prost-types",
  "tokio",
  "tokio-stream",
- "tonic",
+ "tonic 0.11.0",
 ]
 
 [[package]]
@@ -15424,6 +15512,20 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "tower"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project-lite",
+ "sync_wrapper 0.1.2",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "tower-layer"
 version = "0.3.3"
@@ -15448,18 +15550,6 @@ dependencies = [
  "tracing-core",
 ]
 
-[[package]]
-name = "tracing-appender"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf"
-dependencies = [
- "crossbeam-channel",
- "thiserror",
- "time",
- "tracing-subscriber",
-]
-
 [[package]]
 name = "tracing-attributes"
 version = "0.1.27"
@@ -15560,7 +15650,7 @@ version = "1.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "rand 0.8.5",
  "static_assertions",
 ]
@@ -15577,7 +15667,16 @@ version = "0.19.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600"
 dependencies = [
- "typed-builder-macro",
+ "typed-builder-macro 0.19.1",
+]
+
+[[package]]
+name = "typed-builder"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e14ed59dc8b7b26cacb2a92bad2e8b1f098806063898ab42a3bd121d7d45e75"
+dependencies = [
+ "typed-builder-macro 0.20.0",
 ]
 
 [[package]]
@@ -15591,6 +15690,17 @@ dependencies = [
  "syn 2.0.58",
 ]
 
+[[package]]
+name = "typed-builder-macro"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "560b82d656506509d43abe30e0ba64c56b1953ab3d4fe7ba5902747a7a3cedd5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.58",
+]
+
 [[package]]
 name = "typeid"
 version = "1.0.0"
@@ -15708,6 +15818,12 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unicode-properties"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
+
 [[package]]
 name = "unicode-segmentation"
 version = "1.11.0"
@@ -15879,7 +15995,7 @@ checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566"
 dependencies = [
  "anyhow",
  "cargo_metadata",
- "cfg-if 1.0.0",
+ "cfg-if",
  "gix",
  "regex",
  "rustc_version",
@@ -15933,7 +16049,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tokio-stream",
- "tower",
+ "tower 0.4.13",
  "tracing",
 ]
 
@@ -15974,26 +16090,6 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
 
-[[package]]
-name = "vte"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197"
-dependencies = [
- "utf8parse",
- "vte_generate_state_changes",
-]
-
-[[package]]
-name = "vte_generate_state_changes"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e"
-dependencies = [
- "proc-macro2",
- "quote",
-]
-
 [[package]]
 name = "waker-fn"
 version = "1.2.0"
@@ -16048,7 +16144,7 @@ dependencies = [
  "io-lifetimes 2.0.3",
  "log",
  "once_cell",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "system-interface",
  "thiserror",
  "tracing",
@@ -16057,6 +16153,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "wasite"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
+
 [[package]]
 name = "wasix"
 version = "0.12.21"
@@ -16072,7 +16174,7 @@ version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "once_cell",
  "wasm-bindgen-macro",
 ]
@@ -16098,7 +16200,7 @@ version = "0.4.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "js-sys",
  "wasm-bindgen",
  "web-sys",
@@ -16196,7 +16298,7 @@ dependencies = [
  "async-trait",
  "bincode 1.3.3",
  "bumpalo",
- "cfg-if 1.0.0",
+ "cfg-if",
  "encoding_rs",
  "fxprof-processed-profile",
  "gimli 0.28.1",
@@ -16208,7 +16310,7 @@ dependencies = [
  "once_cell",
  "paste",
  "rayon",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "semver",
  "serde",
  "serde_derive",
@@ -16237,7 +16339,7 @@ version = "20.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7515c4d24c8b55c0feab67e3d52a42f999fda8b9cfafbd69a82ed6bcf299d26e"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -16251,7 +16353,7 @@ dependencies = [
  "bincode 1.3.3",
  "directories-next",
  "log",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "serde",
  "serde_derive",
  "sha2",
@@ -16288,7 +16390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "57ec2d9a4b9990bea53a5dfd689d48663dbd19a46903eaf73e2022b3d1ef20d3"
 dependencies = [
  "anyhow",
- "cfg-if 1.0.0",
+ "cfg-if",
  "cranelift-codegen",
  "cranelift-control",
  "cranelift-entity",
@@ -16339,8 +16441,8 @@ checksum = "4dbdf3053e7e7ced0cd4ed76579995b62169a1a43696890584eae2de2e33bf54"
 dependencies = [
  "anyhow",
  "cc",
- "cfg-if 1.0.0",
- "rustix 0.38.34",
+ "cfg-if",
+ "rustix 0.38.37",
  "wasmtime-asm-macros",
  "wasmtime-versioned-export-macros",
  "windows-sys 0.52.0",
@@ -16354,7 +16456,7 @@ checksum = "983ca409f2cd66385ce49486c022da0128acb7910c055beb5230998b49c6084c"
 dependencies = [
  "object 0.33.0",
  "once_cell",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "wasmtime-versioned-export-macros",
 ]
 
@@ -16364,7 +16466,7 @@ version = "20.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ede45379f3b4d395d8947006de8043801806099a240a26db553919b68e96ab15"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "windows-sys 0.52.0",
 ]
@@ -16377,7 +16479,7 @@ checksum = "65019d29d175c567b84173f2adf3b7a3af6d5592f8fe510dccae55d2569ec0d2"
 dependencies = [
  "anyhow",
  "cc",
- "cfg-if 1.0.0",
+ "cfg-if",
  "encoding_rs",
  "indexmap 2.4.0",
  "libc",
@@ -16387,7 +16489,7 @@ dependencies = [
  "memoffset",
  "paste",
  "psm",
- "rustix 0.38.34",
+ "rustix 0.38.37",
  "sptr",
  "wasm-encoder 0.202.0",
  "wasmtime-asm-macros",
@@ -16519,15 +16621,13 @@ dependencies = [
 ]
 
 [[package]]
-name = "wee_alloc"
-version = "0.4.5"
+name = "whoami"
+version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbb3b5a6b2bb17cb6ad44a2e68a43e8d2722c997da10e928665c72ec6c0a0b8e"
+checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d"
 dependencies = [
- "cfg-if 0.1.10",
- "libc",
- "memory_units",
- "winapi",
+ "redox_syscall 0.5.3",
+ "wasite",
 ]
 
 [[package]]
@@ -16913,7 +17013,7 @@ version = "0.50.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "windows-sys 0.48.0",
 ]
 
@@ -16923,7 +17023,7 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "windows-sys 0.48.0",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index ad5b867fb9869..f54b0a9495c80 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -253,10 +253,11 @@ jwt-simple = "0.11.0"
 libc = { version = "0.2.158" }
 match-template = "0.0.1"
 mysql_async = { version = "0.34", default-features = false, features = ["native-tls-tls"] }
-object_store_opendal = "0.47"
+object_store_opendal = "0.46"
+opendal_compat = { version = "1" }
 once_cell = "1.15.0"
 openai_api_rust = "0.1"
-opendal = { version = "0.50.0", features = [
+opendal = { version = "0.50.1", features = [
     "layers-fastrace",
     "layers-prometheus-client",
     "layers-async-backtrace",
diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml
index ab434f8558db9..3c8c81e03834e 100644
--- a/src/common/base/Cargo.toml
+++ b/src/common/base/Cargo.toml
@@ -60,6 +60,8 @@ tikv-jemalloc-sys = "0.5.2"
 tokio = { workspace = true }
 unicode-segmentation = "1.10.1"
 uuid = { workspace = true }
+reqwest = { workspace = true }
+reqwest-hickory-resolver = { workspace = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { version = "^0.16" }
diff --git a/src/common/base/src/http_client.rs b/src/common/base/src/http_client.rs
new file mode 100644
index 0000000000000..af981449cb007
--- /dev/null
+++ b/src/common/base/src/http_client.rs
@@ -0,0 +1,88 @@
+// Copyright 2024 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::env;
+use std::sync::Arc;
+use std::sync::LazyLock;
+use std::time::Duration;
+
+use reqwest_hickory_resolver::HickoryResolver;
+
+/// Global shared hickory resolver.
+static GLOBAL_HICKORY_RESOLVER: LazyLock<Arc<HickoryResolver>> =
+    LazyLock::new(|| Arc::new(HickoryResolver::default()));
+
+/// Global shared http client.
+///
+/// Please create your own http client if you want dedicated http connection pool.
+pub static GLOBAL_HTTP_CLIENT: LazyLock<HttpClient> = LazyLock::new(HttpClient::new);
+
+/// HttpClient that used by databend.
+pub struct HttpClient {
+    client: reqwest::Client,
+}
+
+impl Default for HttpClient {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl HttpClient {
+    /// Create a new http client.
+    ///
+    /// # Notes
+    ///
+    /// This client is optimized for interact with storage services.
+    /// Please tune the settings if you want to use it for other purposes.
+    pub fn new() -> Self {
+        let mut builder = reqwest::ClientBuilder::new();
+
+        // Disable http2 for better performance.
+        builder = builder.http1_only();
+
+        // Set dns resolver.
+        builder = builder.dns_resolver(GLOBAL_HICKORY_RESOLVER.clone());
+
+        // Pool max idle per host controls connection pool size.
+        // Default to no limit, set to `0` for disable it.
+        let pool_max_idle_per_host = env::var("_DATABEND_INTERNAL_POOL_MAX_IDLE_PER_HOST")
+            .ok()
+            .and_then(|v| v.parse::<usize>().ok())
+            .unwrap_or(usize::MAX);
+        builder = builder.pool_max_idle_per_host(pool_max_idle_per_host);
+
+        // Connect timeout default to 30s.
+        let connect_timeout = env::var("_DATABEND_INTERNAL_CONNECT_TIMEOUT")
+            .ok()
+            .and_then(|v| v.parse::<u64>().ok())
+            .unwrap_or(30);
+        builder = builder.connect_timeout(Duration::from_secs(connect_timeout));
+
+        // Enable TCP keepalive if set.
+        if let Ok(v) = env::var("_DATABEND_INTERNAL_TCP_KEEPALIVE") {
+            if let Ok(v) = v.parse::<u64>() {
+                builder = builder.tcp_keepalive(Duration::from_secs(v));
+            }
+        }
+
+        let client = builder.build().expect("http client must be created");
+        HttpClient { client }
+    }
+
+    /// Get the inner reqwest client.
+    pub fn inner(&self) -> reqwest::Client {
+        self.client.clone()
+    }
+}
diff --git a/src/common/base/src/lib.rs b/src/common/base/src/lib.rs
index 31060daffe9ee..b66ab4a8eecde 100644
--- a/src/common/base/src/lib.rs
+++ b/src/common/base/src/lib.rs
@@ -31,6 +31,7 @@ pub mod containers;
 pub mod display;
 pub mod future;
 pub mod headers;
+pub mod http_client;
 pub mod mem_allocator;
 pub mod rangemap;
 pub mod runtime;
diff --git a/src/common/storage/Cargo.toml b/src/common/storage/Cargo.toml
index bacc5c9ecfd62..4ddda845587ae 100644
--- a/src/common/storage/Cargo.toml
+++ b/src/common/storage/Cargo.toml
@@ -30,7 +30,6 @@ parquet = { workspace = true }
 prometheus-client = { workspace = true }
 regex = { workspace = true }
 reqwest = { workspace = true }
-reqwest-hickory-resolver = { workspace = true }
 serde = { workspace = true }
 thiserror = { workspace = true }
 
diff --git a/src/common/storage/src/operator.rs b/src/common/storage/src/operator.rs
index 3923431877319..76dffbf609c14 100644
--- a/src/common/storage/src/operator.rs
+++ b/src/common/storage/src/operator.rs
@@ -16,12 +16,11 @@ use std::env;
 use std::io::Error;
 use std::io::ErrorKind;
 use std::io::Result;
-use std::sync::Arc;
-use std::sync::LazyLock;
 use std::time::Duration;
 
 use anyhow::anyhow;
 use databend_common_base::base::GlobalInstance;
+use databend_common_base::http_client::GLOBAL_HTTP_CLIENT;
 use databend_common_base::runtime::GlobalIORuntime;
 use databend_common_base::runtime::TrySpawn;
 use databend_common_exception::ErrorCode;
@@ -54,22 +53,11 @@ use opendal::raw::HttpClient;
 use opendal::services;
 use opendal::Builder;
 use opendal::Operator;
-use reqwest_hickory_resolver::HickoryResolver;
 
 use crate::metrics_layer::METRICS_LAYER;
 use crate::runtime_layer::RuntimeLayer;
 use crate::StorageConfig;
 
-/// The global dns resolver for opendal.
-static GLOBAL_HICKORY_RESOLVER: LazyLock<Arc<HickoryResolver>> =
-    LazyLock::new(|| Arc::new(HickoryResolver::default()));
-
-static GLOBAL_HTTP_CLIENT: LazyLock<HttpClient> = LazyLock::new(|| {
-    new_storage_http_client().unwrap_or_else(|err| {
-        panic!("http client must be created successfully, but failed for {err}")
-    })
-});
-
 /// init_operator will init an opendal operator based on storage config.
 pub fn init_operator(cfg: &StorageParams) -> Result<Operator> {
     let op = match &cfg {
@@ -173,7 +161,7 @@ pub fn init_azblob_operator(cfg: &StorageAzblobConfig) -> Result<impl Builder> {
         // Credential
         .account_name(&cfg.account_name)
         .account_key(&cfg.account_key)
-        .http_client(GLOBAL_HTTP_CLIENT.clone());
+        .http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner()));
 
     Ok(builder)
 }
@@ -198,7 +186,7 @@ fn init_gcs_operator(cfg: &StorageGcsConfig) -> Result<impl Builder> {
         .bucket(&cfg.bucket)
         .root(&cfg.root)
         .credential(&cfg.credential)
-        .http_client(GLOBAL_HTTP_CLIENT.clone());
+        .http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner()));
 
     Ok(builder)
 }
@@ -295,7 +283,7 @@ fn init_s3_operator(cfg: &StorageS3Config) -> Result<impl Builder> {
         builder = builder.enable_virtual_host_style();
     }
 
-    builder = builder.http_client(GLOBAL_HTTP_CLIENT.clone());
+    builder = builder.http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner()));
 
     Ok(builder)
 }
@@ -312,7 +300,7 @@ fn init_obs_operator(cfg: &StorageObsConfig) -> Result<impl Builder> {
         // Credential
         .access_key_id(&cfg.access_key_id)
         .secret_access_key(&cfg.secret_access_key)
-        .http_client(GLOBAL_HTTP_CLIENT.clone());
+        .http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner()));
 
     Ok(builder)
 }
@@ -328,7 +316,7 @@ fn init_oss_operator(cfg: &StorageOssConfig) -> Result<impl Builder> {
         .root(&cfg.root)
         .server_side_encryption(&cfg.server_side_encryption)
         .server_side_encryption_key_id(&cfg.server_side_encryption_key_id)
-        .http_client(GLOBAL_HTTP_CLIENT.clone());
+        .http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner()));
 
     Ok(builder)
 }
@@ -361,7 +349,7 @@ fn init_cos_operator(cfg: &StorageCosConfig) -> Result<impl Builder> {
         .secret_key(&cfg.secret_key)
         .bucket(&cfg.bucket)
         .root(&cfg.root)
-        .http_client(GLOBAL_HTTP_CLIENT.clone());
+        .http_client(HttpClient::with(GLOBAL_HTTP_CLIENT.inner()));
 
     Ok(builder)
 }
@@ -378,41 +366,6 @@ fn init_huggingface_operator(cfg: &StorageHuggingfaceConfig) -> Result<impl Buil
     Ok(builder)
 }
 
-/// Create a new http client for storage.
-fn new_storage_http_client() -> Result<HttpClient> {
-    let mut builder = reqwest::ClientBuilder::new();
-
-    // Disable http2 for better performance.
-    builder = builder.http1_only();
-
-    // Set dns resolver.
-    builder = builder.dns_resolver(GLOBAL_HICKORY_RESOLVER.clone());
-
-    // Pool max idle per host controls connection pool size.
-    // Default to no limit, set to `0` for disable it.
-    let pool_max_idle_per_host = env::var("_DATABEND_INTERNAL_POOL_MAX_IDLE_PER_HOST")
-        .ok()
-        .and_then(|v| v.parse::<usize>().ok())
-        .unwrap_or(usize::MAX);
-    builder = builder.pool_max_idle_per_host(pool_max_idle_per_host);
-
-    // Connect timeout default to 30s.
-    let connect_timeout = env::var("_DATABEND_INTERNAL_CONNECT_TIMEOUT")
-        .ok()
-        .and_then(|v| v.parse::<u64>().ok())
-        .unwrap_or(30);
-    builder = builder.connect_timeout(Duration::from_secs(connect_timeout));
-
-    // Enable TCP keepalive if set.
-    if let Ok(v) = env::var("_DATABEND_INTERNAL_TCP_KEEPALIVE") {
-        if let Ok(v) = v.parse::<u64>() {
-            builder = builder.tcp_keepalive(Duration::from_secs(v));
-        }
-    }
-
-    Ok(HttpClient::build(builder)?)
-}
-
 pub struct DatabendRetryInterceptor;
 
 impl RetryInterceptor for DatabendRetryInterceptor {
diff --git a/src/meta/app/Cargo.toml b/src/meta/app/Cargo.toml
index d88de76ebe0bc..485a8a7d863b6 100644
--- a/src/meta/app/Cargo.toml
+++ b/src/meta/app/Cargo.toml
@@ -40,6 +40,7 @@ serde_json = { workspace = true }
 sha1 = "0.10.5"
 sha2 = { workspace = true }
 thiserror = { workspace = true }
+reqwest = { workspace = true }
 
 [dev-dependencies]
 anyhow = { workspace = true }
diff --git a/src/meta/app/src/storage/storage_params.rs b/src/meta/app/src/storage/storage_params.rs
index 32af7a146b62c..fa44c04a088c1 100644
--- a/src/meta/app/src/storage/storage_params.rs
+++ b/src/meta/app/src/storage/storage_params.rs
@@ -17,6 +17,7 @@ use std::fmt::Display;
 use std::fmt::Formatter;
 use std::time::Duration;
 
+use databend_common_base::http_client::GLOBAL_HTTP_CLIENT;
 use databend_common_exception::ErrorCode;
 use databend_common_exception::Result;
 use serde::Deserialize;
@@ -123,7 +124,6 @@ impl StorageParams {
     pub async fn auto_detect(self) -> Result<Self> {
         let sp = match self {
             StorageParams::S3(mut s3) if s3.region.is_empty() => {
-                // TODO: endpoint related logic should be moved out from opendal as a new API.
                 // Remove the possible trailing `/` in endpoint.
                 let endpoint = s3.endpoint_url.trim_end_matches('/');
 
@@ -135,22 +135,20 @@ impl StorageParams {
                     format!("https://{}", endpoint)
                 };
 
-                // We should not return error if client create failed, just ignore it.
-                if let Ok(client) = opendal::raw::HttpClient::new() {
-                    // The response itself doesn't important.
-                    let _ = client
-                        .client()
-                        .get(&endpoint)
-                        .timeout(Duration::from_secs(10))
-                        .send()
-                        .await
-                        .map_err(|err| {
-                            ErrorCode::InvalidConfig(format!(
-                                "s3 endpoint_url {} is invalid or incomplete: {err:?}",
-                                s3.endpoint_url
-                            ))
-                        })?;
-                }
+                // The response itself doesn't important.
+                let _ = GLOBAL_HTTP_CLIENT
+                    .inner()
+                    .get(&endpoint)
+                    .timeout(Duration::from_secs(10))
+                    .send()
+                    .await
+                    .map_err(|err| {
+                        ErrorCode::InvalidConfig(format!(
+                            "s3 endpoint_url {} is invalid or incomplete: {err:?}",
+                            s3.endpoint_url
+                        ))
+                    })?;
+
                 s3.region = opendal::services::S3::detect_region(&s3.endpoint_url, &s3.bucket)
                     .await
                     .unwrap_or_default();
diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs b/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs
index a5045f7867168..0d1a87397cb6f 100644
--- a/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs
+++ b/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs
@@ -71,7 +71,7 @@ pub async fn do_vacuum_temporary_files(
             match meta.mode() {
                 EntryMode::DIR => {
                     let life_mills =
-                        match operator.is_exist(&format!("{}finished", de.path())).await? {
+                        match operator.exists(&format!("{}finished", de.path())).await? {
                             true => 0,
                             false => expire_time,
                         };
diff --git a/src/query/ee/tests/it/storages/fuse/operations/virtual_columns.rs b/src/query/ee/tests/it/storages/fuse/operations/virtual_columns.rs
index 6ebd11a7a709a..f39a91ac146bd 100644
--- a/src/query/ee/tests/it/storages/fuse/operations/virtual_columns.rs
+++ b/src/query/ee/tests/it/storages/fuse/operations/virtual_columns.rs
@@ -75,7 +75,7 @@ async fn test_fuse_do_refresh_virtual_column() -> Result<()> {
         for block_meta in block_metas {
             let virtual_loc =
                 TableMetaLocationGenerator::gen_virtual_block_location(&block_meta.location.0);
-            assert!(dal.is_exist(&virtual_loc).await?);
+            assert!(dal.exists(&virtual_loc).await?);
 
             let schema = match storage_format {
                 FuseStorageFormat::Parquet => block_reader.sync_read_schema(&virtual_loc),
diff --git a/src/query/storages/delta/Cargo.toml b/src/query/storages/delta/Cargo.toml
index 428816eac2afb..df637b6d8b146 100644
--- a/src/query/storages/delta/Cargo.toml
+++ b/src/query/storages/delta/Cargo.toml
@@ -22,6 +22,7 @@ databend-storages-common-table-meta = { workspace = true }
 deltalake = { workspace = true }
 fastrace = { workspace = true }
 match-template = "0.0.1"
+opendal_compat = { workspace = true, features = ["v0_50_to_v0_49"] }
 object_store_opendal = { workspace = true }
 parquet = { workspace = true }
 serde = { workspace = true }
diff --git a/src/query/storages/delta/src/table.rs b/src/query/storages/delta/src/table.rs
index 0361a8f725682..7bfc022417edc 100644
--- a/src/query/storages/delta/src/table.rs
+++ b/src/query/storages/delta/src/table.rs
@@ -157,6 +157,7 @@ impl DeltaTable {
     #[async_backtrace::framed]
     pub async fn load(sp: &StorageParams) -> Result<deltalake::table::DeltaTable> {
         let op = init_operator(sp)?;
+        let op = opendal_compat::v0_50_to_v0_49(op);
         let opendal_store = Arc::new(OpendalStore::new(op));
 
         let mut table = DeltaTableBuilder::from_uri(Url::from_directory_path("/").unwrap())
diff --git a/src/query/storages/fuse/src/pruning/bloom_pruner.rs b/src/query/storages/fuse/src/pruning/bloom_pruner.rs
index 24cbd30f59613..06ee5cefd1d04 100644
--- a/src/query/storages/fuse/src/pruning/bloom_pruner.rs
+++ b/src/query/storages/fuse/src/pruning/bloom_pruner.rs
@@ -218,7 +218,7 @@ impl BloomPrunerCreator {
             return Ok(None);
         };
 
-        if self.dal.is_exist(bloom_index_location.0.as_str()).await? {
+        if self.dal.exists(bloom_index_location.0.as_str()).await? {
             info!("bloom index exists, ignore");
             return Ok(None);
         }

From 0c209beff817da977d20242ac8f465248705ef0c Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 23 Oct 2024 12:48:00 +0800
Subject: [PATCH 03/26] Fix cargo.lock

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 270f38540e808..f454764bdb640 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3453,7 +3453,7 @@ dependencies = [
  "geos",
  "geozero 0.13.0",
  "http 1.1.0",
- "opendal 0.50.1",",
+ "opendal 0.50.1",
  "parquet",
  "paste",
  "prost 0.12.6",
@@ -4044,7 +4044,7 @@ dependencies = [
  "flate2",
  "futures",
  "lz4",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "parquet-format-safe",
  "rand 0.8.5",
  "seq-macro",
@@ -4246,7 +4246,7 @@ dependencies = [
  "log",
  "num-derive",
  "num-traits",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "parking_lot 0.12.3",
  "percent-encoding",
  "prqlc",
@@ -4281,7 +4281,7 @@ dependencies = [
  "flagset",
  "futures",
  "log",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "parquet",
  "prometheus-client",
  "regex",
@@ -4310,7 +4310,6 @@ dependencies = [
  "databend-storages-common-table-meta",
  "deltalake",
  "fastrace",
- "maplit",
  "match-template",
  "object_store_opendal",
  "opendal_compat",
@@ -4394,7 +4393,7 @@ dependencies = [
  "itertools 0.10.5",
  "jsonb",
  "log",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4441,7 +4440,8 @@ dependencies = [
  "futures",
  "hive_metastore",
  "log",
-"opendal 0.50.1",
+ "opendal 0.50.1",
+ "parquet",
  "recursive",
  "serde",
  "typetag",
@@ -4554,7 +4554,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "futures-util",
  "log",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "orc-rust",
  "serde",
  "serde_json",
@@ -4591,7 +4591,7 @@ dependencies = [
  "ethnum",
  "futures",
  "log",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4638,7 +4638,7 @@ dependencies = [
  "databend-common-storages-parquet",
  "databend-storages-common-blocks",
  "databend-storages-common-table-meta",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "parquet",
  "serde",
  "serde_json",
@@ -4679,7 +4679,7 @@ dependencies = [
  "enum-as-inner 0.6.0",
  "futures",
  "log",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "parquet",
  "serde",
  "serde_json",
@@ -4743,7 +4743,7 @@ dependencies = [
  "jsonb",
  "log",
  "once_cell",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "parking_lot 0.12.3",
  "regex",
  "serde",
@@ -4997,7 +4997,7 @@ dependencies = [
  "jsonb",
  "jwt-simple 0.11.9",
  "log",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "tantivy",
  "tempfile",
 ]
@@ -5271,7 +5271,7 @@ dependencies = [
  "num",
  "num_cpus",
  "once_cell",
-"opendal 0.50.1",
+ "opendal 0.50.1",
  "opensrv-mysql",
  "opentelemetry",
  "opentelemetry_sdk",
@@ -5474,7 +5474,7 @@ dependencies = [
  "fastrace",
  "futures",
  "log",
-"opendal 0.50.1",
+ "opendal 0.50.1",
 ]
 
 [[package]]
@@ -10858,7 +10858,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "091a08a86c52bd69d7b4be44202d03edf6d0358256805561be2287b5c308fcdf"
 dependencies = [
  "async-trait",
- "opendal 0.49.0",
+ "opendal 0.49.2",
  "opendal 0.50.1",
 ]
 

From 2248c9a4b3d6eb30fbf32d4a5ffe4f9201bcef73 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 23 Oct 2024 13:46:59 +0800
Subject: [PATCH 04/26] Refactor metrics

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 src/common/storage/src/metrics_layer.rs | 612 ++++--------------------
 1 file changed, 102 insertions(+), 510 deletions(-)

diff --git a/src/common/storage/src/metrics_layer.rs b/src/common/storage/src/metrics_layer.rs
index a63cb56ca30c3..6cfadf8ce5d9d 100644
--- a/src/common/storage/src/metrics_layer.rs
+++ b/src/common/storage/src/metrics_layer.rs
@@ -12,565 +12,157 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt;
 use std::fmt::Debug;
-use std::fmt::Formatter;
 use std::sync::Arc;
 use std::sync::LazyLock;
 use std::time::Duration;
-use std::time::Instant;
 
 use databend_common_base::runtime::metrics::register_counter_family;
 use databend_common_base::runtime::metrics::register_histogram_family;
 use databend_common_base::runtime::metrics::FamilyCounter;
 use databend_common_base::runtime::metrics::FamilyHistogram;
-use futures::FutureExt;
-use futures::TryFutureExt;
-use opendal::raw::oio;
+use opendal::layers::observe;
 use opendal::raw::Access;
 use opendal::raw::Layer;
-use opendal::raw::LayeredAccess;
-use opendal::raw::OpBatch;
-use opendal::raw::OpCreateDir;
-use opendal::raw::OpDelete;
-use opendal::raw::OpList;
-use opendal::raw::OpPresign;
-use opendal::raw::OpRead;
-use opendal::raw::OpStat;
-use opendal::raw::OpWrite;
 use opendal::raw::Operation;
-use opendal::raw::RpBatch;
-use opendal::raw::RpCreateDir;
-use opendal::raw::RpDelete;
-use opendal::raw::RpList;
-use opendal::raw::RpPresign;
-use opendal::raw::RpRead;
-use opendal::raw::RpStat;
-use opendal::raw::RpWrite;
-use opendal::Buffer;
 use opendal::ErrorKind;
 use opendal::Scheme;
+use prometheus_client::encoding::EncodeLabel;
+use prometheus_client::encoding::EncodeLabelSet;
+use prometheus_client::encoding::LabelSetEncoder;
 use prometheus_client::metrics::histogram::exponential_buckets;
 
-type OperationLabels = [(&'static str, &'static str); 2];
-type ErrorLabels = [(&'static str, &'static str); 3];
-
 pub static METRICS_LAYER: LazyLock<MetricsLayer> = LazyLock::new(|| MetricsLayer {
-    metrics: Arc::new(MetricsRecorder::new()),
+    metrics: MetricsRecorder::new(),
 });
 
+#[derive(Debug, Clone)]
+pub struct MetricsLayer {
+    metrics: MetricsRecorder,
+}
+
+impl<A: Access> Layer<A> for MetricsLayer {
+    type LayeredAccess = observe::MetricsAccessor<A, MetricsRecorder>;
+
+    fn layer(&self, inner: A) -> Self::LayeredAccess {
+        observe::MetricsLayer::new(self.metrics.clone()).layer(inner)
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct MetricsRecorder {
-    /// Total counter of the specific operation be called.
-    requests_total: FamilyCounter<OperationLabels>,
-    /// Total counter of the errors.
-    errors_total: FamilyCounter<ErrorLabels>,
     /// Latency of the specific operation be called.
-    request_duration_seconds: FamilyHistogram<OperationLabels>,
+    operation_duration_seconds: FamilyHistogram<OperationLabels>,
     /// The histogram of bytes
-    bytes_histogram: FamilyHistogram<OperationLabels>,
-    /// The counter of bytes
-    bytes_total: FamilyCounter<OperationLabels>,
+    operation_bytes: FamilyHistogram<OperationLabels>,
+    /// Total counter of the specific operation be called.
+    operation_errors_total: FamilyCounter<OperationLabels>,
 }
 
 impl MetricsRecorder {
     pub fn new() -> Self {
         MetricsRecorder {
-            requests_total: register_counter_family("opendal_requests"),
-            errors_total: register_counter_family("opendal_errors"),
-            request_duration_seconds: register_histogram_family(
-                "opendal_request_duration_seconds",
+            operation_duration_seconds: register_histogram_family(
+                &observe::METRIC_OPERATION_DURATION_SECONDS.name(),
                 exponential_buckets(0.01, 2.0, 16),
             ),
-            bytes_histogram: register_histogram_family(
-                "opendal_bytes_histogram",
+            operation_bytes: register_histogram_family(
+                &observe::METRIC_OPERATION_BYTES.name(),
                 exponential_buckets(1.0, 2.0, 16),
             ),
-            bytes_total: register_counter_family("opendal_bytes"),
-        }
-    }
-
-    fn increment_errors_total(&self, scheme: Scheme, op: Operation, err: ErrorKind) {
-        let labels = [
-            ("scheme", scheme.into_static()),
-            ("op", op.into_static()),
-            ("err", err.into_static()),
-        ];
-        self.errors_total.get_or_create(&labels).inc();
-    }
-
-    fn increment_request_total(&self, scheme: Scheme, op: Operation) {
-        let labels = [("scheme", scheme.into_static()), ("op", op.into_static())];
-        self.requests_total.get_or_create(&labels).inc();
-    }
-
-    fn observe_bytes_total(&self, scheme: Scheme, op: Operation, bytes: usize) {
-        let labels = [("scheme", scheme.into_static()), ("op", op.into_static())];
-        self.bytes_histogram
-            .get_or_create(&labels)
-            .observe(bytes as f64);
-        self.bytes_total.get_or_create(&labels).inc_by(bytes as u64);
-    }
-
-    fn observe_request_duration(&self, scheme: Scheme, op: Operation, duration: Duration) {
-        let labels = [("scheme", scheme.into_static()), ("op", op.into_static())];
-        self.request_duration_seconds
-            .get_or_create(&labels)
-            .observe(duration.as_secs_f64());
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct MetricsLayer {
-    metrics: Arc<MetricsRecorder>,
-}
-
-impl<A: Access> Layer<A> for MetricsLayer {
-    type LayeredAccess = MetricsLayerAccessor<A>;
-
-    fn layer(&self, inner: A) -> Self::LayeredAccess {
-        let meta = inner.info();
-        let scheme = meta.scheme();
-
-        MetricsLayerAccessor {
-            inner,
-            metrics: self.metrics.clone(),
-            scheme,
+            operation_errors_total: register_counter_family("opendal_operation_errors"),
         }
     }
 }
 
-#[derive(Clone)]
-pub struct MetricsLayerAccessor<A: Access> {
-    inner: A,
-    scheme: Scheme,
-    metrics: Arc<MetricsRecorder>,
-}
-
-impl<A: Access> Debug for MetricsLayerAccessor<A> {
-    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
-        f.debug_struct("MetricsAccessor")
-            .field("inner", &self.inner)
-            .finish_non_exhaustive()
-    }
-}
-
-impl<A: Access> LayeredAccess for MetricsLayerAccessor<A> {
-    type Inner = A;
-    type Reader = OperatorMetricsWrapper<A::Reader>;
-    type BlockingReader = OperatorMetricsWrapper<A::BlockingReader>;
-    type Writer = OperatorMetricsWrapper<A::Writer>;
-    type BlockingWriter = OperatorMetricsWrapper<A::BlockingWriter>;
-    type Lister = A::Lister;
-    type BlockingLister = A::BlockingLister;
-
-    fn inner(&self) -> &Self::Inner {
-        &self.inner
-    }
-
-    async fn create_dir(&self, path: &str, args: OpCreateDir) -> opendal::Result<RpCreateDir> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::CreateDir);
-
-        let start_time = Instant::now();
-        let create_res = self.inner.create_dir(path, args).await;
-
-        self.metrics.observe_request_duration(
-            self.scheme,
-            Operation::CreateDir,
-            start_time.elapsed(),
-        );
-        create_res.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::CreateDir, e.kind());
-        })
-    }
-
-    async fn read(&self, path: &str, args: OpRead) -> opendal::Result<(RpRead, Self::Reader)> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::Read);
-
-        let read_res = self
-            .inner
-            .read(path, args)
-            .map(|v| {
-                v.map(|(rp, r)| {
-                    (
-                        rp,
-                        OperatorMetricsWrapper::new(
-                            r,
-                            Operation::Read,
-                            self.metrics.clone(),
-                            self.scheme,
-                        ),
-                    )
-                })
-            })
-            .await;
-        read_res.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::Read, e.kind());
-        })
-    }
-
-    async fn write(&self, path: &str, args: OpWrite) -> opendal::Result<(RpWrite, Self::Writer)> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::Write);
-
-        let write_res = self
-            .inner
-            .write(path, args)
-            .map(|v| {
-                v.map(|(rp, r)| {
-                    (
-                        rp,
-                        OperatorMetricsWrapper::new(
-                            r,
-                            Operation::Write,
-                            self.metrics.clone(),
-                            self.scheme,
-                        ),
-                    )
-                })
-            })
-            .await;
-
-        write_res.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::Write, e.kind());
-        })
-    }
-
-    async fn stat(&self, path: &str, args: OpStat) -> opendal::Result<RpStat> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::Stat);
-        let start_time = Instant::now();
-
-        let stat_res = self
-            .inner
-            .stat(path, args)
-            .inspect_err(|e| {
-                self.metrics
-                    .increment_errors_total(self.scheme, Operation::Stat, e.kind());
-            })
-            .await;
-
-        self.metrics
-            .observe_request_duration(self.scheme, Operation::Stat, start_time.elapsed());
-        stat_res.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::Stat, e.kind());
-        })
-    }
-
-    async fn delete(&self, path: &str, args: OpDelete) -> opendal::Result<RpDelete> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::Delete);
-        let start_time = Instant::now();
-
-        let delete_res = self.inner.delete(path, args).await;
-
-        self.metrics
-            .observe_request_duration(self.scheme, Operation::Delete, start_time.elapsed());
-        delete_res.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::Delete, e.kind());
-        })
-    }
-
-    async fn list(&self, path: &str, args: OpList) -> opendal::Result<(RpList, Self::Lister)> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::List);
-        let start_time = Instant::now();
-
-        let list_res = self.inner.list(path, args).await;
-
-        self.metrics
-            .observe_request_duration(self.scheme, Operation::List, start_time.elapsed());
-        list_res.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::List, e.kind());
-        })
-    }
-
-    async fn batch(&self, args: OpBatch) -> opendal::Result<RpBatch> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::Batch);
-        let start_time = Instant::now();
-
-        let result = self.inner.batch(args).await;
-
-        self.metrics
-            .observe_request_duration(self.scheme, Operation::Batch, start_time.elapsed());
-        result.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::Batch, e.kind());
-        })
-    }
-
-    async fn presign(&self, path: &str, args: OpPresign) -> opendal::Result<RpPresign> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::Presign);
-        let start_time = Instant::now();
-
-        let result = self.inner.presign(path, args).await;
-
-        self.metrics.observe_request_duration(
-            self.scheme,
-            Operation::Presign,
-            start_time.elapsed(),
-        );
-        result.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::Presign, e.kind());
-        })
-    }
-
-    fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> opendal::Result<RpCreateDir> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::BlockingCreateDir);
-        let start_time = Instant::now();
-
-        let result = self.inner.blocking_create_dir(path, args);
-
-        self.metrics.observe_request_duration(
-            self.scheme,
-            Operation::BlockingCreateDir,
-            start_time.elapsed(),
-        );
-        result.inspect_err(|e| {
-            self.metrics.increment_errors_total(
-                self.scheme,
-                Operation::BlockingCreateDir,
-                e.kind(),
-            );
-        })
-    }
-
-    fn blocking_read(
+impl observe::MetricsIntercept for MetricsRecorder {
+    fn observe_operation_duration_seconds(
         &self,
-        path: &str,
-        args: OpRead,
-    ) -> opendal::Result<(RpRead, Self::BlockingReader)> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::BlockingRead);
-
-        let result = self.inner.blocking_read(path, args).map(|(rp, r)| {
-            (
-                rp,
-                OperatorMetricsWrapper::new(
-                    r,
-                    Operation::BlockingRead,
-                    self.metrics.clone(),
-                    self.scheme,
-                ),
-            )
-        });
-
-        result.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::BlockingRead, e.kind());
-        })
+        scheme: Scheme,
+        namespace: Arc<String>,
+        root: Arc<String>,
+        _path: &str,
+        op: Operation,
+        duration: Duration,
+    ) {
+        self.operation_duration_seconds
+            .get_or_create(&OperationLabels {
+                scheme,
+                namespace,
+                root,
+                operation: op,
+                path: None,
+                error: None,
+            })
+            .observe(duration.as_secs_f64())
     }
 
-    fn blocking_write(
+    fn observe_operation_bytes(
         &self,
-        path: &str,
-        args: OpWrite,
-    ) -> opendal::Result<(RpWrite, Self::BlockingWriter)> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::BlockingWrite);
-
-        let result = self.inner.blocking_write(path, args).map(|(rp, r)| {
-            (
-                rp,
-                OperatorMetricsWrapper::new(
-                    r,
-                    Operation::BlockingWrite,
-                    self.metrics.clone(),
-                    self.scheme,
-                ),
-            )
-        });
-
-        result.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::BlockingWrite, e.kind());
-        })
-    }
-
-    fn blocking_stat(&self, path: &str, args: OpStat) -> opendal::Result<RpStat> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::BlockingStat);
-        let start_time = Instant::now();
-
-        let result = self.inner.blocking_stat(path, args);
-        self.metrics.observe_request_duration(
-            self.scheme,
-            Operation::BlockingStat,
-            start_time.elapsed(),
-        );
-
-        result.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::BlockingStat, e.kind());
-        })
-    }
-
-    fn blocking_delete(&self, path: &str, args: OpDelete) -> opendal::Result<RpDelete> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::BlockingDelete);
-        let start_time = Instant::now();
-
-        let result = self.inner.blocking_delete(path, args);
-
-        self.metrics.observe_request_duration(
-            self.scheme,
-            Operation::BlockingDelete,
-            start_time.elapsed(),
-        );
-        result.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::BlockingDelete, e.kind());
-        })
+        scheme: Scheme,
+        namespace: Arc<String>,
+        root: Arc<String>,
+        _path: &str,
+        op: Operation,
+        bytes: usize,
+    ) {
+        self.operation_bytes
+            .get_or_create(&OperationLabels {
+                scheme,
+                namespace,
+                root,
+                operation: op,
+                path: None,
+                error: None,
+            })
+            .observe(bytes as f64)
     }
 
-    fn blocking_list(
+    fn observe_operation_errors_total(
         &self,
-        path: &str,
-        args: OpList,
-    ) -> opendal::Result<(RpList, Self::BlockingLister)> {
-        self.metrics
-            .increment_request_total(self.scheme, Operation::BlockingList);
-        let start_time = Instant::now();
-
-        let result = self.inner.blocking_list(path, args);
-
-        self.metrics.observe_request_duration(
-            self.scheme,
-            Operation::BlockingList,
-            start_time.elapsed(),
-        );
-        result.inspect_err(|e| {
-            self.metrics
-                .increment_errors_total(self.scheme, Operation::BlockingList, e.kind());
-        })
+        scheme: Scheme,
+        namespace: Arc<String>,
+        root: Arc<String>,
+        _path: &str,
+        op: Operation,
+        error: ErrorKind,
+    ) {
+        self.operation_errors_total
+            .get_or_create(&OperationLabels {
+                scheme,
+                namespace,
+                root,
+                operation: op,
+                path: None,
+                error: Some(error.into_static()),
+            })
+            .inc();
     }
 }
 
-pub struct OperatorMetricsWrapper<R> {
-    inner: R,
-
-    op: Operation,
-    metrics: Arc<MetricsRecorder>,
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+struct OperationLabels {
     scheme: Scheme,
+    namespace: Arc<String>,
+    root: Arc<String>,
+    operation: Operation,
+    path: Option<String>,
+    error: Option<&'static str>,
 }
 
-impl<R> OperatorMetricsWrapper<R> {
-    fn new(inner: R, op: Operation, metrics: Arc<MetricsRecorder>, scheme: Scheme) -> Self {
-        Self {
-            inner,
-            op,
-            metrics,
-            scheme,
+impl EncodeLabelSet for OperationLabels {
+    fn encode(&self, mut encoder: LabelSetEncoder) -> Result<(), fmt::Error> {
+        (observe::LABEL_SCHEME, self.scheme.into_static()).encode(encoder.encode_label())?;
+        (observe::LABEL_NAMESPACE, self.namespace.as_str()).encode(encoder.encode_label())?;
+        (observe::LABEL_ROOT, self.root.as_str()).encode(encoder.encode_label())?;
+        (observe::LABEL_OPERATION, self.operation.into_static()).encode(encoder.encode_label())?;
+        if let Some(path) = &self.path {
+            (observe::LABEL_PATH, path.as_str()).encode(encoder.encode_label())?;
         }
-    }
-}
-
-impl<R: oio::Read> oio::Read for OperatorMetricsWrapper<R> {
-    async fn read(&mut self) -> opendal::Result<Buffer> {
-        let start = Instant::now();
-
-        self.inner
-            .read()
-            .await
-            .inspect(|res| {
-                self.metrics
-                    .observe_bytes_total(self.scheme, self.op, res.len());
-                self.metrics
-                    .observe_request_duration(self.scheme, self.op, start.elapsed());
-            })
-            .inspect_err(|err| {
-                self.metrics
-                    .increment_errors_total(self.scheme, self.op, err.kind());
-            })
-    }
-}
-
-impl<R: oio::BlockingRead> oio::BlockingRead for OperatorMetricsWrapper<R> {
-    fn read(&mut self) -> opendal::Result<Buffer> {
-        let start = Instant::now();
-
-        self.inner
-            .read()
-            .inspect(|res| {
-                self.metrics
-                    .observe_bytes_total(self.scheme, self.op, res.len());
-                self.metrics
-                    .observe_request_duration(self.scheme, self.op, start.elapsed());
-            })
-            .inspect_err(|err| {
-                self.metrics
-                    .increment_errors_total(self.scheme, self.op, err.kind());
-            })
-    }
-}
-
-impl<R: oio::Write> oio::Write for OperatorMetricsWrapper<R> {
-    async fn write(&mut self, bs: Buffer) -> opendal::Result<()> {
-        let start = Instant::now();
-        let size = bs.len();
-
-        self.inner
-            .write(bs)
-            .await
-            .inspect(|_| {
-                self.metrics.observe_bytes_total(self.scheme, self.op, size);
-                self.metrics
-                    .observe_request_duration(self.scheme, self.op, start.elapsed());
-            })
-            .inspect_err(|err| {
-                self.metrics
-                    .increment_errors_total(self.scheme, self.op, err.kind());
-            })
-    }
-
-    async fn close(&mut self) -> opendal::Result<()> {
-        self.inner.close().await.inspect_err(|err| {
-            self.metrics
-                .increment_errors_total(self.scheme, self.op, err.kind());
-        })
-    }
-
-    async fn abort(&mut self) -> opendal::Result<()> {
-        self.inner.abort().await.inspect_err(|err| {
-            self.metrics
-                .increment_errors_total(self.scheme, self.op, err.kind());
-        })
-    }
-}
-
-impl<R: oio::BlockingWrite> oio::BlockingWrite for OperatorMetricsWrapper<R> {
-    fn write(&mut self, bs: Buffer) -> opendal::Result<()> {
-        let start = Instant::now();
-        let size = bs.len();
-
-        self.inner
-            .write(bs)
-            .inspect(|_| {
-                self.metrics.observe_bytes_total(self.scheme, self.op, size);
-                self.metrics
-                    .observe_request_duration(self.scheme, self.op, start.elapsed());
-            })
-            .inspect_err(|err| {
-                self.metrics
-                    .increment_errors_total(self.scheme, self.op, err.kind());
-            })
-    }
-
-    fn close(&mut self) -> opendal::Result<()> {
-        self.inner.close().inspect_err(|err| {
-            self.metrics
-                .increment_errors_total(self.scheme, self.op, err.kind());
-        })
+        if let Some(error) = self.error {
+            (observe::LABEL_ERROR, error).encode(encoder.encode_label())?;
+        }
+        Ok(())
     }
 }

From 0e3827fdc14e58a6a944083ef026c0f4c9dc745f Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 23 Oct 2024 14:01:06 +0800
Subject: [PATCH 05/26] Fix build

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 src/common/storage/src/operator.rs      | 29 ++++++++++++++++++++-----
 src/common/storage/src/runtime_layer.rs |  6 -----
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/common/storage/src/operator.rs b/src/common/storage/src/operator.rs
index 76dffbf609c14..6cf73e49d1240 100644
--- a/src/common/storage/src/operator.rs
+++ b/src/common/storage/src/operator.rs
@@ -90,16 +90,35 @@ pub fn init_operator(cfg: &StorageParams) -> Result<Operator> {
     Ok(op)
 }
 
+/// Please take care about the timing of calling opendal's `finish`.
+///
+/// Layers added before `finish` will use static dispatch, and layers added after `finish`
+/// will use dynamic dispatch. Adding too many layers via static dispatch will increase
+/// the compile time of rustc or even results in a compile error.
+///
+/// ```txt
+/// error[E0275]: overflow evaluating the requirement `http::response::Response<()>: std::marker::Send`
+///      |
+///      = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`databend_common_storage`)
+/// note: required because it appears within the type `h2::proto::peer::PollMessage`
+///     --> /home/xuanwo/.cargo/registry/src/index.crates.io-6f17d22bba15001f/h2-0.4.5/src/proto/peer.rs:43:10
+///      |
+/// 43   | pub enum PollMessage {
+///      |          ^^^^^^^^^^^
+/// ```
+///
+/// Please balance the performance and compile time.
 pub fn build_operator<B: Builder>(builder: B) -> Result<Operator> {
-    let ob = Operator::new(builder)?;
-
-    let op = ob
+    let ob = Operator::new(builder)?
         // NOTE
         //
         // Magic happens here. We will add a layer upon original
         // storage operator so that all underlying storage operations
         // will send to storage runtime.
         .layer(RuntimeLayer::new(GlobalIORuntime::instance()))
+        .finish();
+
+    let mut op = ob
         .layer({
             let retry_timeout = env::var("_DATABEND_INTERNAL_RETRY_TIMEOUT")
                 .ok()
@@ -142,11 +161,11 @@ pub fn build_operator<B: Builder>(builder: B) -> Result<Operator> {
 
     if let Ok(permits) = env::var("_DATABEND_INTERNAL_MAX_CONCURRENT_IO_REQUEST") {
         if let Ok(permits) = permits.parse::<usize>() {
-            return Ok(op.layer(ConcurrentLimitLayer::new(permits)).finish());
+            op = op.layer(ConcurrentLimitLayer::new(permits));
         }
     }
 
-    Ok(op.finish())
+    Ok(op)
 }
 
 /// init_azblob_operator will init an opendal azblob operator.
diff --git a/src/common/storage/src/runtime_layer.rs b/src/common/storage/src/runtime_layer.rs
index 69da2b471b2e5..fc662a83504e9 100644
--- a/src/common/storage/src/runtime_layer.rs
+++ b/src/common/storage/src/runtime_layer.rs
@@ -98,7 +98,6 @@ impl<A: Access> LayeredAccess for RuntimeAccessor<A> {
         &self.inner
     }
 
-    #[async_backtrace::framed]
     async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
         let op = self.inner.clone();
         let path = path.to_string();
@@ -108,7 +107,6 @@ impl<A: Access> LayeredAccess for RuntimeAccessor<A> {
             .expect("join must success")
     }
 
-    #[async_backtrace::framed]
     async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
         let op = self.inner.clone();
         let path = path.to_string();
@@ -123,7 +121,6 @@ impl<A: Access> LayeredAccess for RuntimeAccessor<A> {
             })
     }
 
-    #[async_backtrace::framed]
     async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
         let op = self.inner.clone();
         let path = path.to_string();
@@ -133,7 +130,6 @@ impl<A: Access> LayeredAccess for RuntimeAccessor<A> {
             .expect("join must success")
     }
 
-    #[async_backtrace::framed]
     async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
         let op = self.inner.clone();
         let path = path.to_string();
@@ -143,7 +139,6 @@ impl<A: Access> LayeredAccess for RuntimeAccessor<A> {
             .expect("join must success")
     }
 
-    #[async_backtrace::framed]
     async fn delete(&self, path: &str, args: OpDelete) -> Result<RpDelete> {
         let op = self.inner.clone();
         let path = path.to_string();
@@ -153,7 +148,6 @@ impl<A: Access> LayeredAccess for RuntimeAccessor<A> {
             .expect("join must success")
     }
 
-    #[async_backtrace::framed]
     async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
         let op = self.inner.clone();
         let path = path.to_string();

From adf83330d6906ae71d7b1469c7c930ad6a788636 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 23 Oct 2024 14:31:53 +0800
Subject: [PATCH 06/26] Fix header

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 src/common/base/src/http_client.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/base/src/http_client.rs b/src/common/base/src/http_client.rs
index af981449cb007..182c0a46611da 100644
--- a/src/common/base/src/http_client.rs
+++ b/src/common/base/src/http_client.rs
@@ -1,4 +1,4 @@
-// Copyright 2024 Datafuse Labs.
+// Copyright 2021 Datafuse Labs
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

From e6b1931517dbed800bccaff4a0720d9218b0611a Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 23 Oct 2024 15:29:57 +0800
Subject: [PATCH 07/26] Fix build

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock                                    |  5 ---
 Cargo.toml                                    |  1 -
 .../fuse/operations/vacuum_temporary_files.rs | 22 +++++++++----
 .../it/storages/fuse/operations/vacuum.rs     | 33 +++++++++++++++----
 4 files changed, 42 insertions(+), 19 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index f454764bdb640..55304e1a491f7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17260,8 +17260,3 @@ dependencies = [
  "cc",
  "pkg-config",
 ]
-
-[[patch.unused]]
-name = "deltalake"
-version = "0.18.0"
-source = "git+https://github.com/delta-io/delta-rs?rev=57795da#57795da9d9cc86a460a5888713adfb3d0584b4cc"
diff --git a/Cargo.toml b/Cargo.toml
index 583727b254f82..8b658efec840d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -414,7 +414,6 @@ backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "7226
     "serialize-serde",
 ] }
 color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
-deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "57795da" }
 ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" }
 jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "672e423" }
 openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs b/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs
index 0d1a87397cb6f..81ef70142bb1f 100644
--- a/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs
+++ b/src/query/ee/src/storages/fuse/operations/vacuum_temporary_files.rs
@@ -47,7 +47,7 @@ pub async fn do_vacuum_temporary_files(
 
     let operator = DataOperator::instance().operator();
 
-    let temporary_dir = format!("{}/", temporary_dir);
+    let temporary_dir = format!("{}/", temporary_dir.trim_end_matches('/'));
 
     let mut ds = operator
         .lister_with(&temporary_dir)
@@ -66,6 +66,10 @@ pub async fn do_vacuum_temporary_files(
         let mut batch_size = 0;
 
         while let Some(de) = ds.try_next().await? {
+            if de.path() == temporary_dir {
+                continue;
+            }
+
             let meta = de.metadata();
 
             match meta.mode() {
@@ -159,7 +163,7 @@ async fn vacuum_finished_query(
     removed_temp_files: &mut usize,
     total_cleaned_size: &mut usize,
     batch_size: &mut usize,
-    de: &Entry,
+    parent: &Entry,
     limit: usize,
     timestamp: i64,
     life_mills: i64,
@@ -168,7 +172,7 @@ async fn vacuum_finished_query(
 
     let mut all_files_removed = true;
     let mut ds = operator
-        .lister_with(de.path())
+        .lister_with(parent.path())
         .metakey(Metakey::Mode | Metakey::LastModified)
         .await?;
 
@@ -180,6 +184,10 @@ async fn vacuum_finished_query(
         let mut remove_temp_files_path = Vec::with_capacity(1001);
 
         while let Some(de) = ds.try_next().await? {
+            if de.path() == parent.path() {
+                continue;
+            }
+
             let meta = de.metadata();
             if meta.is_file() {
                 if de.name() == "finished" {
@@ -218,7 +226,7 @@ async fn vacuum_finished_query(
             info!(
                 "vacuum removed {} temp files in {:?}(elapsed: {} seconds), batch size: {} bytes",
                 cur_removed,
-                de.path(),
+                parent.path(),
                 instant.elapsed().as_secs(),
                 *batch_size
             );
@@ -238,8 +246,10 @@ async fn vacuum_finished_query(
     }
 
     if all_files_removed {
-        operator.delete(&format!("{}finished", de.path())).await?;
-        operator.delete(de.path()).await?;
+        operator
+            .delete(&format!("{}finished", parent.path()))
+            .await?;
+        operator.delete(parent.path()).await?;
     }
 
     Ok(())
diff --git a/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs b/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs
index 2efa93ca5f41b..5eefbb43a1cca 100644
--- a/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs
+++ b/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs
@@ -112,6 +112,7 @@ async fn test_fuse_do_vacuum_drop_tables() -> Result<()> {
 
     Ok(())
 }
+
 #[tokio::test(flavor = "multi_thread")]
 async fn test_do_vacuum_temporary_files() -> Result<()> {
     let _fixture = TestFixture::setup().await?;
@@ -121,15 +122,14 @@ async fn test_do_vacuum_temporary_files() -> Result<()> {
     operator.write("test_dir/test2", vec![1, 2]).await?;
     operator.write("test_dir/test3", vec![1, 2]).await?;
 
-    assert_eq!(
-        3,
-        operator.list_with("test_dir/").recursive(true).await?.len()
-    );
+    let size = operator.list_with("test_dir/").recursive(true).await?.len();
+    assert!((3..=4).contains(&size));
 
     tokio::time::sleep(Duration::from_secs(2)).await;
     do_vacuum_temporary_files("test_dir/".to_string(), Some(Duration::from_secs(2)), 1).await?;
 
-    assert_eq!(2, operator.list("test_dir/").await?.len());
+    let size = operator.list("test_dir/").await?.len();
+    assert!((2..=3).contains(&size));
 
     operator.write("test_dir/test4/test4", vec![1, 2]).await?;
     operator.write("test_dir/test5/test5", vec![1, 2]).await?;
@@ -138,11 +138,16 @@ async fn test_do_vacuum_temporary_files() -> Result<()> {
         .await?;
 
     do_vacuum_temporary_files("test_dir/".to_string(), Some(Duration::from_secs(2)), 2).await?;
-    assert_eq!(operator.list("test_dir/").await?.len(), 2);
+    let size = operator.list("test_dir/").await?.len();
+    assert!((2..=3).contains(&size));
 
     tokio::time::sleep(Duration::from_secs(3)).await;
     do_vacuum_temporary_files("test_dir/".to_string(), Some(Duration::from_secs(3)), 1000).await?;
-    assert!(operator.list_with("test_dir/").await?.is_empty());
+
+    dbg!(operator.list_with("test_dir/").await?);
+
+    let size = operator.list("test_dir/").await?.len();
+    assert!((0..=1).contains(&size));
 
     Ok(())
 }
@@ -155,8 +160,10 @@ mod test_accessor {
     use opendal::raw::oio;
     use opendal::raw::oio::Entry;
     use opendal::raw::MaybeSend;
+    use opendal::raw::OpBatch;
     use opendal::raw::OpDelete;
     use opendal::raw::OpList;
+    use opendal::raw::RpBatch;
     use opendal::raw::RpDelete;
     use opendal::raw::RpList;
 
@@ -266,6 +273,18 @@ mod test_accessor {
             }
         }
 
+        async fn batch(&self, _args: OpBatch) -> opendal::Result<RpBatch> {
+            self.hit_delete.store(true, Ordering::Release);
+            if self.inject_delete_faulty {
+                Err(opendal::Error::new(
+                    opendal::ErrorKind::Unexpected,
+                    "does not matter (delete)",
+                ))
+            } else {
+                Ok(RpBatch::new(vec![]))
+            }
+        }
+
         async fn list(&self, path: &str, _args: OpList) -> opendal::Result<(RpList, Self::Lister)> {
             if self.inject_delete_faulty {
                 // While injecting faulty for delete operation, return an empty list;

From e78a2bc3438f74aaff05933979dcec737627e043 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 23 Oct 2024 15:37:18 +0800
Subject: [PATCH 08/26] Format toml

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.toml                          | 2 +-
 src/common/base/Cargo.toml          | 4 ++--
 src/common/storage/Cargo.toml       | 1 -
 src/meta/app/Cargo.toml             | 1 -
 src/query/storages/delta/Cargo.toml | 2 +-
 5 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 8b658efec840d..bde5aa202ce5a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -254,7 +254,6 @@ libc = { version = "0.2.158" }
 match-template = "0.0.1"
 mysql_async = { version = "0.34", default-features = false, features = ["native-tls-tls"] }
 object_store_opendal = "0.46"
-opendal_compat = { version = "1" }
 once_cell = "1.15.0"
 openai_api_rust = "0.1"
 opendal = { version = "0.50.1", features = [
@@ -276,6 +275,7 @@ opendal = { version = "0.50.1", features = [
     "services-huggingface",
     "services-redis",
 ] }
+opendal_compat = { version = "1" }
 openraft = { git = "https://github.com/drmingdrmer/openraft", tag = "v0.10.0-alpha.6", features = [
     "serde",
     "tracing-log",
diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml
index 3c8c81e03834e..1bea77d7eaf36 100644
--- a/src/common/base/Cargo.toml
+++ b/src/common/base/Cargo.toml
@@ -50,6 +50,8 @@ prometheus-parse = "0.2.3"
 rand = { workspace = true, features = ["serde1"] }
 regex = { workspace = true }
 replace_with = "0.1.7"
+reqwest = { workspace = true }
+reqwest-hickory-resolver = { workspace = true }
 rustix = "0.38.37"
 semver = { workspace = true }
 serde = { workspace = true }
@@ -60,8 +62,6 @@ tikv-jemalloc-sys = "0.5.2"
 tokio = { workspace = true }
 unicode-segmentation = "1.10.1"
 uuid = { workspace = true }
-reqwest = { workspace = true }
-reqwest-hickory-resolver = { workspace = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { version = "^0.16" }
diff --git a/src/common/storage/Cargo.toml b/src/common/storage/Cargo.toml
index 4ddda845587ae..816392e7f2211 100644
--- a/src/common/storage/Cargo.toml
+++ b/src/common/storage/Cargo.toml
@@ -29,7 +29,6 @@ opendal = { workspace = true }
 parquet = { workspace = true }
 prometheus-client = { workspace = true }
 regex = { workspace = true }
-reqwest = { workspace = true }
 serde = { workspace = true }
 thiserror = { workspace = true }
 
diff --git a/src/meta/app/Cargo.toml b/src/meta/app/Cargo.toml
index 485a8a7d863b6..d88de76ebe0bc 100644
--- a/src/meta/app/Cargo.toml
+++ b/src/meta/app/Cargo.toml
@@ -40,7 +40,6 @@ serde_json = { workspace = true }
 sha1 = "0.10.5"
 sha2 = { workspace = true }
 thiserror = { workspace = true }
-reqwest = { workspace = true }
 
 [dev-dependencies]
 anyhow = { workspace = true }
diff --git a/src/query/storages/delta/Cargo.toml b/src/query/storages/delta/Cargo.toml
index a28ac855b8a23..309cbbbe6b12b 100644
--- a/src/query/storages/delta/Cargo.toml
+++ b/src/query/storages/delta/Cargo.toml
@@ -24,8 +24,8 @@ databend-storages-common-table-meta = { workspace = true }
 deltalake = { workspace = true }
 fastrace = { workspace = true }
 match-template = "0.0.1"
-opendal_compat = { workspace = true, features = ["v0_50_to_v0_49"] }
 object_store_opendal = { workspace = true }
+opendal_compat = { workspace = true, features = ["v0_50_to_v0_49"] }
 parquet = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }

From cc0b9f4ad1d9c4ec1ed837c5fb301f0fef8a9a55 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 23 Oct 2024 22:01:43 +0800
Subject: [PATCH 09/26] Update cargo lock

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock | 179 ++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 121 insertions(+), 58 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 8329c8299f94b..88874815eb58a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -609,7 +609,7 @@ version = "0.2.6"
 source = "git+https://github.com/datafuse-extras/async-backtrace.git?rev=dea4553#dea4553c47ff2946684f7efb295a4105d5627fac"
 dependencies = [
  "async-backtrace-attributes",
- "dashmap",
+ "dashmap 5.5.3",
  "futures",
  "itertools 0.10.5",
  "loom",
@@ -1481,6 +1481,18 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
 
+[[package]]
+name = "bb8"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8"
+dependencies = [
+ "async-trait",
+ "futures-util",
+ "parking_lot 0.12.3",
+ "tokio",
+]
+
 [[package]]
 name = "beef"
 version = "0.5.2"
@@ -3083,6 +3095,20 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "dashmap"
+version = "6.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+ "hashbrown 0.14.5",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core 0.9.10",
+]
+
 [[package]]
 name = "data-encoding"
 version = "2.6.0"
@@ -3108,7 +3134,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "limits-rs",
  "log",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "serde",
  "serde_json",
  "serfig",
@@ -3179,7 +3205,7 @@ dependencies = [
  "lz4",
  "num",
  "num-traits",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "ordered-float 4.2.2",
  "proptest",
  "quanta 0.11.1",
@@ -3272,6 +3298,8 @@ dependencies = [
  "rand 0.8.5",
  "regex",
  "replace_with",
+ "reqwest",
+ "reqwest-hickory-resolver",
  "rustix 0.38.37",
  "semver",
  "serde",
@@ -3313,7 +3341,7 @@ dependencies = [
  "async-backtrace",
  "async-trait",
  "chrono",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-arrow",
  "databend-common-ast",
  "databend-common-base",
@@ -3427,7 +3455,7 @@ dependencies = [
  "geos",
  "geozero 0.13.0",
  "http 1.1.0",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "parquet",
  "paste",
  "prost 0.12.6",
@@ -3457,7 +3485,7 @@ dependencies = [
  "chrono",
  "chrono-tz 0.8.6",
  "comfy-table 6.2.0",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-arrow",
  "databend-common-ast",
  "databend-common-base",
@@ -3764,7 +3792,7 @@ dependencies = [
  "maplit",
  "num-derive",
  "num-traits",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "paste",
  "prost 0.12.6",
  "serde",
@@ -4017,7 +4045,7 @@ dependencies = [
  "flate2",
  "futures",
  "lz4",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "parquet-format-safe",
  "rand 0.8.5",
  "seq-macro",
@@ -4152,7 +4180,7 @@ version = "0.1.0"
 dependencies = [
  "async-backtrace",
  "chrono-tz 0.8.6",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-ast",
  "databend-common-base",
  "databend-common-config",
@@ -4185,7 +4213,7 @@ dependencies = [
  "cidr",
  "cron",
  "ctor 0.2.8",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-ast",
  "databend-common-base",
  "databend-common-catalog",
@@ -4219,7 +4247,7 @@ dependencies = [
  "log",
  "num-derive",
  "num-traits",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "parking_lot 0.12.3",
  "percent-encoding",
  "prqlc",
@@ -4243,7 +4271,7 @@ dependencies = [
  "arrow-schema",
  "async-backtrace",
  "chrono",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-arrow",
  "databend-common-auth",
  "databend-common-base",
@@ -4254,12 +4282,10 @@ dependencies = [
  "flagset",
  "futures",
  "log",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "parquet",
  "prometheus-client",
  "regex",
- "reqwest",
- "reqwest-hickory-resolver",
  "serde",
  "thiserror",
 ]
@@ -4286,6 +4312,7 @@ dependencies = [
  "fastrace",
  "match-template",
  "object_store_opendal",
+ "opendal_compat",
  "parquet",
  "serde",
  "serde_json",
@@ -4299,7 +4326,7 @@ name = "databend-common-storages-factory"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-catalog",
  "databend-common-config",
  "databend-common-exception",
@@ -4366,7 +4393,7 @@ dependencies = [
  "itertools 0.10.5",
  "jsonb",
  "log",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4413,7 +4440,7 @@ dependencies = [
  "futures",
  "hive_metastore",
  "log",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "parquet",
  "recursive",
  "serde",
@@ -4512,7 +4539,7 @@ dependencies = [
  "async-trait",
  "bytes",
  "chrono",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-base",
  "databend-common-catalog",
  "databend-common-exception",
@@ -4527,7 +4554,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "futures-util",
  "log",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "orc-rust",
  "serde",
  "serde_json",
@@ -4564,7 +4591,7 @@ dependencies = [
  "ethnum",
  "futures",
  "log",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4611,7 +4638,7 @@ dependencies = [
  "databend-common-storages-parquet",
  "databend-storages-common-blocks",
  "databend-storages-common-table-meta",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "parquet",
  "serde",
  "serde_json",
@@ -4652,7 +4679,7 @@ dependencies = [
  "enum-as-inner 0.6.0",
  "futures",
  "log",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "parquet",
  "serde",
  "serde_json",
@@ -4716,7 +4743,7 @@ dependencies = [
  "jsonb",
  "log",
  "once_cell",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "parking_lot 0.12.3",
  "regex",
  "serde",
@@ -4927,7 +4954,7 @@ dependencies = [
  "aws-sdk-s3",
  "chrono",
  "chrono-tz 0.8.6",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-ast",
  "databend-common-base",
  "databend-common-building",
@@ -4970,7 +4997,7 @@ dependencies = [
  "jsonb",
  "jwt-simple 0.11.9",
  "log",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "tantivy",
  "tempfile",
 ]
@@ -5152,7 +5179,7 @@ dependencies = [
  "config",
  "criterion",
  "ctor 0.2.8",
- "dashmap",
+ "dashmap 5.5.3",
  "databend-common-arrow",
  "databend-common-ast",
  "databend-common-base",
@@ -5244,7 +5271,7 @@ dependencies = [
  "naive-cityhash",
  "num",
  "num_cpus",
- "opendal 0.49.2",
+ "opendal 0.50.1",
  "opensrv-mysql",
  "opentelemetry",
  "opentelemetry_sdk",
@@ -5447,7 +5474,7 @@ dependencies = [
  "fastrace",
  "futures",
  "log",
- "opendal 0.49.2",
+ "opendal 0.50.1",
 ]
 
 [[package]]
@@ -5591,12 +5618,13 @@ checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
 
 [[package]]
 name = "delta_kernel"
-version = "0.1.1"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1ddfe35af3696786ab5f23cd995df33a66f6cff272ac1f85e09c1a6316acd4c"
+checksum = "fa08a82239f51e6d3d249c38f0f5bf7c8a78b28587e1b466893c9eac84d252d8"
 dependencies = [
  "arrow-arith",
  "arrow-array",
+ "arrow-cast",
  "arrow-json",
  "arrow-ord",
  "arrow-schema",
@@ -5614,6 +5642,7 @@ dependencies = [
  "rustc_version",
  "serde",
  "serde_json",
+ "strum 0.26.3",
  "thiserror",
  "tracing",
  "url",
@@ -5624,9 +5653,9 @@ dependencies = [
 
 [[package]]
 name = "delta_kernel_derive"
-version = "0.1.1"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4d2127a34b12919a6bce08225f0ca6fde8a19342a32675370edfc8795e7c38a"
+checksum = "ec5c4fb5b59b1bd55ed8ebcf941f27a327d600c19a4a4103546846c358be93ff"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5635,16 +5664,18 @@ dependencies = [
 
 [[package]]
 name = "deltalake"
-version = "0.18.0"
-source = "git+https://github.com/delta-io/delta-rs?rev=57795da#57795da9d9cc86a460a5888713adfb3d0584b4cc"
+version = "0.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cdaf5eed6cf6be7d94ce89ee9d7325324fc7c6c0b1ca8b911b0a5d95f6b1af5"
 dependencies = [
  "deltalake-core",
 ]
 
 [[package]]
 name = "deltalake-core"
-version = "0.18.0"
-source = "git+https://github.com/delta-io/delta-rs?rev=57795da#57795da9d9cc86a460a5888713adfb3d0584b4cc"
+version = "0.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ebab21c8c8820f9accb3ee74cc6ab7d930adf44323f39b7a764fd34e34aa7f4"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -5661,7 +5692,7 @@ dependencies = [
  "bytes",
  "cfg-if",
  "chrono",
- "dashmap",
+ "dashmap 6.1.0",
  "delta_kernel",
  "either",
  "errno",
@@ -5687,10 +5718,12 @@ dependencies = [
  "roaring",
  "serde",
  "serde_json",
+ "sqlparser 0.51.0",
  "thiserror",
  "tokio",
  "tracing",
  "url",
+ "urlencoding",
  "uuid",
  "z85",
 ]
@@ -7835,7 +7868,7 @@ version = "14.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "006acf5a613e0b5cf095d8e4b3f48c12a60d9062aa2b2dd105afaf8344a5600c"
 dependencies = [
- "dashmap",
+ "dashmap 5.5.3",
  "gix-fs",
  "libc",
  "once_cell",
@@ -8696,7 +8729,7 @@ dependencies = [
  "murmur3",
  "num-bigint",
  "once_cell",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "ordered-float 4.2.2",
  "parquet",
  "paste",
@@ -10758,67 +10791,77 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b04d09b9822c2f75a1d2fc513a2c1279c70e91e7407936fffdf6a6976ec530a"
 dependencies = [
  "anyhow",
- "async-backtrace",
  "async-trait",
  "backon 0.4.4",
  "base64 0.22.1",
  "bytes",
  "chrono",
- "crc32c",
- "fastrace",
  "flagset",
  "futures",
  "getrandom 0.2.15",
- "hdrs",
  "http 1.1.0",
  "log",
  "md-5",
- "moka",
  "once_cell",
  "percent-encoding",
- "prometheus-client",
- "prost 0.13.1",
  "quick-xml 0.36.1",
- "redis",
- "reqsign",
  "reqwest",
  "serde",
  "serde_json",
- "sha2",
  "tokio",
  "uuid",
 ]
 
 [[package]]
 name = "opendal"
-version = "0.50.0"
+version = "0.50.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36e44fc43be9ffe18dad3e3ef9d61c1ae01991ee6f1c8c026978c35777a711bf"
+checksum = "213222b6c86949314d8f51acb26d8241e7c8dd0879b016a79471d49f21ee592f"
 dependencies = [
  "anyhow",
+ "async-backtrace",
  "async-trait",
  "backon 1.2.0",
  "base64 0.22.1",
+ "bb8",
  "bytes",
  "chrono",
  "crc32c",
+ "fastrace",
  "flagset",
  "futures",
  "getrandom 0.2.15",
+ "hdrs",
  "http 1.1.0",
  "log",
  "md-5",
+ "moka",
  "once_cell",
  "percent-encoding",
+ "prometheus-client",
+ "prost 0.13.1",
  "quick-xml 0.36.1",
+ "redis",
  "reqsign",
  "reqwest",
  "serde",
  "serde_json",
+ "sha2",
  "tokio",
  "uuid",
 ]
 
+[[package]]
+name = "opendal_compat"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "091a08a86c52bd69d7b4be44202d03edf6d0358256805561be2287b5c308fcdf"
+dependencies = [
+ "async-trait",
+ "opendal 0.49.2",
+ "opendal 0.50.1",
+]
+
 [[package]]
 name = "openraft"
 version = "0.10.0"
@@ -12611,9 +12654,9 @@ dependencies = [
 
 [[package]]
 name = "redis"
-version = "0.26.1"
+version = "0.27.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e902a69d09078829137b4a5d9d082e0490393537badd7c91a3d69d14639e115f"
+checksum = "81cccf17a692ce51b86564334614d72dcae1def0fd5ecebc9f02956da74352b5"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -12636,7 +12679,7 @@ dependencies = [
  "sha1_smol",
  "socket2 0.5.7",
  "tokio",
- "tokio-retry",
+ "tokio-retry2",
  "tokio-rustls 0.26.0",
  "tokio-util",
  "url",
@@ -14174,6 +14217,15 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "sqlparser"
+version = "0.51.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7"
+dependencies = [
+ "log",
+]
+
 [[package]]
 name = "sqlx"
 version = "0.8.2"
@@ -15123,9 +15175,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.39.2"
+version = "1.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"
+checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb"
 dependencies = [
  "backtrace",
  "bytes",
@@ -15194,6 +15246,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-retry2"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "903934dba1c4c2f2e9cb460ef10b5695e0b0ecad3bf9ee7c8675e540c5e8b2d1"
+dependencies = [
+ "pin-project",
+ "rand 0.8.5",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.24.1"
@@ -15944,7 +16007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71bc0fbf23ebdfc8718d40ba5f31a520376767edd6ebb9fe06c27eb977379796"
 dependencies = [
  "async-broadcast",
- "dashmap",
+ "dashmap 5.5.3",
  "faststr",
  "futures",
  "lazy_static",

From 74ac594ac1f30f5aa12a22fc0101daaa00284052 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Wed, 23 Oct 2024 22:03:32 +0800
Subject: [PATCH 10/26] Fix services for opendal

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock                          | 3 +++
 src/query/storages/delta/Cargo.toml | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index 88874815eb58a..09be1433ba182 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4312,6 +4312,7 @@ dependencies = [
  "fastrace",
  "match-template",
  "object_store_opendal",
+ "opendal 0.49.2",
  "opendal_compat",
  "parquet",
  "serde",
@@ -10796,6 +10797,7 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "chrono",
+ "crc32c",
  "flagset",
  "futures",
  "getrandom 0.2.15",
@@ -10805,6 +10807,7 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "quick-xml 0.36.1",
+ "reqsign",
  "reqwest",
  "serde",
  "serde_json",
diff --git a/src/query/storages/delta/Cargo.toml b/src/query/storages/delta/Cargo.toml
index 309cbbbe6b12b..ece145c07bcfe 100644
--- a/src/query/storages/delta/Cargo.toml
+++ b/src/query/storages/delta/Cargo.toml
@@ -25,6 +25,8 @@ deltalake = { workspace = true }
 fastrace = { workspace = true }
 match-template = "0.0.1"
 object_store_opendal = { workspace = true }
+# enable features to make delta happy.
+opendal = { version = "0.49.2", features = ["services-s3", "services-fs"] }
 opendal_compat = { workspace = true, features = ["v0_50_to_v0_49"] }
 parquet = { workspace = true }
 serde = { workspace = true }
@@ -37,4 +39,4 @@ url = "2.4.1"
 workspace = true
 
 [package.metadata.cargo-machete]
-ignored = ["match-template"]
+ignored = ["match-template", "opendal"]

From 04787414b8331c8ef0b61b0e513c6e9aae3aa958 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 11:59:41 +0800
Subject: [PATCH 11/26] Fix opendal 0.49

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock                          | 39 ++++++++++++++++++++++++++++-
 src/query/storages/delta/Cargo.toml | 17 ++++++++++++-
 2 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 09be1433ba182..ef61a545acfea 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -10804,13 +10804,17 @@ dependencies = [
  "http 1.1.0",
  "log",
  "md-5",
+ "moka",
  "once_cell",
  "percent-encoding",
+ "prost 0.13.1",
  "quick-xml 0.36.1",
+ "redis 0.26.1",
  "reqsign",
  "reqwest",
  "serde",
  "serde_json",
+ "sha2",
  "tokio",
  "uuid",
 ]
@@ -10844,7 +10848,7 @@ dependencies = [
  "prometheus-client",
  "prost 0.13.1",
  "quick-xml 0.36.1",
- "redis",
+ "redis 0.27.5",
  "reqsign",
  "reqwest",
  "serde",
@@ -12655,6 +12659,39 @@ dependencies = [
  "syn 2.0.58",
 ]
 
+[[package]]
+name = "redis"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e902a69d09078829137b4a5d9d082e0490393537badd7c91a3d69d14639e115f"
+dependencies = [
+ "arc-swap",
+ "async-trait",
+ "bytes",
+ "combine",
+ "crc16",
+ "futures",
+ "futures-util",
+ "itoa",
+ "log",
+ "num-bigint",
+ "percent-encoding",
+ "pin-project-lite",
+ "rand 0.8.5",
+ "rustls 0.23.12",
+ "rustls-native-certs 0.7.1",
+ "rustls-pemfile 2.1.3",
+ "rustls-pki-types",
+ "ryu",
+ "sha1_smol",
+ "socket2 0.5.7",
+ "tokio",
+ "tokio-retry",
+ "tokio-rustls 0.26.0",
+ "tokio-util",
+ "url",
+]
+
 [[package]]
 name = "redis"
 version = "0.27.5"
diff --git a/src/query/storages/delta/Cargo.toml b/src/query/storages/delta/Cargo.toml
index ece145c07bcfe..4a3f5f4d4b5a9 100644
--- a/src/query/storages/delta/Cargo.toml
+++ b/src/query/storages/delta/Cargo.toml
@@ -26,7 +26,22 @@ fastrace = { workspace = true }
 match-template = "0.0.1"
 object_store_opendal = { workspace = true }
 # enable features to make delta happy.
-opendal = { version = "0.49.2", features = ["services-s3", "services-fs"] }
+opendal = { version = "0.49.2", features = [
+    "services-s3",
+    "services-fs",
+    "services-gcs",
+    "services-cos",
+    "services-obs",
+    "services-oss",
+    "services-azblob",
+    "services-azdls",
+    "services-ipfs",
+    "services-http",
+    "services-moka",
+    "services-webhdfs",
+    "services-huggingface",
+    "services-redis",
+] }
 opendal_compat = { workspace = true, features = ["v0_50_to_v0_49"] }
 parquet = { workspace = true }
 serde = { workspace = true }

From 0d0069712893bbd4f5a7d4b32b4a60b4fe73d278 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 14:04:22 +0800
Subject: [PATCH 12/26] Fix opendal compat

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock | 68 +++++++++++++++++++++++++++++++++++++-----------------
 Cargo.toml |  1 +
 2 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 00cd582ae378c..8203be0e61ad1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3134,7 +3134,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "limits-rs",
  "log",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde",
  "serde_json",
  "serfig",
@@ -3205,7 +3205,7 @@ dependencies = [
  "lz4",
  "num",
  "num-traits",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "ordered-float 4.2.2",
  "proptest",
  "quanta 0.11.1",
@@ -3455,7 +3455,7 @@ dependencies = [
  "geos",
  "geozero 0.13.0",
  "http 1.1.0",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parquet",
  "paste",
  "prost 0.12.6",
@@ -3792,7 +3792,7 @@ dependencies = [
  "maplit",
  "num-derive",
  "num-traits",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "paste",
  "prost 0.12.6",
  "serde",
@@ -4045,7 +4045,7 @@ dependencies = [
  "flate2",
  "futures",
  "lz4",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parquet-format-safe",
  "rand 0.8.5",
  "seq-macro",
@@ -4247,7 +4247,7 @@ dependencies = [
  "log",
  "num-derive",
  "num-traits",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.12.3",
  "percent-encoding",
  "prqlc",
@@ -4282,7 +4282,7 @@ dependencies = [
  "flagset",
  "futures",
  "log",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parquet",
  "prometheus-client",
  "regex",
@@ -4394,7 +4394,7 @@ dependencies = [
  "itertools 0.10.5",
  "jsonb",
  "log",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4441,7 +4441,7 @@ dependencies = [
  "futures",
  "hive_metastore",
  "log",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parquet",
  "recursive",
  "serde",
@@ -4555,7 +4555,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "futures-util",
  "log",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "orc-rust",
  "serde",
  "serde_json",
@@ -4592,7 +4592,7 @@ dependencies = [
  "ethnum",
  "futures",
  "log",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4639,7 +4639,7 @@ dependencies = [
  "databend-common-storages-parquet",
  "databend-storages-common-blocks",
  "databend-storages-common-table-meta",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parquet",
  "serde",
  "serde_json",
@@ -4680,7 +4680,7 @@ dependencies = [
  "enum-as-inner 0.6.0",
  "futures",
  "log",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parquet",
  "serde",
  "serde_json",
@@ -4744,7 +4744,7 @@ dependencies = [
  "jsonb",
  "log",
  "once_cell",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.12.3",
  "regex",
  "serde",
@@ -5003,7 +5003,7 @@ dependencies = [
  "jsonb",
  "jwt-simple 0.11.9",
  "log",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "tantivy",
  "tempfile",
 ]
@@ -5278,7 +5278,7 @@ dependencies = [
  "naive-cityhash",
  "num",
  "num_cpus",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "opensrv-mysql",
  "opentelemetry",
  "opentelemetry_sdk",
@@ -5481,7 +5481,7 @@ dependencies = [
  "fastrace",
  "futures",
  "log",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -8736,7 +8736,7 @@ dependencies = [
  "murmur3",
  "num-bigint",
  "once_cell",
- "opendal 0.50.1",
+ "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "ordered-float 4.2.2",
  "parquet",
  "paste",
@@ -10864,15 +10864,41 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "opendal"
+version = "0.50.1"
+source = "git+https://github.com/apache/opendal?rev=0ef4fea#0ef4feab9355000bbf8791ab1b8da6e41053a5ac"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "backon 1.2.0",
+ "base64 0.22.1",
+ "bytes",
+ "chrono",
+ "flagset",
+ "futures",
+ "getrandom 0.2.15",
+ "http 1.1.0",
+ "log",
+ "md-5",
+ "once_cell",
+ "percent-encoding",
+ "quick-xml 0.36.1",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "tokio",
+ "uuid",
+]
+
 [[package]]
 name = "opendal_compat"
 version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "091a08a86c52bd69d7b4be44202d03edf6d0358256805561be2287b5c308fcdf"
+source = "git+https://github.com/apache/opendal?rev=0ef4fea#0ef4feab9355000bbf8791ab1b8da6e41053a5ac"
 dependencies = [
  "async-trait",
  "opendal 0.49.2",
- "opendal 0.50.1",
+ "opendal 0.50.1 (git+https://github.com/apache/opendal?rev=0ef4fea)",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 817dea929b153..11fd1bacf906d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -417,6 +417,7 @@ color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
 ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" }
 jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "672e423" }
 openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
+opendal_compat = { git = "https://github.com/apache/opendal", rev = "0ef4fea" }
 orc-rust = { git = "https://github.com/datafuse-extras/datafusion-orc", rev = "03372b97" }
 recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }

From 54c4eedd052479a1cc34f2b3e995c987f1dab011 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 14:12:25 +0800
Subject: [PATCH 13/26] Fix version mismatch

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock | 68 ++++++++++++++++--------------------------------------
 Cargo.toml |  1 +
 2 files changed, 21 insertions(+), 48 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 8203be0e61ad1..6ac8abf2fba26 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3134,7 +3134,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "limits-rs",
  "log",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "serde",
  "serde_json",
  "serfig",
@@ -3205,7 +3205,7 @@ dependencies = [
  "lz4",
  "num",
  "num-traits",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "ordered-float 4.2.2",
  "proptest",
  "quanta 0.11.1",
@@ -3455,7 +3455,7 @@ dependencies = [
  "geos",
  "geozero 0.13.0",
  "http 1.1.0",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "parquet",
  "paste",
  "prost 0.12.6",
@@ -3792,7 +3792,7 @@ dependencies = [
  "maplit",
  "num-derive",
  "num-traits",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "paste",
  "prost 0.12.6",
  "serde",
@@ -4045,7 +4045,7 @@ dependencies = [
  "flate2",
  "futures",
  "lz4",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "parquet-format-safe",
  "rand 0.8.5",
  "seq-macro",
@@ -4247,7 +4247,7 @@ dependencies = [
  "log",
  "num-derive",
  "num-traits",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "parking_lot 0.12.3",
  "percent-encoding",
  "prqlc",
@@ -4282,7 +4282,7 @@ dependencies = [
  "flagset",
  "futures",
  "log",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "parquet",
  "prometheus-client",
  "regex",
@@ -4394,7 +4394,7 @@ dependencies = [
  "itertools 0.10.5",
  "jsonb",
  "log",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4441,7 +4441,7 @@ dependencies = [
  "futures",
  "hive_metastore",
  "log",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "parquet",
  "recursive",
  "serde",
@@ -4555,7 +4555,7 @@ dependencies = [
  "databend-storages-common-table-meta",
  "futures-util",
  "log",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "orc-rust",
  "serde",
  "serde_json",
@@ -4592,7 +4592,7 @@ dependencies = [
  "ethnum",
  "futures",
  "log",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "parquet",
  "rand 0.8.5",
  "serde",
@@ -4639,7 +4639,7 @@ dependencies = [
  "databend-common-storages-parquet",
  "databend-storages-common-blocks",
  "databend-storages-common-table-meta",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "parquet",
  "serde",
  "serde_json",
@@ -4680,7 +4680,7 @@ dependencies = [
  "enum-as-inner 0.6.0",
  "futures",
  "log",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "parquet",
  "serde",
  "serde_json",
@@ -4744,7 +4744,7 @@ dependencies = [
  "jsonb",
  "log",
  "once_cell",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "parking_lot 0.12.3",
  "regex",
  "serde",
@@ -5003,7 +5003,7 @@ dependencies = [
  "jsonb",
  "jwt-simple 0.11.9",
  "log",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "tantivy",
  "tempfile",
 ]
@@ -5278,7 +5278,7 @@ dependencies = [
  "naive-cityhash",
  "num",
  "num_cpus",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "opensrv-mysql",
  "opentelemetry",
  "opentelemetry_sdk",
@@ -5481,7 +5481,7 @@ dependencies = [
  "fastrace",
  "futures",
  "log",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
 ]
 
 [[package]]
@@ -8736,7 +8736,7 @@ dependencies = [
  "murmur3",
  "num-bigint",
  "once_cell",
- "opendal 0.50.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "opendal 0.50.1",
  "ordered-float 4.2.2",
  "parquet",
  "paste",
@@ -10828,8 +10828,7 @@ dependencies = [
 [[package]]
 name = "opendal"
 version = "0.50.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "213222b6c86949314d8f51acb26d8241e7c8dd0879b016a79471d49f21ee592f"
+source = "git+https://github.com/apache/opendal?rev=0ef4fea#0ef4feab9355000bbf8791ab1b8da6e41053a5ac"
 dependencies = [
  "anyhow",
  "async-backtrace",
@@ -10864,33 +10863,6 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "opendal"
-version = "0.50.1"
-source = "git+https://github.com/apache/opendal?rev=0ef4fea#0ef4feab9355000bbf8791ab1b8da6e41053a5ac"
-dependencies = [
- "anyhow",
- "async-trait",
- "backon 1.2.0",
- "base64 0.22.1",
- "bytes",
- "chrono",
- "flagset",
- "futures",
- "getrandom 0.2.15",
- "http 1.1.0",
- "log",
- "md-5",
- "once_cell",
- "percent-encoding",
- "quick-xml 0.36.1",
- "reqwest",
- "serde",
- "serde_json",
- "tokio",
- "uuid",
-]
-
 [[package]]
 name = "opendal_compat"
 version = "1.0.0"
@@ -10898,7 +10870,7 @@ source = "git+https://github.com/apache/opendal?rev=0ef4fea#0ef4feab9355000bbf87
 dependencies = [
  "async-trait",
  "opendal 0.49.2",
- "opendal 0.50.1 (git+https://github.com/apache/opendal?rev=0ef4fea)",
+ "opendal 0.50.1",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 11fd1bacf906d..ecf1152515e1d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -417,6 +417,7 @@ color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
 ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" }
 jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "672e423" }
 openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
+opendal = { git = "https://github.com/apache/opendal", rev = "0ef4fea" }
 opendal_compat = { git = "https://github.com/apache/opendal", rev = "0ef4fea" }
 orc-rust = { git = "https://github.com/datafuse-extras/datafusion-orc", rev = "03372b97" }
 recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" }

From 9d708b329dab78413de4c220ac4af1f1ddfcd8dc Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 14:20:35 +0800
Subject: [PATCH 14/26] Fix patch

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock | 3 ++-
 Cargo.toml | 5 ++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 6ac8abf2fba26..159f6e77f1576 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -10828,7 +10828,8 @@ dependencies = [
 [[package]]
 name = "opendal"
 version = "0.50.1"
-source = "git+https://github.com/apache/opendal?rev=0ef4fea#0ef4feab9355000bbf8791ab1b8da6e41053a5ac"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213222b6c86949314d8f51acb26d8241e7c8dd0879b016a79471d49f21ee592f"
 dependencies = [
  "anyhow",
  "async-backtrace",
diff --git a/Cargo.toml b/Cargo.toml
index ecf1152515e1d..1942194365529 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -417,7 +417,6 @@ color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
 ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" }
 jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "672e423" }
 openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
-opendal = { git = "https://github.com/apache/opendal", rev = "0ef4fea" }
 opendal_compat = { git = "https://github.com/apache/opendal", rev = "0ef4fea" }
 orc-rust = { git = "https://github.com/datafuse-extras/datafusion-orc", rev = "03372b97" }
 recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" }
@@ -426,3 +425,7 @@ tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370"
 tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370", package = "tantivy-common" }
 tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "0e300e9" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }
+
+# This patch makes sure opendal_compat also used the same version of opendal instead of git one.
+[patch.'https://github.com/apache/opendal']
+opendal = { version = "0.50.1" }

From bdc0e73156a5a4407049ad4842f64edeabd38b1e Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 15:15:47 +0800
Subject: [PATCH 15/26] Add ping support in redis mock

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 .../src/mock_source/redis_source.rs           | 37 ++++++++++++-------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/tests/sqllogictests/src/mock_source/redis_source.rs b/tests/sqllogictests/src/mock_source/redis_source.rs
index 905de7cbf6037..e502b55a29090 100644
--- a/tests/sqllogictests/src/mock_source/redis_source.rs
+++ b/tests/sqllogictests/src/mock_source/redis_source.rs
@@ -46,19 +46,27 @@ async fn process(stream: TcpStream) {
                 let request = String::from_utf8(buf.clone()).unwrap();
                 let cmds = parse_resp(request);
                 for cmd in cmds {
-                    if let Command::Get(key) = cmd {
-                        // Return a value if the first character of the key is ASCII alphanumeric,
-                        // otherwise treat it as the key does not exist.
-                        let ret_value = if key.starts_with(|c: char| c.is_ascii_alphanumeric()) {
-                            let v = format!("{}_value", key);
-                            format!("${}\r\n{}\r\n", v.len(), v)
-                        } else {
-                            "$-1\r\n".to_string()
-                        };
-                        ret_values.push_back(ret_value);
-                    } else {
-                        let ret_value = "+OK\r\n".to_string();
-                        ret_values.push_back(ret_value);
+                    match cmd {
+                        Command::Get(key) => {
+                            // Return a value if the first character of the key is ASCII alphanumeric,
+                            // otherwise treat it as the key does not exist.
+                            let ret_value = if key.starts_with(|c: char| c.is_ascii_alphanumeric())
+                            {
+                                let v = format!("{}_value", key);
+                                format!("${}\r\n{}\r\n", v.len(), v)
+                            } else {
+                                "$-1\r\n".to_string()
+                            };
+                            ret_values.push_back(ret_value);
+                        }
+                        Command::Ping => {
+                            let ret_value = "+PONG\r\n".to_string();
+                            ret_values.push_back(ret_value);
+                        }
+                        _ => {
+                            let ret_value = "+OK\r\n".to_string();
+                            ret_values.push_back(ret_value);
+                        }
                     }
                 }
             }
@@ -91,6 +99,7 @@ async fn process(stream: TcpStream) {
 // Redis command, only support get, other commands are ignored.
 enum Command {
     Get(String),
+    Ping,
     Invalid,
     Other,
 }
@@ -115,6 +124,8 @@ fn parse_resp(request: String) -> Vec<Command> {
         if lines[2] == "GET" {
             let cmd = Command::Get(lines[4].to_string());
             cmds.push(cmd);
+        } else if lines[2] == "PING" {
+            cmds.push(Command::Ping)
         } else {
             cmds.push(Command::Other);
         }

From 86a5b07c6355fc500cb4d7541be036cef56f88ad Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 16:27:21 +0800
Subject: [PATCH 16/26] Fix hang on auto detect

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 src/meta/app/src/storage/storage_params.rs | 37 ++++++++++++++--------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/meta/app/src/storage/storage_params.rs b/src/meta/app/src/storage/storage_params.rs
index fa44c04a088c1..558b4f25e3bdf 100644
--- a/src/meta/app/src/storage/storage_params.rs
+++ b/src/meta/app/src/storage/storage_params.rs
@@ -18,6 +18,8 @@ use std::fmt::Formatter;
 use std::time::Duration;
 
 use databend_common_base::http_client::GLOBAL_HTTP_CLIENT;
+use databend_common_base::runtime::GlobalIORuntime;
+use databend_common_base::runtime::TrySpawn;
 use databend_common_exception::ErrorCode;
 use databend_common_exception::Result;
 use serde::Deserialize;
@@ -135,21 +137,28 @@ impl StorageParams {
                     format!("https://{}", endpoint)
                 };
 
-                // The response itself doesn't important.
-                let _ = GLOBAL_HTTP_CLIENT
-                    .inner()
-                    .get(&endpoint)
-                    .timeout(Duration::from_secs(10))
-                    .send()
+                let endpoint_clone = endpoint.clone();
+                // The first call to http client must be inside global io runtime.
+                GlobalIORuntime::instance()
+                    .spawn(async move {
+                        GLOBAL_HTTP_CLIENT
+                            .inner()
+                            .get(&endpoint_clone)
+                            .timeout(Duration::from_secs(10))
+                            .send()
+                            .await
+                            // The response itself doesn't important, just drop it.
+                            .map(|_| ())
+                            .map_err(|err| {
+                                ErrorCode::InvalidConfig(format!(
+                                    "s3 endpoint_url {endpoint_clone} is invalid or incomplete: {err:?}",
+                                ))
+                            })
+                    })
                     .await
-                    .map_err(|err| {
-                        ErrorCode::InvalidConfig(format!(
-                            "s3 endpoint_url {} is invalid or incomplete: {err:?}",
-                            s3.endpoint_url
-                        ))
-                    })?;
-
-                s3.region = opendal::services::S3::detect_region(&s3.endpoint_url, &s3.bucket)
+                    .unwrap()?;
+
+                s3.region = opendal::services::S3::detect_region(&endpoint, &s3.bucket)
                     .await
                     .unwrap_or_default();
                 StorageParams::S3(s3)

From 5de9bb0c625a55b9aa87944f64fe3ae5aca93198 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 16:34:02 +0800
Subject: [PATCH 17/26] Ignore other errors

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 src/meta/app/src/storage/storage_params.rs | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/meta/app/src/storage/storage_params.rs b/src/meta/app/src/storage/storage_params.rs
index 558b4f25e3bdf..4c3ebc15d21f6 100644
--- a/src/meta/app/src/storage/storage_params.rs
+++ b/src/meta/app/src/storage/storage_params.rs
@@ -141,7 +141,7 @@ impl StorageParams {
                 // The first call to http client must be inside global io runtime.
                 GlobalIORuntime::instance()
                     .spawn(async move {
-                        GLOBAL_HTTP_CLIENT
+                        if let Err(err) = GLOBAL_HTTP_CLIENT
                             .inner()
                             .get(&endpoint_clone)
                             .timeout(Duration::from_secs(10))
@@ -149,11 +149,14 @@ impl StorageParams {
                             .await
                             // The response itself doesn't important, just drop it.
                             .map(|_| ())
-                            .map_err(|err| {
-                                ErrorCode::InvalidConfig(format!(
+                            {
+                            if err.is_builder() {
+                                return Err(ErrorCode::InvalidConfig(format!(
                                     "s3 endpoint_url {endpoint_clone} is invalid or incomplete: {err:?}",
-                                ))
-                            })
+                                )))
+                            }
+                        }
+                        Ok(())
                     })
                     .await
                     .unwrap()?;

From d9aa859a77895780c883760ab2dfc4922faf8c9f Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 17:03:10 +0800
Subject: [PATCH 18/26] Remove unexpected check

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 src/meta/app/src/storage/storage_params.rs | 29 ----------------------
 1 file changed, 29 deletions(-)

diff --git a/src/meta/app/src/storage/storage_params.rs b/src/meta/app/src/storage/storage_params.rs
index 4c3ebc15d21f6..73e7e8bd2209d 100644
--- a/src/meta/app/src/storage/storage_params.rs
+++ b/src/meta/app/src/storage/storage_params.rs
@@ -15,12 +15,7 @@
 use std::fmt::Debug;
 use std::fmt::Display;
 use std::fmt::Formatter;
-use std::time::Duration;
 
-use databend_common_base::http_client::GLOBAL_HTTP_CLIENT;
-use databend_common_base::runtime::GlobalIORuntime;
-use databend_common_base::runtime::TrySpawn;
-use databend_common_exception::ErrorCode;
 use databend_common_exception::Result;
 use serde::Deserialize;
 use serde::Serialize;
@@ -137,30 +132,6 @@ impl StorageParams {
                     format!("https://{}", endpoint)
                 };
 
-                let endpoint_clone = endpoint.clone();
-                // The first call to http client must be inside global io runtime.
-                GlobalIORuntime::instance()
-                    .spawn(async move {
-                        if let Err(err) = GLOBAL_HTTP_CLIENT
-                            .inner()
-                            .get(&endpoint_clone)
-                            .timeout(Duration::from_secs(10))
-                            .send()
-                            .await
-                            // The response itself doesn't important, just drop it.
-                            .map(|_| ())
-                            {
-                            if err.is_builder() {
-                                return Err(ErrorCode::InvalidConfig(format!(
-                                    "s3 endpoint_url {endpoint_clone} is invalid or incomplete: {err:?}",
-                                )))
-                            }
-                        }
-                        Ok(())
-                    })
-                    .await
-                    .unwrap()?;
-
                 s3.region = opendal::services::S3::detect_region(&endpoint, &s3.bucket)
                     .await
                     .unwrap_or_default();

From 2f86c73c61c3d2c0817e4e54afa31693ebc379c0 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 22:31:24 +0800
Subject: [PATCH 19/26] Fix compat

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock | 2 +-
 Cargo.toml | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 159f6e77f1576..2628ed224ad25 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -10867,7 +10867,7 @@ dependencies = [
 [[package]]
 name = "opendal_compat"
 version = "1.0.0"
-source = "git+https://github.com/apache/opendal?rev=0ef4fea#0ef4feab9355000bbf8791ab1b8da6e41053a5ac"
+source = "git+https://github.com/apache/opendal?rev=4e6fddd#4e6fddd10f35560e7d5cefc29d0c099a36eaac4e"
 dependencies = [
  "async-trait",
  "opendal 0.49.2",
diff --git a/Cargo.toml b/Cargo.toml
index 1942194365529..00322b1033894 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -417,7 +417,7 @@ color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
 ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" }
 jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "672e423" }
 openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
-opendal_compat = { git = "https://github.com/apache/opendal", rev = "0ef4fea" }
+opendal_compat = { git = "https://github.com/apache/opendal", rev = "4e6fddd" }
 orc-rust = { git = "https://github.com/datafuse-extras/datafusion-orc", rev = "03372b97" }
 recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }
@@ -426,6 +426,7 @@ tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "75
 tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "0e300e9" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }
 
+
 # This patch makes sure opendal_compat also used the same version of opendal instead of git one.
 [patch.'https://github.com/apache/opendal']
 opendal = { version = "0.50.1" }

From eb7e53953043510435bf1d85062c6398a9d07806 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 22:52:22 +0800
Subject: [PATCH 20/26] Format toml

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 00322b1033894..f8b7fe0ba64f7 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -426,7 +426,6 @@ tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "75
 tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "0e300e9" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }
 
-
 # This patch makes sure opendal_compat also used the same version of opendal instead of git one.
 [patch.'https://github.com/apache/opendal']
 opendal = { version = "0.50.1" }

From a6c4a432460d7a394338472ff168dd03bd49f046 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 23:26:29 +0800
Subject: [PATCH 21/26] Use main branch of opendal_compat

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock | 2 +-
 Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2628ed224ad25..cf187e0ab4ddd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -10867,7 +10867,7 @@ dependencies = [
 [[package]]
 name = "opendal_compat"
 version = "1.0.0"
-source = "git+https://github.com/apache/opendal?rev=4e6fddd#4e6fddd10f35560e7d5cefc29d0c099a36eaac4e"
+source = "git+https://github.com/apache/opendal?rev=f6e60f6#f6e60f621dec82765a49283caf0ddc1ff8b1d260"
 dependencies = [
  "async-trait",
  "opendal 0.49.2",
diff --git a/Cargo.toml b/Cargo.toml
index f8b7fe0ba64f7..c85f60d3ea209 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -417,7 +417,7 @@ color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
 ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" }
 jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "672e423" }
 openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
-opendal_compat = { git = "https://github.com/apache/opendal", rev = "4e6fddd" }
+opendal_compat = { git = "https://github.com/apache/opendal", rev = "f6e60f6" }
 orc-rust = { git = "https://github.com/datafuse-extras/datafusion-orc", rev = "03372b97" }
 recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }

From ec7c101a40830fd09d79915a96320e92879abbcf Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Thu, 24 Oct 2024 23:56:11 +0800
Subject: [PATCH 22/26] ignore the dir itself

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 src/query/storages/hive/hive/src/hive_table.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/query/storages/hive/hive/src/hive_table.rs b/src/query/storages/hive/hive/src/hive_table.rs
index 12fe4e01274bf..2ca9237129c80 100644
--- a/src/query/storages/hive/hive/src/hive_table.rs
+++ b/src/query/storages/hive/hive/src/hive_table.rs
@@ -622,6 +622,9 @@ async fn do_list_files_from_dir(
                 all_files.push(HivePartInfo::create(location, vec![], length));
             }
             EntryMode::DIR => {
+                if path == location {
+                    continue;
+                }
                 all_dirs.push(path.to_string());
             }
             _ => {

From fbbb523a79687a6468c7629cb6054a5d6abc6fb6 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Fri, 25 Oct 2024 01:44:28 +0800
Subject: [PATCH 23/26] Fix path handling

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 src/query/storages/hive/hive/src/hive_table.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/query/storages/hive/hive/src/hive_table.rs b/src/query/storages/hive/hive/src/hive_table.rs
index 2ca9237129c80..b2cebe6058f37 100644
--- a/src/query/storages/hive/hive/src/hive_table.rs
+++ b/src/query/storages/hive/hive/src/hive_table.rs
@@ -614,6 +614,10 @@ async fn do_list_files_from_dir(
         if path[file_offset..].starts_with('.') || path[file_offset..].starts_with('_') {
             continue;
         }
+        // Ignore the location itself
+        if path.trim_end_matches('/') == location.trim_end_matches('/') {
+            continue;
+        }
 
         match meta.mode() {
             EntryMode::FILE => {
@@ -622,9 +626,6 @@ async fn do_list_files_from_dir(
                 all_files.push(HivePartInfo::create(location, vec![], length));
             }
             EntryMode::DIR => {
-                if path == location {
-                    continue;
-                }
                 all_dirs.push(path.to_string());
             }
             _ => {

From dc7eea836a9de9bd90f7398eb7d7b9b54e988ff0 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Fri, 25 Oct 2024 17:55:51 +0800
Subject: [PATCH 24/26] Try make sure location always end with /

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 src/query/storages/hive/hive/src/converters.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/storages/hive/hive/src/converters.rs b/src/query/storages/hive/hive/src/converters.rs
index 0410a1ced0864..9e38aabee6f6a 100644
--- a/src/query/storages/hive/hive/src/converters.rs
+++ b/src/query/storages/hive/hive/src/converters.rs
@@ -78,7 +78,7 @@ pub fn try_into_table_info(
         storage
             .location
             .as_ref()
-            .map(|location| location.to_string())
+            .map(|location| format!("{}/", location.trim_end_matches('/')))
     } else {
         None
     };

From 3c54434d4e47330224ba005b735afc593cbaf6e5 Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Fri, 25 Oct 2024 18:23:45 +0800
Subject: [PATCH 25/26] Revert "Try make sure location always end with /"

This reverts commit dc7eea836a9de9bd90f7398eb7d7b9b54e988ff0.
---
 src/query/storages/hive/hive/src/converters.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/storages/hive/hive/src/converters.rs b/src/query/storages/hive/hive/src/converters.rs
index 9e38aabee6f6a..0410a1ced0864 100644
--- a/src/query/storages/hive/hive/src/converters.rs
+++ b/src/query/storages/hive/hive/src/converters.rs
@@ -78,7 +78,7 @@ pub fn try_into_table_info(
         storage
             .location
             .as_ref()
-            .map(|location| format!("{}/", location.trim_end_matches('/')))
+            .map(|location| location.to_string())
     } else {
         None
     };

From 830170431d7946dddf4ae4b42b8e94f7f089b0ed Mon Sep 17 00:00:00 2001
From: Xuanwo <github@xuanwo.io>
Date: Fri, 25 Oct 2024 23:31:24 +0800
Subject: [PATCH 26/26] Fix hive may return same files

Signed-off-by: Xuanwo <github@xuanwo.io>
---
 Cargo.lock                                     | 11 +++++++++++
 src/common/tracing/Cargo.toml                  |  1 +
 src/common/tracing/src/init.rs                 |  6 +++++-
 src/query/storages/hive/hive/src/hive_table.rs |  2 +-
 4 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e89bcdbdbffa5..80aeebcb0389b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6228,6 +6228,16 @@ dependencies = [
  "syn 2.0.58",
 ]
 
+[[package]]
+name = "env_filter"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab"
+dependencies = [
+ "log",
+ "regex",
+]
+
 [[package]]
 name = "env_logger"
 version = "0.10.2"
@@ -9753,6 +9763,7 @@ dependencies = [
  "anyhow",
  "colored",
  "crossbeam-channel",
+ "env_filter",
  "fastrace",
  "jiff",
  "log",
diff --git a/src/common/tracing/Cargo.toml b/src/common/tracing/Cargo.toml
index 822411af6cac1..bb3b6c0509030 100644
--- a/src/common/tracing/Cargo.toml
+++ b/src/common/tracing/Cargo.toml
@@ -28,6 +28,7 @@ logforth = { version = "0.12", features = [
     'rolling_file',
     'opentelemetry',
     'fastrace',
+    "env-filter",
 ] }
 opentelemetry = { workspace = true }
 opentelemetry-otlp = { workspace = true }
diff --git a/src/common/tracing/src/init.rs b/src/common/tracing/src/init.rs
index 93b5793dfdea1..1b66f0c519398 100644
--- a/src/common/tracing/src/init.rs
+++ b/src/common/tracing/src/init.rs
@@ -22,7 +22,9 @@ use databend_common_base::runtime::Thread;
 use fastrace::prelude::*;
 use log::LevelFilter;
 use log::Metadata;
+use logforth::filter::env::EnvFilterBuilder;
 use logforth::filter::CustomFilter;
+use logforth::filter::EnvFilter;
 use logforth::filter::FilterResult;
 use logforth::filter::TargetFilter;
 use logforth::Dispatch;
@@ -213,7 +215,9 @@ pub fn init_logging(
                 "databend::log::structlog",
                 LevelFilter::Off,
             ))
-            .filter(cfg.stderr.level.parse().unwrap_or(LevelFilter::Info))
+            .filter(EnvFilter::new(
+                EnvFilterBuilder::new().parse(&cfg.stderr.level),
+            ))
             .layout(get_layout(&cfg.stderr.format))
             .append(logforth::append::Stderr);
         logger = logger.dispatch(dispatch);
diff --git a/src/query/storages/hive/hive/src/hive_table.rs b/src/query/storages/hive/hive/src/hive_table.rs
index b2cebe6058f37..fa728e47c2264 100644
--- a/src/query/storages/hive/hive/src/hive_table.rs
+++ b/src/query/storages/hive/hive/src/hive_table.rs
@@ -615,7 +615,7 @@ async fn do_list_files_from_dir(
             continue;
         }
         // Ignore the location itself
-        if path.trim_end_matches('/') == location.trim_end_matches('/') {
+        if path.trim_matches('/') == location.trim_matches('/') {
             continue;
         }