feat(object store): support azblob workload identity

Cargo.toml

@@ -304,6 +304,8 @@ tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev =
 futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev = "05b33b4" }
 # patch: unlimit 4MB message size for grpc client
 etcd-client = { git = "https://github.com/risingwavelabs/etcd-client.git", rev = "4e84d40" }
+# todo(wcy-fdu): remove this patch fork after opendal release a new version to apply azure workload identity change.
+reqsign = { git = "https://github.com/wcy-fdu/reqsign.git", rev = "e6cb304" }
 
 [workspace.metadata.dylint]
 libraries = [{ path = "./lints" }]

risedev.yml

@@ -70,7 +70,7 @@ profile:
       - use: frontend
       - use: compactor
 
-  # `dev-compute-node` have the same settings as default except the the compute node will be started by user.
+  # `dev-compute-node` have the same settings as default except the compute node will be started by user.
   dev-compute-node:
     steps:
       - use: meta-node
@@ -343,6 +343,18 @@ profile:
       - use: compute-node
       - use: frontend
 
+  meta-1cn-1fe-sqlite:
+    steps:
+      - use: minio
+      - use: sqlite
+      - use: meta-node
+        port: 5690
+        dashboard-port: 5691
+        exporter-port: 1250
+      - use: compactor
+      - use: compute-node
+      - use: frontend
+
   java-binding-demo:
     steps:
       - use: minio
@@ -1013,6 +1025,13 @@ template:
     # Other etcd nodes
     provide-etcd: "etcd*"
 
+  sqlite:
+    # Id of this instance
+    id: sqlite
+
+    # File name of the sqlite database
+    file: metadata.db
+
   compute-node:
     # Compute-node advertise address
     address: "127.0.0.1"
@@ -1087,6 +1106,9 @@ template:
     # Etcd backend config
     provide-etcd-backend: "etcd*"
 
+    # Sqlite backend config
+    provide-sqlite-backend: "sqlite*"
+
     # Prometheus nodes used by dashboard service
     provide-prometheus: "prometheus*"
 
@@ -1348,4 +1370,4 @@ template:
     port: 6379
 
     # address of redis
-    address: "127.0.0.1"
+    address: "127.0.0.1"

src/object_store/Cargo.toml

@@ -37,6 +37,10 @@ thiserror-ext = { workspace = true }
 tokio = { version = "0.2", package = "madsim-tokio", features = ["fs"] }
 tokio-retry = "0.3"
 tracing = "0.1"
+
+[patch.crates-io]
+reqsign = { git = "https://github.com/wcy-fdu/reqsign.git", rev = "f6ff677" }
+
 # This crate is excluded from hakari (see hakari.toml) after hdfs is introduced...## [target.'cfg(not(madsim))'.dependencies]
 # workspace-hack = { path = "../workspace-hack" }
 #

src/object_store/src/object/opendal_engine/azblob.rs

@@ -18,6 +18,8 @@ use opendal::Operator;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
+
+const AZBLOB_ENDPOINT: &str = "AZBLOB_ENDPOINT";
 impl OpendalObjectStore {
     /// create opendal azblob engine.
     pub fn new_azblob_engine(container_name: String, root: String) -> ObjectResult<Self> {
@@ -26,16 +28,10 @@ impl OpendalObjectStore {
         builder.root(&root);
         builder.container(&container_name);
 
-        let endpoint = std::env::var("AZBLOB_ENDPOINT")
+        let endpoint = std::env::var(AZBLOB_ENDPOINT)
             .unwrap_or_else(|_| panic!("AZBLOB_ENDPOINT not found from environment variables"));
-        let account_name = std::env::var("AZBLOB_ACCOUNT_NAME")
-            .unwrap_or_else(|_| panic!("AZBLOB_ACCOUNT_NAME not found from environment variables"));
-        let account_key = std::env::var("AZBLOB_ACCOUNT_KEY")
-            .unwrap_or_else(|_| panic!("AZBLOB_ACCOUNT_KEY not found from environment variables"));
 
         builder.endpoint(&endpoint);
-        builder.account_name(&account_name);
-        builder.account_key(&account_key);
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
             .layer(RetryLayer::default())