refactor

e2e_test/webhook/check_1.slt.part

@@ -1,19 +1,24 @@
 query TT
-select data ->> 'source', data->> 'auth_algo' from github_sha256;
+select data ->> 'source', data->> 'auth_algo' from github_hmac_sha256;
 ----
-github sha256
+github hmac_sha256
 
 query TT
-select data ->> 'source', data->> 'auth_algo' from github_sha1;
+select data ->> 'source', data->> 'auth_algo' from github_hmac_sha1;
 ----
-github sha1
+github hmac_sha1
 
 query TT
 select data ->> 'source', data->> 'auth_algo' from rudderstack;
 ----
 rudderstack plain
 
 query TT
-select data ->> 'source', data->> 'auth_algo' from segment_encode_hmac;
+select data ->> 'source', data->> 'auth_algo' from segment_hmac_sha1;
 ----
-segment encode_hmac
+segment hmac_sha1
+
+query TT
+select data ->> 'source', data->> 'auth_algo' from hubspot_sha256_v2;
+----
+hubspot sha256_v2

e2e_test/webhook/check_2.slt.part

@@ -1,14 +1,14 @@
 query TT
-select data ->> 'source', data->> 'auth_algo' from github_sha256;
+select data ->> 'source', data->> 'auth_algo' from github_hmac_sha256;
 ----
-github sha256
-github sha256
+github hmac_sha256
+github hmac_sha256
 
 query TT
-select data ->> 'source', data->> 'auth_algo' from github_sha1;
+select data ->> 'source', data->> 'auth_algo' from github_hmac_sha1;
 ----
-github sha1
-github sha1
+github hmac_sha1
+github hmac_sha1
 
 query TT
 select data ->> 'source', data->> 'auth_algo' from rudderstack;
@@ -17,7 +17,13 @@ rudderstack plain
 rudderstack plain
 
 query TT
-select data ->> 'source', data->> 'auth_algo' from segment_encode_hmac;
+select data ->> 'source', data->> 'auth_algo' from segment_hmac_sha1;
 ----
-segment encode_hmac
-segment encode_hmac
+segment hmac_sha1
+segment hmac_sha1
+
+query TT
+select data ->> 'source', data->> 'auth_algo' from hubspot_sha256_v2;
+----
+hubspot sha256_v2
+hubspot sha256_v2

e2e_test/webhook/check_3.slt.part

@@ -1,16 +1,16 @@
 query TT
-select data ->> 'source', data->> 'auth_algo' from github_sha256;
+select data ->> 'source', data->> 'auth_algo' from github_hmac_sha256;
 ----
-github sha256
-github sha256
-github sha256
+github hmac_sha256
+github hmac_sha256
+github hmac_sha256
 
 query TT
-select data ->> 'source', data->> 'auth_algo' from github_sha1;
+select data ->> 'source', data->> 'auth_algo' from github_hmac_sha1;
 ----
-github sha1
-github sha1
-github sha1
+github hmac_sha1
+github hmac_sha1
+github hmac_sha1
 
 query TT
 select data ->> 'source', data->> 'auth_algo' from rudderstack;
@@ -20,8 +20,15 @@ rudderstack plain
 rudderstack plain
 
 query TT
-select data ->> 'source', data->> 'auth_algo' from segment_encode_hmac;
+select data ->> 'source', data->> 'auth_algo' from segment_hmac_sha1;
 ----
-segment encode_hmac
-segment encode_hmac
-segment encode_hmac
+segment hmac_sha1
+segment hmac_sha1
+segment hmac_sha1
+
+query TT
+select data ->> 'source', data->> 'auth_algo' from hubspot_sha256_v2;
+----
+hubspot sha256_v2
+hubspot sha256_v2
+hubspot sha256_v2

e2e_test/webhook/create_table.slt.part

@@ -9,31 +9,47 @@ create table rudderstack (
 );
 
 statement ok
-create table github_sha1 (
+create table github_hmac_sha1 (
   data JSONB
 ) WITH (
   connector = 'webhook',
 ) VALIDATE SECRET test_secret AS secure_compare(
   headers->>'x-hub-signature',
-  hmac(test_secret, data, 'sha1', 'sha1=')
+  'sha1=' || encode(hmac(test_secret, data, 'sha1'), 'hex')
 );
 
 statement ok
-create table github_sha256 (
+create table github_hmac_sha256 (
   data JSONB
 ) WITH (
   connector = 'webhook',
 ) VALIDATE SECRET test_secret AS secure_compare(
   headers->>'x-hub-signature-256',
-  hmac(test_secret, data, 'sha256', 'sha256=')
+  'sha256=' || encode(hmac(test_secret, data, 'sha256'), 'hex')
 );
 
 statement ok
-create table segment_sha1 (
+create table segment_hmac_sha1 (
   data JSONB
 ) WITH (
   connector = 'webhook',
 ) VALIDATE SECRET test_secret AS secure_compare(
   headers->>'x-signature',
-  hmac(test_secret, data, 'sha1')
+  encode(hmac(test_secret, data, 'sha1'), 'hex')
+);
+
+# https://developers.hubspot.com/beta-docs/guides/apps/authentication/validating-requests#validate-requests-using-the-v2-request-signature
+statement ok
+create table hubspot_sha256_v2 (
+  data JSONB
+) WITH (
+  connector = 'webhook',
+) VALIDATE SECRET test_secret AS secure_compare(
+  headers->>'x-hubspot-signature',
+  encode(
+    sha256(
+      convert_to(
+        (test_secret || 'POST' || 'http://127.0.0.1:4560/message/dev/public/' || convert_from(data, 'utf8'))
+      , 'UTF8')
+    ), 'hex')
 );

e2e_test/webhook/drop_table.slt.part

@@ -1,12 +1,14 @@
+statement ok
+DROP TABLE hubspot_sha256_v2;
 
 statement ok
-DROP TABLE segment_encode_hmac;
+DROP TABLE segment_hmac_sha1;
 
 statement ok
-DROP TABLE github_sha256;
+DROP TABLE github_hmac_sha256;
 
 statement ok
-DROP TABLE github_sha1;
+DROP TABLE github_hmac_sha1;
 
 statement ok
 DROP TABLE rudderstack;

e2e_test/webhook/sender.py

@@ -18,7 +18,7 @@
     "timestamp": 1639581841
 }
 
-SERVER_URL = "http://127.0.0.1:8080/message/root/dev/public/"
+SERVER_URL = "http://127.0.0.1:4560/message/dev/public/"
 
 
 def generate_signature_hmac(secret, payload, auth_algo, prefix):
@@ -46,11 +46,11 @@ def send_webhook(url, headers, payload_json):
         sys.exit(1)  # Exit the program with an error
 
 
-def send_github_sha1(secret):
+def send_github_hmac_sha1(secret):
     payload = message
     payload['source'] = "github"
-    payload['auth_algo'] = "sha1"
-    url = SERVER_URL + "github_sha1"
+    payload['auth_algo'] = "hmac_sha1"
+    url = SERVER_URL + "github_hmac_sha1"
 
     payload_json = json.dumps(payload)
     signature = generate_signature_hmac(secret, payload_json, 'sha1', "sha1=")
@@ -62,11 +62,11 @@ def send_github_sha1(secret):
     send_webhook(url, headers, payload_json)
 
 
-def send_github_sha256(secret):
+def send_github_hmac_sha256(secret):
     payload = message
     payload['source'] = "github"
-    payload['auth_algo'] = "sha256"
-    url = SERVER_URL + "github_sha256"
+    payload['auth_algo'] = "hmac_sha256"
+    url = SERVER_URL + "github_hmac_sha256"
 
     payload_json = json.dumps(payload)
     signature = generate_signature_hmac(secret, payload_json, 'sha256', "sha256=")
@@ -79,11 +79,11 @@ def send_github_sha256(secret):
 
 
 def send_rudderstack(secret):
-    # apply to both rudderstack and AWS EventBridge
+    # apply to both rudderstack, AWS EventBridge and HubSpot with API Key.
     payload = message
     payload['source'] = "rudderstack"
     payload['auth_algo'] = "plain"
-    url = SERVER_URL + "rudderstack"
+    url = SERVER_URL + "rudderstack" 
 
     payload_json = json.dumps(payload)
     signature = secret
@@ -95,12 +95,11 @@ def send_rudderstack(secret):
     send_webhook(url, headers, payload_json)
 
 
-def send_segment_encode_hmac(secret):
-    # apply to both rudderstack and AWS EventBridge
+def send_segment_hmac_sha1(secret):
     payload = message
     payload['source'] = "segment"
-    payload['auth_algo'] = "encode_hmac"
-    url = SERVER_URL + "segment_encode_hmac"
+    payload['auth_algo'] = "hmac_sha1"
+    url = SERVER_URL + "segment_hmac_sha1"
 
     payload_json = json.dumps(payload)
     signature = generate_signature_hmac(secret, payload_json, 'sha1', '')
@@ -112,12 +111,35 @@ def send_segment_encode_hmac(secret):
     send_webhook(url, headers, payload_json)
 
 
+def send_hubspot_sha256_v2(secret):
+    payload = message
+    payload['source'] = "hubspot"
+    payload['auth_algo'] = "sha256_v2"
+    url = SERVER_URL + "hubspot_sha256_v2"
+
+    payload_json = json.dumps(payload)
+    payload = secret + 'POST' + SERVER_URL + str(payload_json)
+    signature = hashlib.sha256(payload.encode('utf-8')).hexdigest()
+    # Webhook message headers
+    headers = {
+        "Content-Type": "application/json",
+        "x-hubspot-signature": signature  # Custom signature header
+    }
+    send_webhook(url, headers, payload_json) 
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Simulate sending Webhook messages")
     parser.add_argument("--secret", required=True, help="Secret key for generating signature")
     args = parser.parse_args()
     secret = args.secret
     # send data
-    send_github_sha1(secret)
-    send_github_sha256(secret)
+    # github
+    send_github_hmac_sha1(secret)
+    send_github_hmac_sha256(secret)
+    # rudderstack, AWS EventBridge and HubSpot with API Key.
     send_rudderstack(secret)
+    # segment
+    send_segment_hmac_sha1(secret)
+    # hubspot
+    send_hubspot_sha256_v2(secret)

src/common/secret/src/secret_manager.rs

@@ -18,6 +18,7 @@ use std::io::Write;
 use std::path::PathBuf;
 
 use anyhow::{anyhow, Context};
+use parking_lot::lock_api::RwLockReadGuard;
 use parking_lot::RwLock;
 use prost::Message;
 use risingwave_pb::catalog::PbSecret;
@@ -118,32 +119,23 @@ impl LocalSecretManager {
     ) -> SecretResult<BTreeMap<String, String>> {
         let secret_guard = self.secrets.read();
         for (option_key, secret_ref) in secret_refs {
-            let secret_id = secret_ref.secret_id;
-            let pb_secret_bytes = secret_guard
-                .get(&secret_id)
-                .ok_or(SecretError::ItemNotFound(secret_id))?;
-            let secret_value_bytes = Self::get_secret_value(pb_secret_bytes)?;
-            match secret_ref.ref_as() {
-                RefAsType::Text => {
-                    // We converted the secret string from sql to bytes using `as_bytes` in frontend.
-                    // So use `from_utf8` here to convert it back to string.
-                    options.insert(option_key, String::from_utf8(secret_value_bytes.clone())?);
-                }
-                RefAsType::File => {
-                    let path_str =
-                        self.get_or_init_secret_file(secret_id, secret_value_bytes.clone())?;
-                    options.insert(option_key, path_str);
-                }
-                RefAsType::Unspecified => {
-                    return Err(SecretError::UnspecifiedRefType(secret_id));
-                }
-            }
+            let path_str = self.fill_secret_inner(secret_ref, &secret_guard)?;
+            options.insert(option_key, path_str);
         }
         Ok(options)
     }
 
     pub fn fill_secret(&self, secret_ref: PbSecretRef) -> SecretResult<String> {
-        let secret_guard = self.secrets.read();
+        let secret_guard: RwLockReadGuard<'_, parking_lot::RawRwLock, HashMap<u32, Vec<u8>>> =
+            self.secrets.read();
+        self.fill_secret_inner(secret_ref, &secret_guard)
+    }
+
+    fn fill_secret_inner(
+        &self,
+        secret_ref: PbSecretRef,
+        secret_guard: &RwLockReadGuard<'_, parking_lot::RawRwLock, HashMap<u32, Vec<u8>>>,
+    ) -> SecretResult<String> {
         let secret_id = secret_ref.secret_id;
         let pb_secret_bytes = secret_guard
             .get(&secret_id)