Metabase 0.47 support, reworked base types mapping (#181)

* Replace the usage of u/snake-key with u/->snake_case_en (#160)

* replace the usage of u/snake-key with u/->snake_case_en

* Update check.yml

---------

Co-authored-by: Serge Klochkov <[email protected]>

* Metabase 0.47 support, reworked base types mapping

* Back to RC1 due to a failing test (not related to the driver)

* Use the latest 0.47.x branch commit

---------

Co-authored-by: Naveen Yellapu <[email protected]>

.github/workflows/check.yml

@@ -17,7 +17,8 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: metabase/metabase
-          ref: v0.46.5
+          # The latest commit on release-0.47.x branch - fixed failing MB tests
+          ref: dba05c0b6f6cbc6e2f8247c03cc5fcc986cd3fe5
 
       - name: Checkout Driver Repo
         uses: actions/checkout@v2
@@ -51,7 +52,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v2
         with:
-          node-version: "16"
+          node-version: "18"
           cache: "yarn"
 
       - name: Get M2 cache

CHANGELOG.md

@@ -1,3 +1,12 @@
+# 1.2.0
+
+### New features
+* Metabase 0.47 support
+* Connection impersonation support (0.47 feature)
+
+### Bug fixes
+* `DateTime64` is now correctly mapped to `:type/DateTime`
+
 # 1.1.7
 
 ### New features

docker-compose.yml

@@ -32,7 +32,7 @@ services:
     hostname: server.clickhouseconnect.test
 
   metabase:
-    image: metabase/metabase:v0.46.5
+    image: metabase/metabase:v0.47.0-RC2
     container_name: metabase-with-clickhouse-driver
     environment:
       'MB_HTTP_TIMEOUT': '5000'

resources/metabase-plugin.yaml

@@ -1,6 +1,6 @@
 info:
   name: Metabase ClickHouse Driver
-  version: 1.1.7
+  version: 1.2.0
   description: Allows Metabase to connect to ClickHouse databases.
 contact-info:
   name: ClickHouse

src/metabase/driver/clickhouse.clj

@@ -7,8 +7,8 @@
             [java-time :as t]
             [metabase [config :as config] [driver :as driver] [util :as u]]
             [metabase.driver.clickhouse-nippy]
-            [metabase.driver.common :as driver.common]
             [metabase.driver.ddl.interface :as ddl.i]
+            [metabase.driver.sql :as driver.sql]
             [metabase.driver.sql-jdbc [common :as sql-jdbc.common]
              [connection :as sql-jdbc.conn] [execute :as sql-jdbc.execute]
              [sync :as sql-jdbc.sync]]
@@ -36,9 +36,11 @@
   (sql-jdbc.sync/pattern-based-database-type->base-type
    [[#"Array" :type/Array]
     [#"Bool" :type/Boolean]
+    ;; TODO: test it with :type/DateTimeWithTZ
     [#"DateTime64" :type/DateTime]
     [#"DateTime" :type/DateTime]
     [#"Date" :type/Date]
+    [#"Date32" :type/Date]
     [#"Decimal" :type/Decimal]
     [#"Enum8" :type/Text]
     [#"Enum16" :type/Text]
@@ -60,25 +62,28 @@
     [#"UInt64" :type/BigInteger]
     [#"UUID" :type/UUID]]))
 
+;; Enum8(UInt8) -> Enum8, DateTime64(Europe/Amsterdam) -> DateTime64,
+;; Nullable(DateTime) -> DateTime, SimpleAggregateFunction(sum, Int64) -> Int64, etc
+(defn- ^:private normalize-database-type
+  [database-type]
+  (let [db-type    (subs (str database-type) 1) ;; keyword->str; `name` call does not work well
+        normalized (second (re-find #"(?:Nullable\(|LowCardinality\()?(\w+)?\({0,1}.*" db-type))]
+    ;; slightly different normalization for SimpleAggregateFunction - we need to take the second arg
+    (or (keyword (if (= normalized "SimpleAggregateFunction")
+                   (second (re-find #"SimpleAggregateFunction\(\w+?, {0,1}(.+)?\)" db-type))
+                   normalized))
+        database-type))) ;; basically, fall back to :type/* later
+
 (defmethod sql-jdbc.sync/database-type->base-type :clickhouse
   [_ database-type]
-  (let [base-type (database-type->base-type
-                   (let [normalized ;; extract the type from Nullable or LowCardinality first
-                         (str/replace (name database-type)
-                                      #"(?:Nullable|LowCardinality)\((\S+)\)"
-                                      "$1")]
-                     (cond
-                       (str/starts-with? normalized "Array(") "Array"
-                       (str/starts-with? normalized "Map(") "Map"
-                       :else normalized)))]
-    base-type))
+  (database-type->base-type (normalize-database-type database-type)))
 
 (def ^:private excluded-schemas #{"system" "information_schema" "INFORMATION_SCHEMA"})
 (defmethod sql-jdbc.sync/excluded-schemas :clickhouse [_] excluded-schemas)
 
 (def ^:private default-connection-details
   {:user "default", :password "", :dbname "default", :host "localhost", :port "8123"})
-(def ^:private product-name "metabase/1.1.7")
+(def ^:private product-name "metabase/1.2.0")
 
 (defmethod sql-jdbc.conn/connection-details->spec :clickhouse
   [_ details]
@@ -170,14 +175,16 @@
      (get-all-tables db)
      (get-tables-in-dbs db))})
 
+(defn- ^:private is-db-required?
+  [field]
+  (not (str/starts-with? (get-in field [:database-type]) "Nullable")))
+
 (defmethod driver/describe-table :clickhouse
   [_ database table]
   (let [table-metadata (sql-jdbc.sync/describe-table :clickhouse database table)
         filtered-fields (for [field (:fields table-metadata)
-                              :let [updated-field
-                                    (update-in field [:database-type]
-                                               ;; Enum8(UInt8) -> Enum8
-                                               clojure.string/replace #"^(Enum.+)\(.+\)" "$1")]
+                              :let [updated-field (update-in field [:database-required]
+                                                             (fn [_] (is-db-required? field)))]
                               ;; Skip all AggregateFunction (but keeping SimpleAggregateFunction) columns
                               ;; JDBC does not support that and it crashes the data browser
                               :when (not (re-matches #"^AggregateFunction\(.+$"
@@ -579,10 +586,14 @@
 
 (defmethod driver/display-name :clickhouse [_] "ClickHouse")
 
-(defmethod driver/supports? [:clickhouse :standard-deviation-aggregations] [_ _] true)
-(defmethod driver/supports? [:clickhouse :set-timezone] [_ _] false)
-(defmethod driver/supports? [:clickhouse :foreign-keys] [_ _] (not config/is-test?))
-(defmethod driver/supports? [:clickhouse :test/jvm-timezone-setting] [_ _] false)
+(doseq [[feature supported?] {:standard-deviation-aggregations true
+                              :set-timezone                    false
+                              :foreign-keys                    (not config/is-test?)
+                              :test/jvm-timezone-setting       false
+                              :connection-impersonation        true
+                              :schemas                         true}]
+
+  (defmethod driver/database-supports? [:clickhouse feature] [_driver _feature _db] supported?))
 
 (defmethod sql-jdbc.sync/db-default-timezone :clickhouse
   [_ spec]
@@ -593,4 +604,14 @@
 (defmethod driver/db-start-of-week :clickhouse [_] :monday)
 
 (defmethod ddl.i/format-name :clickhouse [_ table-or-field-name]
-  (u/snake-key table-or-field-name))
+  (str/replace table-or-field-name #"-" "_"))
+
+;;; ------------------------------------------ User Impersonation ------------------------------------------
+
+(defmethod driver.sql/set-role-statement :clickhouse
+  [_ role]
+  (format "SET ROLE %s;" role))
+
+(defmethod driver.sql/default-database-role :clickhouse
+  [_ _]
+  "NONE")