Support grants

.changes/unreleased/Features-20221005-134613.yaml

@@ -0,0 +1,7 @@
+kind: Features
+body: Support security grants
+time: 2022-10-05T13:46:13.789545+02:00
+custom:
+  Author: mdesmet
+  Issue: ""
+  PR: "130"

Makefile

@@ -1,3 +1,9 @@
+.EXPORT_ALL_VARIABLES:
+
+DBT_TEST_USER_1=user1
+DBT_TEST_USER_2=user2
+DBT_TEST_USER_3=user3
+
 start-trino:
 	docker network create dbt-net || true
 	./docker/dbt/build.sh

dbt/adapters/trino/connections.py

@@ -94,6 +94,7 @@ class TrinoNoneCredentials(TrinoCredentials):
     host: str
     port: Port
     user: str
+    roles: Optional[Dict[str, str]] = None
     cert: Optional[str] = None
     http_scheme: HttpScheme = HttpScheme.HTTP
     http_headers: Optional[Dict[str, str]] = None
@@ -116,6 +117,7 @@ class TrinoCertificateCredentials(TrinoCredentials):
     client_certificate: str
     client_private_key: str
     user: Optional[str] = None
+    roles: Optional[Dict[str, str]] = None
     cert: Optional[str] = None
     http_headers: Optional[Dict[str, str]] = None
     session_properties: Dict[str, Any] = field(default_factory=dict)
@@ -143,6 +145,7 @@ class TrinoLdapCredentials(TrinoCredentials):
     port: Port
     user: str
     password: str
+    roles: Optional[Dict[str, str]] = None
     cert: Optional[str] = None
     http_headers: Optional[Dict[str, str]] = None
     session_properties: Dict[str, Any] = field(default_factory=dict)
@@ -166,6 +169,7 @@ class TrinoKerberosCredentials(TrinoCredentials):
     host: str
     port: Port
     user: str
+    roles: Optional[Dict[str, str]] = None
     keytab: Optional[str] = None
     principal: Optional[str] = None
     krb5_config: Optional[str] = None
@@ -210,6 +214,7 @@ class TrinoJwtCredentials(TrinoCredentials):
     port: Port
     jwt_token: str
     user: Optional[str] = None
+    roles: Optional[Dict[str, str]] = None
     cert: Optional[str] = None
     http_headers: Optional[Dict[str, str]] = None
     session_properties: Dict[str, Any] = field(default_factory=dict)
@@ -233,6 +238,7 @@ class TrinoOauthCredentials(TrinoCredentials):
     host: str
     port: Port
     user: Optional[str] = None
+    roles: Optional[Dict[str, str]] = None
     cert: Optional[str] = None
     http_headers: Optional[Dict[str, str]] = None
     session_properties: Dict[str, Any] = field(default_factory=dict)
@@ -404,6 +410,7 @@ def open(cls, connection):
             host=credentials.host,
             port=credentials.port,
             user=credentials.user,
+            roles=credentials.roles,
             catalog=credentials.database,
             schema=credentials.schema,
             http_scheme=credentials.http_scheme.value,

dbt/include/trino/macros/apply_grants.sql

@@ -0,0 +1,45 @@
+{% macro trino__get_show_grant_sql(relation) -%}
+    select
+        grantee,
+        lower(privilege_type) as privilege_type
+    from information_schema.table_privileges
+    where table_catalog = '{{ relation.database }}'
+    and table_schema = '{{ relation.schema }}'
+    and table_name = '{{ relation.identifier }}'
+{%- endmacro %}
+
+{% macro trino__copy_grants() %}
+    {#
+        -- This macro should return true or false depending on the answer to
+        -- following question:
+        -- when an object is fully replaced on your database, do grants copy over?
+        -- e.g. on Postgres this is never true,
+        -- on Spark this is different for views vs. non-Delta tables vs. Delta tables,
+        -- on Snowflake it depends on the user-supplied copy_grants configuration.
+        -- true by default, which means “play it safe”: grants MIGHT have copied over,
+        -- so dbt will run an extra query to check them + calculate diffs.
+    #}
+    {{ return(False) }}
+{% endmacro %}
+
+{%- macro trino__get_grant_sql(relation, privilege, grantees) -%}
+    grant {{ privilege }} on {{ relation }} to {{ adapter.quote(grantees[0]) }}
+{%- endmacro %}
+
+{%- macro trino__support_multiple_grantees_per_dcl_statement() -%}
+    {#
+        -- This macro should return true or false depending on the answer to
+        -- following question:
+        -- does this database support grant {privilege} to user_a, user_b, ...?
+        -- or do user_a + user_b need their own separate grant statements?
+    #}
+    {{ return(False) }}
+{%- endmacro -%}
+
+{% macro trino__call_dcl_statements(dcl_statement_list) %}
+    {% for dcl_statement in dcl_statement_list %}
+        {% call statement('grant_or_revoke') %}
+            {{ dcl_statement }}
+        {% endcall %}
+    {% endfor %}
+{% endmacro %}

dbt/include/trino/macros/materializations/incremental.sql

@@ -130,6 +130,9 @@
 
   {{ run_hooks(pre_hooks) }}
 
+  -- grab current tables grants config for comparision later on
+  {% set grant_config = config.get('grants') %}
+
   {% if existing_relation is none %}
     {% set build_sql = create_table_as(False, target_relation, sql) %}
   {% elif existing_relation.is_view or full_refresh_mode %}
@@ -153,6 +156,10 @@
 
   {{ run_hooks(post_hooks) }}
 
+  {% set should_revoke =
+   should_revoke(existing_relation.is_table, full_refresh_mode) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
   {% do persist_docs(target_relation, model) %}
 
   {{ return({'relations': [target_relation]}) }}

dbt/include/trino/macros/materializations/table.sql

@@ -37,6 +37,9 @@
 
   {{ run_hooks(pre_hooks) }}
 
+  -- grab current tables grants config for comparision later on
+  {% set grant_config = config.get('grants') %}
+
   {% if on_table_exists == 'rename' %}
       {#-- build modeldock #}
       {% call statement('main') -%}
@@ -65,9 +68,12 @@
       {%- endcall %}
   {% endif %}
 
-  {{ run_hooks(post_hooks) }}
-
   {% do persist_docs(target_relation, model) %}
 
+  {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
+  {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
+
+  {{ run_hooks(post_hooks) }}
+
   {{ return({'relations': [target_relation]}) }}
 {% endmaterialization %}

docker-compose-starburst.yml

@@ -26,7 +26,7 @@ services:
 
   hive-metastore:
     hostname: hive-metastore
-    image: 'starburstdata/hive:3.1.2-e.15'
+    image: 'starburstdata/hive:3.1.2-e.18'
     ports:
       - '9083:9083' # Metastore Thrift
     environment:
@@ -53,6 +53,7 @@ services:
       AZURE_ABFS_OAUTH_SECRET: ""
       AZURE_ABFS_OAUTH_ENDPOINT: ""
       AZURE_WASB_ACCESS_KEY: ""
+      HIVE_METASTORE_USERS_IN_ADMIN_ROLE: "admin"
     depends_on:
       - metastore_db
 

docker-compose-trino.yml

@@ -27,7 +27,7 @@ services:
 
   hive-metastore:
     hostname: hive-metastore
-    image: 'starburstdata/hive:3.1.2-e.15'
+    image: 'starburstdata/hive:3.1.2-e.18'
     ports:
       - '9083:9083' # Metastore Thrift
     environment:
@@ -54,6 +54,7 @@ services:
       AZURE_ABFS_OAUTH_SECRET: ""
       AZURE_ABFS_OAUTH_ENDPOINT: ""
       AZURE_WASB_ACCESS_KEY: ""
+      HIVE_METASTORE_USERS_IN_ADMIN_ROLE: "admin"
     depends_on:
       - metastore_db
 

docker/starburst/catalog/hive.properties

@@ -0,0 +1,10 @@
+connector.name=hive
+hive.metastore.uri=thrift://hive-metastore:9083
+hive.s3.endpoint=http://minio:9000
+hive.s3.path-style-access=true
+hive.s3.aws-access-key=minio
+hive.s3.aws-secret-key=minio123
+hive.metastore-cache-ttl=0s
+hive.metastore-refresh-interval=5s
+hive.metastore-timeout=10s
+hive.security=sql-standard

docker/trino/catalog/hive.properties

@@ -0,0 +1,10 @@
+connector.name=hive
+hive.metastore.uri=thrift://hive-metastore:9083
+hive.s3.endpoint=http://minio:9000
+hive.s3.path-style-access=true
+hive.s3.aws-access-key=minio
+hive.s3.aws-secret-key=minio123
+hive.metastore-cache-ttl=0s
+hive.metastore-refresh-interval=5s
+hive.metastore-timeout=10s
+hive.security=sql-standard

pytest.ini

@@ -8,6 +8,7 @@ testpaths =
 markers =
     delta
     iceberg
+    hive
     postgresql
     prepared_statements_disabled
     skip_profile(profile)

tests/conftest.py

@@ -31,6 +31,7 @@ def dbt_profile_target(request):
     postgresql = request.node.get_closest_marker("postgresql")
     iceberg = request.node.get_closest_marker("iceberg")
     delta = request.node.get_closest_marker("delta")
+    hive = request.node.get_closest_marker("hive")
 
     if sum(bool(x) for x in (postgresql, iceberg, delta)) > 1:
         raise ValueError("Only one of postgresql, iceberg, delta can be specified as a marker")
@@ -44,6 +45,9 @@ def dbt_profile_target(request):
     if iceberg:
         target.update({"catalog": "iceberg"})
 
+    if hive:
+        target.update({"catalog": "hive"})
+
     return target
 
 
@@ -56,6 +60,9 @@ def get_trino_starburst_target():
         "port": 8080,
         "user": "admin",
         "password": "",
+        "roles": {
+            "hive": "ROLE{admin}",
+        },
         "catalog": "memory",
         "schema": "default",
     }

tests/functional/adapter/test_model_grants.py

@@ -0,0 +1,32 @@
+import pytest
+from dbt.context.base import BaseContext  # diff_of_two_dicts only
+from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants
+
+
+@pytest.mark.hive
+# TODO: setup Galaxy and Starbust tests
+#   See https://github.com/starburstdata/dbt-trino/issues/147
+#   and also https://github.com/starburstdata/dbt-trino/issues/146
+@pytest.mark.skip_profile("starburst_galaxy")
+# To run this test locally add following env vars:
+# DBT_TEST_USER_1=user1
+# DBT_TEST_USER_2=user2
+# DBT_TEST_USER_3=user3
+class TestModelGrants(BaseModelGrants):
+    def assert_expected_grants_match_actual(self, project, relation_name, expected_grants):
+        actual_grants = self.get_grants_on_relation(project, relation_name)
+        # Remove the creation user
+        try:
+            for privilege in ["delete", "update", "insert", "select"]:
+                if privilege in actual_grants:
+                    actual_grants[privilege].remove("admin")
+                    if len(actual_grants[privilege]) == 0:
+                        del actual_grants[privilege]
+        except ValueError:
+            pass
+
+        # need a case-insensitive comparison
+        # so just a simple "assert expected == actual_grants" won't work
+        diff_a = BaseContext.diff_of_two_dicts(actual_grants, expected_grants)
+        diff_b = BaseContext.diff_of_two_dicts(expected_grants, actual_grants)
+        assert diff_a == diff_b == {}

tox.ini

@@ -13,7 +13,7 @@ deps =
 [testenv:integration]
 basepython = python3
 commands = /bin/bash -c '{envpython} -m pytest tests/functional'
-passenv = DBT_INVOCATION_ENV DBT_TEST_TRINO_HOST
+passenv = DBT_INVOCATION_ENV DBT_TEST_TRINO_HOST DBT_TEST_USER_1 DBT_TEST_USER_2 DBT_TEST_USER_3
 deps =
     -r{toxinidir}/dev_requirements.txt
     -e.