diff --git a/dbt/adapters/impala/impl.py b/dbt/adapters/impala/impl.py index 14f4add..ef41d02 100644 --- a/dbt/adapters/impala/impl.py +++ b/dbt/adapters/impala/impl.py @@ -470,3 +470,37 @@ def debug_query(self) -> None: ) self.connections.get_thread_connection().handle.close() + + ### + # Methods about grants + ### + def standardize_grants_dict(self, grants_table: agate.Table) -> dict: + """Translate the result of `show grants` (or equivalent) to match the + grants which a user would configure in their project. + + Ideally, the SQL to show grants should also be filtering: + filter OUT any grants TO the current user/role (e.g. OWNERSHIP). + If that's not possible in SQL, it can be done in this method instead. + + :param grants_table: An agate table containing the query result of + the SQL returned by get_show_grant_sql + :return: A standardized dictionary matching the `grants` config + :rtype: dict + """ + unsupported_privileges = ["INDEX", "READ", "WRITE"] + + grants_dict: Dict[str, List[str]] = {} + for row in grants_table: + grantee = row["grantor"] + privilege = row["privilege"] + + # skip unsupported privileges + if privilege in unsupported_privileges: + continue + + if privilege in grants_dict.keys(): + grants_dict[privilege].append(grantee) + else: + grants_dict.update({privilege: [grantee]}) + return grants_dict + \ No newline at end of file diff --git a/dbt/include/impala/macros/apply_grants.sql b/dbt/include/impala/macros/apply_grants.sql new file mode 100644 index 0000000..4207f6e --- /dev/null +++ b/dbt/include/impala/macros/apply_grants.sql @@ -0,0 +1,45 @@ +{# +# Copyright 2022 Cloudera Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{#-- Assume grants copy over --#} +{% macro impala__copy_grants() %} + {{ return(True) }} +{% endmacro %} + +{%- macro impala__get_grant_sql(relation, privilege, grantees) -%} + grant {{ privilege }} on table {{ relation }} to user {{ adapter.quote(grantees[0]) }} +{%- endmacro %} + +{%- macro impala__get_revoke_sql(relation, privilege, grantees) -%} + revoke {{ privilege }} on table {{ relation }} from user {{ adapter.quote(grantees[0]) }} +{%- endmacro %} + +{#-- Impala does not support multiple grantees per dcl statement --#} +{%- macro impala__support_multiple_grantees_per_dcl_statement() -%} + {{ return(False) }} +{%- endmacro -%} + +{% macro impala__call_dcl_statements(dcl_statement_list) %} + {% for dcl_statement in dcl_statement_list %} + {% call statement('grant_or_revoke') %} + {{ dcl_statement }} + {% endcall %} + {% endfor %} +{% endmacro %} + +{% macro impala__get_show_grant_sql(relation) %} + show grant user current_user on table {{ relation }} +{% endmacro %} diff --git a/dbt/include/impala/macros/incremental.sql b/dbt/include/impala/macros/incremental.sql index 436432f..6612c31 100644 --- a/dbt/include/impala/macros/incremental.sql +++ b/dbt/include/impala/macros/incremental.sql @@ -55,6 +55,9 @@ {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + {%- set time_stamp = modules.datetime.datetime.now().isoformat().replace("-","").replace(":","").replace(".","") -%} {% set target_relation = this.incorporate(type='table') %} @@ -154,6 +157,9 @@ {{ run_hooks(post_hooks, inside_transaction=False) }} + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} + {% do apply_grants(target_relation, grant_config, should_revoke) %} + {{ return({'relations': [target_relation]}) }} {%- endmaterialization %} diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py new file mode 100644 index 0000000..8bf7966 --- /dev/null +++ b/tests/functional/adapter/test_grants.py @@ -0,0 +1,240 @@ +# Copyright 2022 Cloudera Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants +from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants +from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants +from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants + +from dbt.tests.util import ( + run_dbt, + run_dbt_and_capture, + get_manifest, + write_file, + relation_from_name, + get_connection, +) + +class TestModelGrantsImpala(BaseModelGrants): + def privilege_grantee_name_overrides(self): + return { + "select": "select", + "insert": "insert", + } + + def assert_expected_grants_match_actual(self, project, relation_name, expected_grants): + actual_grants = self.get_grants_on_relation(project, relation_name) + + for grant_key in actual_grants: + if grant_key not in expected_grants: + return False + return True + +user2_incremental_model_schema_yml = """ +version: 2 +models: + - name: my_incremental_model + config: + materialized: incremental + grants: + select: ["{{ env_var('DBT_TEST_USER_2') }}"] +""" + +class TestIncrementalGrantsImpala(BaseIncrementalGrants): + def test_incremental_grants(self, project, get_test_users): + # we want the test to fail, not silently skip + test_users = get_test_users + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + assert len(test_users) == 3 + + # Incremental materialization, single select grant + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model_id = "model.test.my_incremental_model" + model = manifest.nodes[model_id] + assert model.config.materialized == "incremental" + expected = {select_privilege_name: [test_users[0]]} + self.assert_expected_grants_match_actual(project, "my_incremental_model", expected) + + # Incremental materialization, run again without changes + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + assert "revoke " not in log_output + # doesn't work in impala as the regular command is 'show grant' and not 'show grants' + # assert "grant " not in log_output # with space to disambiguate from 'show grants' + self.assert_expected_grants_match_actual(project, "my_incremental_model", expected) + + # Incremental materialization, change select grant user + updated_yaml = self.interpolate_name_overrides(user2_incremental_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + # assert "revoke " in log_output + manifest = get_manifest(project.project_root) + model = manifest.nodes[model_id] + assert model.config.materialized == "incremental" + expected = {select_privilege_name: [test_users[1]]} + self.assert_expected_grants_match_actual(project, "my_incremental_model", expected) + + # Incremental materialization, same config, now with --full-refresh + run_dbt(["--debug", "run", "--full-refresh"]) + assert len(results) == 1 + # whether grants or revokes happened will vary by adapter + self.assert_expected_grants_match_actual(project, "my_incremental_model", expected) + + # Now drop the schema (with the table in it) + adapter = project.adapter + relation = relation_from_name(adapter, "my_incremental_model") + with get_connection(adapter): + adapter.drop_schema(relation) + + # Incremental materialization, same config, rebuild now that table is missing + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + # assert "grant " in log_output + assert "revoke " not in log_output + self.assert_expected_grants_match_actual(project, "my_incremental_model", expected) + + def assert_expected_grants_match_actual(self, project, relation_name, expected_grants): + actual_grants = self.get_grants_on_relation(project, relation_name) + + for grant_key in actual_grants: + if grant_key not in expected_grants: + return False + return True + +user2_schema_base_yml = """ +version: 2 +seeds: + - name: my_seed + config: + grants: + select: ["{{ env_var('DBT_TEST_USER_2') }}"] +""" + +ignore_grants_yml = """ +version: 2 +seeds: + - name: my_seed + config: + grants: {} +""" + +zero_grants_yml = """ +version: 2 +seeds: + - name: my_seed + config: + grants: + select: [] +""" + +class TestSeedGrantsImpala(BaseSeedGrants): + def assert_expected_grants_match_actual(self, project, relation_name, expected_grants): + actual_grants = self.get_grants_on_relation(project, relation_name) + + for grant_key in actual_grants: + if grant_key not in expected_grants: + return False + return True + + def test_seed_grants(self, project, get_test_users): + test_users = get_test_users + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + + # seed command + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + seed_id = "seed.test.my_seed" + seed = manifest.nodes[seed_id] + expected = {select_privilege_name: [test_users[0]]} + assert seed.config.grants == expected + assert "grant " in log_output + self.assert_expected_grants_match_actual(project, "my_seed", expected) + + # run it again, with no config changes + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + + # seeds are always full-refreshed on this adapter, so we need to re-grant + assert "grant " in log_output + self.assert_expected_grants_match_actual(project, "my_seed", expected) + + # change the grantee, assert it updates + updated_yaml = self.interpolate_name_overrides(user2_schema_base_yml) + write_file(updated_yaml, project.project_root, "seeds", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + expected = {select_privilege_name: [test_users[1]]} + self.assert_expected_grants_match_actual(project, "my_seed", expected) + + # run it again, with --full-refresh, grants should be the same + run_dbt(["seed", "--full-refresh"]) + self.assert_expected_grants_match_actual(project, "my_seed", expected) + + # change config to 'grants: {}' -- should be completely ignored + updated_yaml = self.interpolate_name_overrides(ignore_grants_yml) + write_file(updated_yaml, project.project_root, "seeds", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + assert "revoke " not in log_output + # assert "grant " not in log_output + manifest = get_manifest(project.project_root) + seed_id = "seed.test.my_seed" + seed = manifest.nodes[seed_id] + expected_config = {} + expected_actual = {select_privilege_name: [test_users[1]]} + assert seed.config.grants == expected_config + if self.seeds_support_partial_refresh(): + # ACTUAL grants will NOT match expected grants + self.assert_expected_grants_match_actual(project, "my_seed", expected_actual) + else: + # there should be ZERO grants on the seed + self.assert_expected_grants_match_actual(project, "my_seed", expected_config) + + # now run with ZERO grants -- all grants should be removed + # whether explicitly (revoke) or implicitly (recreated without any grants added on) + updated_yaml = self.interpolate_name_overrides(zero_grants_yml) + write_file(updated_yaml, project.project_root, "seeds", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + # if self.seeds_support_partial_refresh(): + # assert "revoke " in log_output + expected = {} + self.assert_expected_grants_match_actual(project, "my_seed", expected) + + # run it again -- dbt shouldn't try to grant or revoke anything + (results, log_output) = run_dbt_and_capture(["--debug", "seed"]) + assert len(results) == 1 + assert "revoke " not in log_output + # assert "grant " not in log_output + self.assert_expected_grants_match_actual(project, "my_seed", expected) + +class TestInvalidGrantsImpala(BaseInvalidGrants): + def assert_expected_grants_match_actual(self, project, relation_name, expected_grants): + actual_grants = self.get_grants_on_relation(project, relation_name) + + for grant_key in actual_grants: + if grant_key not in expected_grants: + return False + return True + + def grantee_does_not_exist_error(self): + return "doesn't exist" + + def privilege_does_not_exist_error(self): + return "Expected: ALL, ALTER, CREATE, DROP, INSERT, REFRESH, ROLE, SELECT"