Skip to content

Commit

Permalink
Modify impala__list_relations_without_caching macro implementation to…
Browse files Browse the repository at this point in the history
… query each object type to return the object type (table / view) (#41)

Test plan:
1. Merge PR #33 to ensure new adpater test framework code is present.
2. Ensure pytest is installed
2. Run : python3 -m pytest tests/functional -k 'TestSimpleMaterializationsImpala' - This test should succeed.
  • Loading branch information
tovganesh authored Jul 15, 2022
1 parent 2d6bbc8 commit 40a22d8
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 11 deletions.
10 changes: 3 additions & 7 deletions dbt/adapters/impala/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,23 +128,19 @@ def list_relations_without_caching(

relations = []
for row in results:
if len(row) != 1:
if len(row) != 2:
raise dbt.exceptions.RuntimeException(
f'Invalid value from "show table extended ...", '
f'got {len(row)} values, expected 4'
)
_identifier = row[0]

# TODO: the following is taken from spark, needs to see what is there in impala
# TODO: this modification is not really right, need fix
rel_type = RelationType.View \
if 'view' in _identifier else RelationType.Table
_rel_type = row[1]

relation = self.Relation.create(
database=schema_relation.database,
schema=schema_relation.schema,
identifier=_identifier,
type=rel_type,
type=_rel_type,
information=_identifier,
)
relations.append(relation)
Expand Down
10 changes: 10 additions & 0 deletions dbt/adapters/impala/relation.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,13 @@ class ImpalaRelation(BaseRelation):

def render(self):
return super().render()

def new_copy(self, name, identifier):
new_relation = ImpalaRelation.create(
database=name,
schema=name,
identifier=identifier,
information=identifier,
)

return new_relation
20 changes: 16 additions & 4 deletions dbt/include/impala/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,24 @@
{{ return(load_result('list_schemas').table) }}
{% endmacro %}

{# Note: This function currently needs to query each object to determine its type. Depending on the schema, this function could be expensive. #}
{% macro impala__list_relations_without_caching(relation) %}
{% call statement('list_relations_without_caching', fetch_result=True) -%}
show tables in {{ relation }}
{% endcall %}
{% set result_set = run_query('show tables in ' ~ relation) %}
{% set objects_with_type = [] %}

{%- for rs in result_set -%}
{% set obj_type = [] %}

{% do obj_type.append(rs[0]) %}

{% set obj_rel = relation.new_copy(relation.schema, rs[0]) %}
{% set rel_type = get_relation_type(obj_rel) %}
{% do obj_type.append(rel_type) %}

{% do objects_with_type.append(obj_type) %}
{%- endfor -%}

{% do return(load_result('list_relations_without_caching').table) %}
{{ return(objects_with_type) }}
{% endmacro %}

{% macro impala__create_table_as(temporary, relation, sql) -%}
Expand Down

0 comments on commit 40a22d8

Please sign in to comment.