diff --git a/dbt/adapters/impala/__version__.py b/dbt/adapters/impala/__version__.py index a1572b5..5cf4794 100644 --- a/dbt/adapters/impala/__version__.py +++ b/dbt/adapters/impala/__version__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -version = "1.4.2" +version = "1.4.3" diff --git a/dbt/adapters/impala/impl.py b/dbt/adapters/impala/impl.py index 9c5619b..36c4b2c 100644 --- a/dbt/adapters/impala/impl.py +++ b/dbt/adapters/impala/impl.py @@ -35,6 +35,8 @@ LIST_SCHEMAS_MACRO_NAME = "list_schemas" LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching" +LIST_TABLES_IN_RELATION_MACRO_NAME = "list_tables_in_relation" +GET_RELATIONSHIP_TYPE_MACRO_NAME = "get_relation_type" KEY_TABLE_OWNER = "Owner" KEY_TABLE_STATISTICS = "Statistics" @@ -95,31 +97,35 @@ def list_schemas(self, database: str) -> List[str]: return schemas + def fetch_relation_type(self, relation: Relation) -> str: + try: + kwargs = {"relation": relation} + relation_type = self.execute_macro(GET_RELATIONSHIP_TYPE_MACRO_NAME, kwargs=kwargs) + except dbt.exceptions.DbtRuntimeError as e: + logger.error( + f"Unable to fetch relation type {relation.schema}.{relation.identifier}: {e.msg}" + ) + return None + return relation_type + def list_relations_without_caching( self, schema_relation: ImpalaRelation ) -> List[ImpalaRelation]: - kwargs = {"schema_relation": schema_relation} - try: - results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) + kwargs = {"relation": schema_relation} + table_relations = self.execute_macro(LIST_TABLES_IN_RELATION_MACRO_NAME, kwargs=kwargs) except dbt.exceptions.DbtRuntimeError as e: errmsg = getattr(e, "msg", "") - if f"Database '{schema_relation}' not found" in errmsg: + if f"Database does not exist" in errmsg: return [] else: - description = "Error while retrieving information about" - logger.error(f"{description} {schema_relation}: {e.msg}") + logger.error(f"Unable to extract tables in relation {schema_relation}: {errmsg}") raise e relations = [] - for row in results: - if len(row) != 2: - raise dbt.exceptions.DbtRuntimeError( - f'Invalid value from "show table extended ...", ' - f"got {len(row)} values, expected 4" - ) - _identifier = row[0] - _rel_type = row[1] + for table_relation in table_relations: + _rel_type = self.fetch_relation_type(table_relation) + _identifier = table_relation.identifier relation = self.Relation.create( database=None, diff --git a/dbt/include/impala/macros/adapters.sql b/dbt/include/impala/macros/adapters.sql index 53ac921..8cbe597 100644 --- a/dbt/include/impala/macros/adapters.sql +++ b/dbt/include/impala/macros/adapters.sql @@ -125,6 +125,21 @@ {{ return(load_result('list_schemas').table) }} {% endmacro %} +{% macro list_tables_in_relation(relation) %} + {% set result_set = run_query('show tables in ' ~ relation) %} + + {% set tables = [] %} + + {% if execute %} + {%- for rs in result_set -%} + {% do tables.append(relation.new_copy(relation.schema, rs[0])) %} + {%- endfor -%} + {% endif %} + + {{ return(tables) }} +{% endmacro %} + + {# Note: This function currently needs to query each object to determine its type. Depending on the schema, this function could be expensive. #} {% macro impala__list_relations_without_caching(relation) %} {% set result_set = run_query('show tables in ' ~ relation) %} diff --git a/setup.py b/setup.py index 4862e5b..1672674 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def _get_dbt_core_version(): package_name = "dbt-impala" # make sure this always matches dbt/adapters/dbt_impala/__version__.py -package_version = "1.4.2" +package_version = "1.4.3" description = """The Impala adapter plugin for dbt""" dbt_core_version = _get_dbt_core_version()