Skip to content

Commit

Permalink
Fixed issue
Browse files Browse the repository at this point in the history
  • Loading branch information
rahulgoyal2987 committed Dec 9, 2024
1 parent 107769d commit 92ba3bf
Showing 1 changed file with 5 additions and 7 deletions.
12 changes: 5 additions & 7 deletions dbt/adapters/spark/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,6 @@ def _get_relation_information(self, row: "agate.Row") -> RelationInfo:

return _schema, name, information


def list_relations_without_caching(self, schema_relation: BaseRelation) -> List[BaseRelation]:
"""Distinct Spark compute engines may not support the same SQL featureset. Thus, we must
try different methods to fetch relation information."""
Expand Down Expand Up @@ -209,7 +208,7 @@ def list_relations_without_caching(self, schema_relation: BaseRelation) -> List[
logger.debug(f"{description} {schema_relation}: {e.msg}")
return []

def set_relation_information(self, relation: BaseRelation):
def set_relation_information(self, relation: BaseRelation) -> BaseRelation:
if relation.information:
return relation
rows: List[agate.Row] = super().get_columns_in_relation(relation)
Expand All @@ -224,7 +223,7 @@ def set_relation_information(self, relation: BaseRelation):
is_delta: bool = "Provider: delta" in information
is_hudi: bool = "Provider: hudi" in information
is_iceberg: bool = "Provider: iceberg" in information
relation: BaseRelation = self.Relation.create(
updated_relation: BaseRelation = self.Relation.create(
schema=relation.schema,
identifier=relation.identifier,
type=rel_type,
Expand All @@ -233,15 +232,14 @@ def set_relation_information(self, relation: BaseRelation):
is_iceberg=is_iceberg,
is_hudi=is_hudi,
)
return relation

return updated_relation

def get_relation(self, database: str, schema: str, identifier: str) -> Optional[BaseRelation]:
if not self.Relation.get_default_include_policy().database:
database = None # type: ignore

relation = super().get_relation(database, schema, identifier)
self.set_relation_information(relation) if relation else None
return self.set_relation_information(relation) if relation else None

def parse_describe_extended(
self, relation: BaseRelation, raw_rows: AttrDict
Expand Down Expand Up @@ -519,4 +517,4 @@ def debug_query(self) -> None:
diff_count.num_missing as num_mismatched
from row_count_diff
cross join diff_count
""".strip()
""".strip()

0 comments on commit 92ba3bf

Please sign in to comment.