feat(ingest): enable CLL for dbt by default (datahub-project#9466)

acryldata · Dec 18, 2023 · 9d386fb · 9d386fb
1 parent b4fe451
commit 9d386fb
Show file tree

Hide file tree

Showing 9 changed files with 1,630 additions and 4 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
@@ -300,7 +300,7 @@ class DBTCommonConfig(
         description="When enabled, schemas will be inferred from the dbt node definition.",
     )
     include_column_lineage: bool = Field(
-        default=False,
+        default=True,
         description="When enabled, column-level lineage will be extracted from the dbt node definition. Requires `infer_dbt_schemas` to be enabled. "
         "If you run into issues where the column name casing does not match up with properly, providing a datahub_api or using the rest sink will improve accuracy.",
     )
@@ -696,7 +696,10 @@ def get_column_type(
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion")
 @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
-@capability(SourceCapability.LINEAGE_FINE, "Enabled using `include_column_lineage`")
+@capability(
+    SourceCapability.LINEAGE_FINE,
+    "Enabled by default, configure using `include_column_lineage`",
+)
 class DBTSourceBase(StatefulIngestionSourceBase):
     def __init__(self, config: DBTCommonConfig, ctx: PipelineContext, platform: str):
         super().__init__(config, ctx)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
@@ -1015,7 +1015,7 @@ def __init__(
         self.report = report
         self.source_config = source_config
 
-    @lru_cache()
+    @lru_cache(maxsize=200)
     def get_explore(self, model: str, explore: str) -> Optional[LookerExplore]:
         looker_explore = LookerExplore.from_api(
             model,

diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_lib_wrapper.py
@@ -114,7 +114,7 @@ def get_available_permissions(self) -> Set[str]:
 
         return permissions
 
-    @lru_cache(maxsize=2000)
+    @lru_cache(maxsize=1000)
     def get_user(self, id_: str, user_fields: str) -> Optional[User]:
         self.client_stats.user_calls += 1
         try:

diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json
diff --git a/...ata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json b/...ata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json
diff --git a/...ata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json b/...ata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json
diff --git a/...ta-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json b/...ta-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json
diff --git a/...a-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json b/...a-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json