Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: remove testable mangled names #303

Merged
merged 1 commit into from
Dec 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dbtmetabase/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
]

try:
from ._version import __version__ as version # type: ignore
from ._version import __version__ as version

__version__ = version
except ModuleNotFoundError:
Expand Down
64 changes: 33 additions & 31 deletions dbtmetabase/_exposures.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def dbname(details: Mapping) -> str:
return details[key]
return ""

ctx = self.__Context(
ctx = _Context(
model_refs={m.alias_path.lower(): m.ref for m in models if m.ref},
database_names={
d["id"]: dbname(d["details"]) for d in self.metabase.get_databases()
Expand Down Expand Up @@ -127,7 +127,7 @@ def dbname(details: Mapping) -> str:
uid=collection["id"],
models=("card", "dashboard"),
):
exposure = self.__Exposure(
exposure = _Exposure(
model=item["model"],
uid=item["id"],
label="Exposure [Unresolved Name]",
Expand All @@ -153,7 +153,7 @@ def dbname(details: Mapping) -> str:
f"Visualization: {entity.get('display', 'Unknown').title()}"
)

self.__exposure_card(ctx, exposure, entity)
self._exposure_card(ctx, exposure, entity)

if average_query_time_ms := entity.get("average_query_time"):
average_query_time_s = average_query_time_ms / 1000
Expand All @@ -179,7 +179,7 @@ def dbname(details: Mapping) -> str:
continue

if card := self.metabase.find_card(uid=card["id"]):
self.__exposure_card(ctx, exposure, card)
self._exposure_card(ctx, exposure, card)
else:
_logger.warning("Unexpected collection item '%s'", item["model"])
continue
Expand Down Expand Up @@ -236,7 +236,7 @@ def dbname(details: Mapping) -> str:

return exposures

def __exposure_card(self, ctx: __Context, exposure: __Exposure, card: Mapping):
def _exposure_card(self, ctx: _Context, exposure: _Exposure, card: Mapping):
"""Extracts exposures from Metabase questions."""

dataset_query = card.get("dataset_query", {})
Expand All @@ -248,7 +248,7 @@ def __exposure_card(self, ctx: __Context, exposure: __Exposure, card: Mapping):
else:
_logger.warning("Unsupported card type '%s'", card_type)

def __exposure_query(self, ctx: __Context, exposure: __Exposure, card: Mapping):
def __exposure_query(self, ctx: _Context, exposure: _Exposure, card: Mapping):
"""Extracts exposures from Metabase GUI queries."""

dataset_query = card.get("dataset_query", {})
Expand All @@ -259,7 +259,7 @@ def __exposure_query(self, ctx: __Context, exposure: __Exposure, card: Mapping):
# Question based on another question
source_card_uid = query_source.split("__")[-1]
if source_card := self.metabase.find_card(uid=source_card_uid):
self.__exposure_card(ctx, exposure, source_card)
self._exposure_card(ctx, exposure, source_card)

elif isinstance(query_source, int) and query_source in ctx.table_names:
# Question based on table
Expand All @@ -274,7 +274,7 @@ def __exposure_query(self, ctx: __Context, exposure: __Exposure, card: Mapping):
# Question based on another question
source_card_uid = join_source.split("__")[-1]
if source_card := self.metabase.find_card(uid=source_card_uid):
self.__exposure_card(ctx, exposure, source_card)
self._exposure_card(ctx, exposure, source_card)

continue

Expand All @@ -284,7 +284,7 @@ def __exposure_query(self, ctx: __Context, exposure: __Exposure, card: Mapping):
exposure.depends.add(joined_table)
_logger.info("Extracted model '%s' from join", joined_table)

def __exposure_native(self, ctx: __Context, exposure: __Exposure, card: Mapping):
def __exposure_native(self, ctx: _Context, exposure: _Exposure, card: Mapping):
"""Extracts exposures from Metabase native queries."""

dataset_query = card.get("dataset_query", {})
Expand Down Expand Up @@ -410,8 +410,8 @@ def __format_exposure(

return exposure

@staticmethod
def __write_exposures(
self,
exposures: Iterable[Mapping],
output_path: str,
output_grouping: Optional[str],
Expand Down Expand Up @@ -448,24 +448,26 @@ def __write_exposures(
stream=f,
)

@dc.dataclass
class __Context:
model_refs: Mapping[str, str]
database_names: Mapping[int, str]
table_names: Mapping[int, str]

@dc.dataclass
class __Exposure:
model: str
uid: str
label: str
name: str = ""
description: str = ""
created_at: str = ""
header: str = ""
creator_name: str = ""
creator_email: str = ""
average_query_time: Optional[str] = None
last_used_at: Optional[str] = None
native_query: Optional[str] = None
depends: Set[str] = dc.field(default_factory=set)

@dc.dataclass
class _Context:
model_refs: Mapping[str, str]
database_names: Mapping[int, str]
table_names: Mapping[int, str]


@dc.dataclass
class _Exposure:
model: str
uid: str
label: str
name: str = ""
description: str = ""
created_at: str = ""
header: str = ""
creator_name: str = ""
creator_email: str = ""
average_query_time: Optional[str] = None
last_used_at: Optional[str] = None
native_query: Optional[str] = None
depends: Set[str] = dc.field(default_factory=set)
59 changes: 30 additions & 29 deletions dbtmetabase/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@

_logger = logging.getLogger(__name__)

_SYNC_PERIOD = 5


class ModelsMixin(metaclass=ABCMeta):
"""Abstraction for exporting models."""

__SYNC_PERIOD = 5

DEFAULT_MODELS_SYNC_TIMEOUT = 30

@property
Expand Down Expand Up @@ -57,7 +57,7 @@ def export_models(
order_fields (bool, optional): Preserve column order in dbt project.
"""

ctx = self.__Context()
ctx = _Context()
success = True

database = self.metabase.find_database(name=metabase_database)
Expand All @@ -77,9 +77,9 @@ def export_models(
deadline = int(time.time()) + sync_timeout
synced = False
while not synced:
time.sleep(self.__SYNC_PERIOD)
time.sleep(_SYNC_PERIOD)

tables = self.__get_tables(database["id"])
tables = self._get_metabase_tables(database["id"])

synced = True
for model in models:
Expand Down Expand Up @@ -124,7 +124,7 @@ def export_models(
raise MetabaseStateError("Unable to sync models with Metabase")

for model in models:
success &= self.__export_model(
success &= self._export_model(
ctx=ctx,
model=model,
append_tags=append_tags,
Expand Down Expand Up @@ -159,9 +159,9 @@ def export_models(
if not success:
raise MetabaseStateError("Non-critical errors encountered, see above")

def __export_model(
def _export_model(
self,
ctx: __Context,
ctx: _Context,
model: Model,
append_tags: bool,
docs_url: Optional[str],
Expand Down Expand Up @@ -250,7 +250,7 @@ def __export_model(

def __export_model_column_order(
self,
ctx: __Context,
ctx: _Context,
model: Model,
api_table: Mapping,
table_key: str,
Expand Down Expand Up @@ -303,7 +303,7 @@ def __export_model_column_order(

def __export_column(
self,
ctx: __Context,
ctx: _Context,
schema_name: str,
model_name: str,
column: Column,
Expand Down Expand Up @@ -430,7 +430,7 @@ def __export_column(

return success

def __get_tables(self, database_id: str) -> Mapping[str, MutableMapping]:
def _get_metabase_tables(self, database_id: str) -> Mapping[str, MutableMapping]:
tables = {}

metadata = self.metabase.get_database_metadata(database_id)
Expand Down Expand Up @@ -461,8 +461,8 @@ def __get_tables(self, database_id: str) -> Mapping[str, MutableMapping]:

return tables

@staticmethod
def __filtered_models(
self,
models: Iterable[Model],
database_filter: Optional[Filter],
schema_filter: Optional[Filter],
Expand All @@ -479,25 +479,26 @@ def selected(m: Model) -> bool:

return list(filter(selected, models))

@dc.dataclass
class __Context:
tables: Mapping[str, MutableMapping] = dc.field(default_factory=dict)
updates: MutableMapping[str, MutableMapping] = dc.field(default_factory=dict)

def get_field(self, table_key: str, field_key: str) -> MutableMapping:
return self.tables.get(table_key, {}).get("fields", {}).get(field_key, {})
@dc.dataclass
class _Context:
tables: Mapping[str, MutableMapping] = dc.field(default_factory=dict)
updates: MutableMapping[str, MutableMapping] = dc.field(default_factory=dict)

def get_field(self, table_key: str, field_key: str) -> MutableMapping:
return self.tables.get(table_key, {}).get("fields", {}).get(field_key, {})

def update(self, entity: MutableMapping, change: Mapping, label: str):
entity.update(change)
def update(self, entity: MutableMapping, change: Mapping, label: str):
entity.update(change)

key = f"{entity['kind']}.{entity['id']}"
update = self.updates.get(key, {})
update["kind"] = entity["kind"]
update["id"] = entity["id"]
update["label"] = label
key = f"{entity['kind']}.{entity['id']}"
update = self.updates.get(key, {})
update["kind"] = entity["kind"]
update["id"] = entity["id"]
update["label"] = label

body = update.get("body", {})
body.update(change)
update["body"] = body
body = update.get("body", {})
body.update(change)
update["body"] = body

self.updates[key] = update
self.updates[key] = update
3 changes: 2 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
import pytest

import dbtmetabase._models
from tests._mocks import TMP_PATH, MockDbtMetabase, MockMetabase


@pytest.fixture(name="core")
def fixture_core() -> MockDbtMetabase:
c = MockDbtMetabase()
c._ModelsMixin__SYNC_PERIOD = 1 # type: ignore
dbtmetabase._models._SYNC_PERIOD = 1
return c


Expand Down
7 changes: 4 additions & 3 deletions tests/test_exposures.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import pytest
import yaml

from dbtmetabase._exposures import _Context, _Exposure
from tests._mocks import FIXTURES_PATH, TMP_PATH, MockDbtMetabase


Expand Down Expand Up @@ -96,20 +97,20 @@ def test_extract_exposures_native_depends(
query: str,
expected: set,
):
ctx = MockDbtMetabase._ExposuresMixin__Context( # type: ignore
ctx = _Context(
model_refs={
"database.schema.table0": "model0",
"database.public.table1": "model1",
},
database_names={1: "database"},
table_names={},
)
exposure = MockDbtMetabase._ExposuresMixin__Exposure( # type: ignore
exposure = _Exposure(
model="card",
uid="",
label="",
)
core._ExposuresMixin__exposure_card( # type: ignore
core._exposure_card(
ctx=ctx,
exposure=exposure,
card={
Expand Down
2 changes: 1 addition & 1 deletion tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def test_build_lookups(core: MockDbtMetabase):
"INVENTORY.SKUS": {"SKU_ID", "PRODUCT"},
}

actual_tables = core._ModelsMixin__get_tables(database_id="2") # type: ignore
actual_tables = core._get_metabase_tables(database_id="2")

assert set(actual_tables.keys()) == set(expected.keys())

Expand Down
Loading