Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Repository unit tests for ManifestLoader.get_full_manifest #10147

Merged
merged 6 commits into from
May 15, 2024
1 change: 1 addition & 0 deletions tests/unit/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from dbt.contracts.graph.nodes import SourceDefinition

# All manifest related fixtures.
from tests.unit.utils.adapter import * # noqa
from tests.unit.utils.manifest import * # noqa
from tests.unit.utils.project import * # noqa

Expand Down
98 changes: 84 additions & 14 deletions tests/unit/parser/test_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,14 @@
from unittest.mock import MagicMock, patch

import pytest
from pytest_mock import MockerFixture

from dbt.config import RuntimeConfig
from dbt.contracts.graph.manifest import Manifest
from dbt.flags import set_from_args
from dbt.parser.manifest import ManifestLoader


@pytest.fixture
def mock_project():
mock_project = MagicMock(RuntimeConfig)
mock_project.cli_vars = {}
mock_project.args = MagicMock()
mock_project.args.profile = "test"
mock_project.args.target = "test"
mock_project.project_env_vars = {}
mock_project.profile_env_vars = {}
mock_project.project_target_path = "mock_target_path"
mock_project.credentials = MagicMock()
return mock_project
from dbt.parser.read_files import FileDiff
from dbt.tracking import User


class TestPartialParse:
Expand Down Expand Up @@ -91,3 +80,84 @@ def test_partial_parse_safe_update_project_parser_files_partially(
assert "full_reparse_reason" in exc_info
assert "KeyError: 'Whoopsie!'" == exc_info["exception"]
assert isinstance(exc_info["code"], str) or isinstance(exc_info["code"], type(None))


class TestGetFullManifest:
@pytest.fixture
def set_required_mocks(
self, mocker: MockerFixture, manifest: Manifest, mock_adapter: MagicMock
):
mocker.patch("dbt.parser.manifest.get_adapter").return_value = mock_adapter
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to do the patch inside the fixture? So users of those fixtures can just request the fixture then the patch is done?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried doing so but, as noted in the commit message of 791ec1d, the mocks stopped propagating when I did so. I can take another stab at it, but can't make any promises. It definitely would be nice if we could do it as a fixture though.

Copy link
Contributor Author

@QMalcolm QMalcolm May 15, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Converting it to a fixture does seem to work now. Not sure why it didn't before and why it does now 🤔 🤷

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder whether it was not imported properly before

mocker.patch("dbt.parser.manifest.ManifestLoader.load").return_value = manifest
mocker.patch("dbt.parser.manifest._check_manifest").return_value = None
mocker.patch(
"dbt.parser.manifest.ManifestLoader.save_macros_to_adapter"
).return_value = None
mocker.patch("dbt.tracking.active_user").return_value = User(None)

def test_write_perf_info(
self,
mock_project: MagicMock,
mocker: MockerFixture,
set_required_mocks,
) -> None:
write_perf_info = mocker.patch("dbt.parser.manifest.ManifestLoader.write_perf_info")

ManifestLoader.get_full_manifest(
config=mock_project,
# write_perf_info=False let it default instead
)
assert not write_perf_info.called

ManifestLoader.get_full_manifest(config=mock_project, write_perf_info=False)
assert not write_perf_info.called

ManifestLoader.get_full_manifest(config=mock_project, write_perf_info=True)
assert write_perf_info.called

def test_reset(
self,
mock_project: MagicMock,
mock_adapter: MagicMock,
set_required_mocks,
) -> None:

ManifestLoader.get_full_manifest(
config=mock_project,
# reset=False let it default instead
)
assert not mock_project.clear_dependencies.called
assert not mock_adapter.clear_macro_resolver.called

ManifestLoader.get_full_manifest(config=mock_project, reset=False)
assert not mock_project.clear_dependencies.called
assert not mock_adapter.clear_macro_resolver.called

ManifestLoader.get_full_manifest(config=mock_project, reset=True)
assert mock_project.clear_dependencies.called
assert mock_adapter.clear_macro_resolver.called

def test_partial_parse_file_diff_flag(
self,
mock_project: MagicMock,
mocker: MockerFixture,
set_required_mocks,
) -> None:

# FileDiff.from_dict is only called if PARTIAL_PARSE_FILE_DIFF == False
# So we can track this function call to check if setting PARTIAL_PARSE_FILE_DIFF
# works appropriately
mock_file_diff = mocker.patch("dbt.parser.read_files.FileDiff.from_dict")
mock_file_diff.return_value = FileDiff([], [], [])

set_from_args(Namespace(), {})
ManifestLoader.get_full_manifest(config=mock_project)
assert not mock_file_diff.called

set_from_args(Namespace(PARTIAL_PARSE_FILE_DIFF=True), {})
ManifestLoader.get_full_manifest(config=mock_project)
assert not mock_file_diff.called

set_from_args(Namespace(PARTIAL_PARSE_FILE_DIFF=False), {})
ManifestLoader.get_full_manifest(config=mock_project)
assert mock_file_diff.called
21 changes: 21 additions & 0 deletions tests/unit/utils/adapter.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
from unittest.mock import MagicMock

import pytest

from dbt.adapters.postgres import PostgresAdapter
from dbt.adapters.sql import SQLConnectionManager


@pytest.fixture
def mock_connection_manager() -> MagicMock:
mock_connection_manager = MagicMock(SQLConnectionManager)
mock_connection_manager.set_query_header = lambda query_header_context: None
return mock_connection_manager


@pytest.fixture
def mock_adapter(mock_connection_manager: MagicMock) -> MagicMock:
mock_adapter = MagicMock(PostgresAdapter)
mock_adapter.connections = mock_connection_manager
mock_adapter.clear_macro_resolver = MagicMock()
return mock_adapter
18 changes: 18 additions & 0 deletions tests/unit/utils/project.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
from unittest.mock import MagicMock

import pytest

from dbt.adapters.contracts.connection import QueryComment
from dbt.config import RuntimeConfig
from dbt.config.project import Project, RenderComponents, VarProvider
from dbt.config.selectors import SelectorConfig
from dbt.contracts.project import PackageConfig
Expand Down Expand Up @@ -68,3 +71,18 @@ def project(selector_config: SelectorConfig) -> Project:
restrict_access=False,
dbt_cloud={},
)


@pytest.fixture
def mock_project():
mock_project = MagicMock(RuntimeConfig)
mock_project.cli_vars = {}
mock_project.args = MagicMock()
mock_project.args.profile = "test"
mock_project.args.target = "test"
mock_project.project_env_vars = {}
mock_project.profile_env_vars = {}
mock_project.project_target_path = "mock_target_path"
mock_project.credentials = MagicMock()
mock_project.clear_dependencies = MagicMock()
return mock_project