Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Repository unit tests for ManifestLoader.get_full_manifest #10147

Merged
merged 6 commits into from
May 15, 2024
107 changes: 107 additions & 0 deletions tests/unit/parser/test_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,16 @@
from unittest.mock import MagicMock, patch

import pytest
from pytest_mock import MockerFixture

from dbt.adapters.postgres import PostgresAdapter
from dbt.adapters.sql import SQLConnectionManager
from dbt.config import RuntimeConfig
from dbt.contracts.graph.manifest import Manifest
from dbt.flags import set_from_args
from dbt.parser.manifest import ManifestLoader
from dbt.parser.read_files import FileDiff
from dbt.tracking import User


@pytest.fixture
Expand All @@ -20,9 +25,25 @@ def mock_project():
mock_project.profile_env_vars = {}
mock_project.project_target_path = "mock_target_path"
mock_project.credentials = MagicMock()
mock_project.clear_dependencies = MagicMock()
return mock_project


@pytest.fixture
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shoude this live somewhere in the utils being used by other unittests?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can lift these 👍

def mock_connection_manager():
mock_connection_manager = MagicMock(SQLConnectionManager)
mock_connection_manager.set_query_header = lambda query_header_context: None
return mock_connection_manager


@pytest.fixture
def mock_adapter(mock_connection_manager: MagicMock):
mock_adapter = MagicMock(PostgresAdapter)
mock_adapter.connections = mock_connection_manager
mock_adapter.clear_macro_resolver = MagicMock()
return mock_adapter


class TestPartialParse:
@patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check")
@patch("dbt.parser.manifest.os.path.exists")
Expand Down Expand Up @@ -91,3 +112,89 @@ def test_partial_parse_safe_update_project_parser_files_partially(
assert "full_reparse_reason" in exc_info
assert "KeyError: 'Whoopsie!'" == exc_info["exception"]
assert isinstance(exc_info["code"], str) or isinstance(exc_info["code"], type(None))


class TestGetFullManifest:
def set_required_mocks(
self, mocker: MockerFixture, manifest: Manifest, mock_adapter: MagicMock
):
mocker.patch("dbt.parser.manifest.get_adapter").return_value = mock_adapter
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to do the patch inside the fixture? So users of those fixtures can just request the fixture then the patch is done?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried doing so but, as noted in the commit message of 791ec1d, the mocks stopped propagating when I did so. I can take another stab at it, but can't make any promises. It definitely would be nice if we could do it as a fixture though.

Copy link
Contributor Author

@QMalcolm QMalcolm May 15, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Converting it to a fixture does seem to work now. Not sure why it didn't before and why it does now 🤔 🤷

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder whether it was not imported properly before

mocker.patch("dbt.parser.manifest.ManifestLoader.load").return_value = manifest
mocker.patch("dbt.parser.manifest._check_manifest").return_value = None
mocker.patch(
"dbt.parser.manifest.ManifestLoader.save_macros_to_adapter"
).return_value = None
mocker.patch("dbt.tracking.active_user").return_value = User(None)

def test_write_perf_info(
self,
manifest: Manifest,
mock_project: MagicMock,
mock_adapter: MagicMock,
mocker: MockerFixture,
) -> None:
self.set_required_mocks(mocker, manifest, mock_adapter)
write_perf_info = mocker.patch("dbt.parser.manifest.ManifestLoader.write_perf_info")

ManifestLoader.get_full_manifest(
config=mock_project,
# write_perf_info=False let it default instead
)
assert not write_perf_info.called

ManifestLoader.get_full_manifest(config=mock_project, write_perf_info=False)
assert not write_perf_info.called

ManifestLoader.get_full_manifest(config=mock_project, write_perf_info=True)
assert write_perf_info.called

def test_reset(
self,
manifest: Manifest,
mock_project: MagicMock,
mock_adapter: MagicMock,
mocker: MockerFixture,
) -> None:
self.set_required_mocks(mocker, manifest, mock_adapter)

ManifestLoader.get_full_manifest(
config=mock_project,
# reset=False let it default instead
)
assert not mock_project.clear_dependencies.called
assert not mock_adapter.clear_macro_resolver.called

ManifestLoader.get_full_manifest(config=mock_project, reset=False)
assert not mock_project.clear_dependencies.called
assert not mock_adapter.clear_macro_resolver.called

ManifestLoader.get_full_manifest(config=mock_project, reset=True)
assert mock_project.clear_dependencies.called
assert mock_adapter.clear_macro_resolver.called

def test_partial_parse_file_diff_flag(
self,
manifest: Manifest,
mock_project: MagicMock,
mock_adapter: MagicMock,
mocker: MockerFixture,
) -> None:
self.set_required_mocks(mocker, manifest, mock_adapter)

# FileDiff.from_dict is only called if PARTIAL_PARSE_FILE_DIFF == False
# So we can track this function call to check if setting PARTIAL_PARSE_FILE_DIFF
# works appropriately
mock_file_diff = mocker.patch("dbt.parser.read_files.FileDiff.from_dict")
mock_file_diff.return_value = FileDiff([], [], [])

set_from_args(Namespace(), {})
ManifestLoader.get_full_manifest(config=mock_project)
assert not mock_file_diff.called

set_from_args(Namespace(PARTIAL_PARSE_FILE_DIFF=True), {})
ManifestLoader.get_full_manifest(config=mock_project)
assert not mock_file_diff.called

set_from_args(Namespace(PARTIAL_PARSE_FILE_DIFF=False), {})
ManifestLoader.get_full_manifest(config=mock_project)
assert mock_file_diff.called