-
Notifications
You must be signed in to change notification settings - Fork 1.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Repository unit tests for ManifestLoader.get_full_manifest
#10147
Changes from 4 commits
15f27b6
a54d734
791ec1d
8dc4228
184fe9a
16edcbb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,11 +2,16 @@ | |
from unittest.mock import MagicMock, patch | ||
|
||
import pytest | ||
from pytest_mock import MockerFixture | ||
|
||
from dbt.adapters.postgres import PostgresAdapter | ||
from dbt.adapters.sql import SQLConnectionManager | ||
from dbt.config import RuntimeConfig | ||
from dbt.contracts.graph.manifest import Manifest | ||
from dbt.flags import set_from_args | ||
from dbt.parser.manifest import ManifestLoader | ||
from dbt.parser.read_files import FileDiff | ||
from dbt.tracking import User | ||
|
||
|
||
@pytest.fixture | ||
|
@@ -20,9 +25,25 @@ def mock_project(): | |
mock_project.profile_env_vars = {} | ||
mock_project.project_target_path = "mock_target_path" | ||
mock_project.credentials = MagicMock() | ||
mock_project.clear_dependencies = MagicMock() | ||
return mock_project | ||
|
||
|
||
@pytest.fixture | ||
def mock_connection_manager(): | ||
mock_connection_manager = MagicMock(SQLConnectionManager) | ||
mock_connection_manager.set_query_header = lambda query_header_context: None | ||
return mock_connection_manager | ||
|
||
|
||
@pytest.fixture | ||
def mock_adapter(mock_connection_manager: MagicMock): | ||
mock_adapter = MagicMock(PostgresAdapter) | ||
mock_adapter.connections = mock_connection_manager | ||
mock_adapter.clear_macro_resolver = MagicMock() | ||
return mock_adapter | ||
|
||
|
||
class TestPartialParse: | ||
@patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check") | ||
@patch("dbt.parser.manifest.os.path.exists") | ||
|
@@ -91,3 +112,89 @@ def test_partial_parse_safe_update_project_parser_files_partially( | |
assert "full_reparse_reason" in exc_info | ||
assert "KeyError: 'Whoopsie!'" == exc_info["exception"] | ||
assert isinstance(exc_info["code"], str) or isinstance(exc_info["code"], type(None)) | ||
|
||
|
||
class TestGetFullManifest: | ||
def set_required_mocks( | ||
self, mocker: MockerFixture, manifest: Manifest, mock_adapter: MagicMock | ||
): | ||
mocker.patch("dbt.parser.manifest.get_adapter").return_value = mock_adapter | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is it possible to do the patch inside the fixture? So users of those fixtures can just request the fixture then the patch is done? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I tried doing so but, as noted in the commit message of 791ec1d, the mocks stopped propagating when I did so. I can take another stab at it, but can't make any promises. It definitely would be nice if we could do it as a fixture though. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Converting it to a fixture does seem to work now. Not sure why it didn't before and why it does now 🤔 🤷 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder whether it was not imported properly before |
||
mocker.patch("dbt.parser.manifest.ManifestLoader.load").return_value = manifest | ||
mocker.patch("dbt.parser.manifest._check_manifest").return_value = None | ||
mocker.patch( | ||
"dbt.parser.manifest.ManifestLoader.save_macros_to_adapter" | ||
).return_value = None | ||
mocker.patch("dbt.tracking.active_user").return_value = User(None) | ||
|
||
def test_write_perf_info( | ||
self, | ||
manifest: Manifest, | ||
mock_project: MagicMock, | ||
mock_adapter: MagicMock, | ||
mocker: MockerFixture, | ||
) -> None: | ||
self.set_required_mocks(mocker, manifest, mock_adapter) | ||
write_perf_info = mocker.patch("dbt.parser.manifest.ManifestLoader.write_perf_info") | ||
|
||
ManifestLoader.get_full_manifest( | ||
config=mock_project, | ||
# write_perf_info=False let it default instead | ||
) | ||
assert not write_perf_info.called | ||
|
||
ManifestLoader.get_full_manifest(config=mock_project, write_perf_info=False) | ||
assert not write_perf_info.called | ||
|
||
ManifestLoader.get_full_manifest(config=mock_project, write_perf_info=True) | ||
assert write_perf_info.called | ||
|
||
def test_reset( | ||
self, | ||
manifest: Manifest, | ||
mock_project: MagicMock, | ||
mock_adapter: MagicMock, | ||
mocker: MockerFixture, | ||
) -> None: | ||
self.set_required_mocks(mocker, manifest, mock_adapter) | ||
|
||
ManifestLoader.get_full_manifest( | ||
config=mock_project, | ||
# reset=False let it default instead | ||
) | ||
assert not mock_project.clear_dependencies.called | ||
assert not mock_adapter.clear_macro_resolver.called | ||
|
||
ManifestLoader.get_full_manifest(config=mock_project, reset=False) | ||
assert not mock_project.clear_dependencies.called | ||
assert not mock_adapter.clear_macro_resolver.called | ||
|
||
ManifestLoader.get_full_manifest(config=mock_project, reset=True) | ||
assert mock_project.clear_dependencies.called | ||
assert mock_adapter.clear_macro_resolver.called | ||
|
||
def test_partial_parse_file_diff_flag( | ||
self, | ||
manifest: Manifest, | ||
mock_project: MagicMock, | ||
mock_adapter: MagicMock, | ||
mocker: MockerFixture, | ||
) -> None: | ||
self.set_required_mocks(mocker, manifest, mock_adapter) | ||
|
||
# FileDiff.from_dict is only called if PARTIAL_PARSE_FILE_DIFF == False | ||
# So we can track this function call to check if setting PARTIAL_PARSE_FILE_DIFF | ||
# works appropriately | ||
mock_file_diff = mocker.patch("dbt.parser.read_files.FileDiff.from_dict") | ||
mock_file_diff.return_value = FileDiff([], [], []) | ||
|
||
set_from_args(Namespace(), {}) | ||
ManifestLoader.get_full_manifest(config=mock_project) | ||
assert not mock_file_diff.called | ||
|
||
set_from_args(Namespace(PARTIAL_PARSE_FILE_DIFF=True), {}) | ||
ManifestLoader.get_full_manifest(config=mock_project) | ||
assert not mock_file_diff.called | ||
|
||
set_from_args(Namespace(PARTIAL_PARSE_FILE_DIFF=False), {}) | ||
ManifestLoader.get_full_manifest(config=mock_project) | ||
assert mock_file_diff.called |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Shoude this live somewhere in the utils being used by other unittests?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We can lift these 👍