From a0e36bfeea1f2730ec8c91ccb8e39015dc1b8ae0 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Fri, 12 Jan 2024 10:42:26 -0500 Subject: [PATCH] Revert "replace dbt/common with dbt-common" --- .../Dependencies-20240109-120530.yaml | 6 - .../Under the Hood-20240110-105734.yaml | 6 - core/dbt/README.md | 2 + core/dbt/adapters/base/column.py | 2 +- core/dbt/adapters/base/connections.py | 28 +- core/dbt/adapters/base/impl.py | 12 +- core/dbt/adapters/base/meta.py | 2 +- core/dbt/adapters/base/query_headers.py | 2 +- core/dbt/adapters/base/relation.py | 8 +- core/dbt/adapters/cache.py | 5 +- core/dbt/adapters/clients/jinja.py | 2 +- core/dbt/adapters/contracts/connection.py | 12 +- core/dbt/adapters/contracts/macros.py | 2 +- core/dbt/adapters/contracts/relation.py | 8 +- core/dbt/adapters/events/base_types.py | 2 +- core/dbt/adapters/events/logging.py | 8 +- core/dbt/adapters/events/types.py | 2 +- core/dbt/adapters/exceptions/alias.py | 2 +- core/dbt/adapters/exceptions/cache.py | 2 +- core/dbt/adapters/exceptions/compilation.py | 4 +- core/dbt/adapters/exceptions/connection.py | 2 +- core/dbt/adapters/exceptions/database.py | 2 +- core/dbt/adapters/factory.py | 6 +- core/dbt/adapters/protocol.py | 4 +- .../adapters/relation_configs/config_base.py | 2 +- .../relation_configs/config_change.py | 2 +- .../relation_configs/config_validation.py | 2 +- core/dbt/adapters/sql/connections.py | 22 +- core/dbt/adapters/sql/impl.py | 2 +- core/dbt/artifacts/base.py | 8 +- core/dbt/artifacts/catalog.py | 6 +- core/dbt/artifacts/freshness.py | 4 +- core/dbt/artifacts/results.py | 21 +- core/dbt/artifacts/run.py | 2 +- core/dbt/cli/flags.py | 10 +- core/dbt/cli/main.py | 3 +- core/dbt/cli/option_types.py | 4 +- core/dbt/cli/params.py | 6 + core/dbt/cli/requires.py | 12 +- core/dbt/cli/types.py | 2 +- core/dbt/clients/git.py | 4 +- core/dbt/clients/jinja.py | 4 +- core/dbt/clients/jinja_static.py | 4 +- core/dbt/clients/registry.py | 6 +- core/dbt/clients/yaml_helper.py | 6 +- core/dbt/common/__init__.py | 0 core/dbt/common/clients/__init__.py | 0 core/dbt/common/clients/_jinja_blocks.py | 360 +++++++++++ core/dbt/common/clients/agate_helper.py | 251 ++++++++ core/dbt/common/clients/jinja.py | 505 ++++++++++++++++ core/dbt/common/clients/system.py | 571 ++++++++++++++++++ core/dbt/common/constants.py | 1 + core/dbt/common/contracts/__init__.py | 0 core/dbt/common/contracts/config/__init__.py | 0 core/dbt/common/contracts/config/base.py | 259 ++++++++ .../contracts/config/materialization.py | 11 + core/dbt/common/contracts/config/metadata.py | 69 +++ .../dbt/common/contracts/config/properties.py | 63 ++ core/dbt/common/contracts/connection.py | 0 core/dbt/common/contracts/constraints.py | 43 ++ core/dbt/common/contracts/util.py | 7 + core/dbt/common/dataclass_schema.py | 165 +++++ core/dbt/common/events/README.md | 41 ++ core/dbt/common/events/__init__.py | 9 + core/dbt/common/events/base_types.py | 185 ++++++ core/dbt/common/events/contextvars.py | 114 ++++ core/dbt/common/events/event_handler.py | 40 ++ core/dbt/common/events/event_manager.py | 66 ++ .../dbt/common/events/event_manager_client.py | 29 + core/dbt/common/events/format.py | 56 ++ core/dbt/common/events/functions.py | 162 +++++ core/dbt/common/events/helpers.py | 14 + core/dbt/common/events/interfaces.py | 7 + core/dbt/common/events/logger.py | 180 ++++++ core/dbt/common/events/types.proto | 121 ++++ core/dbt/common/events/types.py | 124 ++++ core/dbt/common/events/types_pb2.py | 69 +++ core/dbt/common/exceptions/__init__.py | 7 + core/dbt/common/exceptions/base.py | 275 +++++++++ core/dbt/common/exceptions/connection.py | 7 + core/dbt/common/exceptions/contracts.py | 17 + core/dbt/common/exceptions/events.py | 9 + core/dbt/common/exceptions/jinja.py | 85 +++ core/dbt/common/exceptions/macros.py | 110 ++++ core/dbt/common/exceptions/system.py | 50 ++ core/dbt/common/helper_types.py | 122 ++++ core/dbt/common/invocation.py | 12 + core/dbt/common/semver.py | 473 +++++++++++++++ core/dbt/common/ui.py | 68 +++ core/dbt/common/utils/__init__.py | 26 + core/dbt/common/utils/casting.py | 25 + core/dbt/common/utils/connection.py | 33 + core/dbt/common/utils/dict.py | 128 ++++ core/dbt/common/utils/encoding.py | 56 ++ core/dbt/common/utils/executor.py | 67 ++ core/dbt/common/utils/formatting.py | 8 + core/dbt/common/utils/jinja.py | 33 + core/dbt/compilation.py | 12 +- core/dbt/config/profile.py | 8 +- core/dbt/config/project.py | 10 +- core/dbt/config/renderer.py | 6 +- core/dbt/config/runtime.py | 6 +- core/dbt/config/selectors.py | 6 +- core/dbt/config/utils.py | 4 +- core/dbt/constants.py | 2 +- core/dbt/context/base.py | 6 +- core/dbt/context/context_config.py | 4 +- core/dbt/context/exceptions_jinja.py | 4 +- core/dbt/context/providers.py | 10 +- core/dbt/contracts/files.py | 2 +- core/dbt/contracts/graph/manifest.py | 27 +- core/dbt/contracts/graph/model_config.py | 8 +- core/dbt/contracts/graph/nodes.py | 12 +- core/dbt/contracts/graph/saved_queries.py | 2 +- .../contracts/graph/semantic_layer_common.py | 2 +- core/dbt/contracts/graph/semantic_manifest.py | 6 +- core/dbt/contracts/graph/semantic_models.py | 2 +- core/dbt/contracts/graph/unparsed.py | 18 +- core/dbt/contracts/project.py | 4 +- core/dbt/contracts/selection.py | 2 +- core/dbt/contracts/sql.py | 14 +- core/dbt/contracts/state.py | 2 +- core/dbt/contracts/util.py | 4 +- core/dbt/deprecations.py | 5 +- core/dbt/deps/base.py | 6 +- core/dbt/deps/git.py | 6 +- core/dbt/deps/local.py | 4 +- core/dbt/deps/registry.py | 4 +- core/dbt/deps/tarball.py | 6 +- core/dbt/events/__init__.py | 2 +- core/dbt/events/base_types.py | 2 +- core/dbt/events/logging.py | 10 +- core/dbt/events/types.py | 6 +- core/dbt/exceptions.py | 6 +- core/dbt/flags.py | 18 + core/dbt/graph/cli.py | 2 +- core/dbt/graph/graph.py | 2 +- core/dbt/graph/selector.py | 2 +- core/dbt/graph/selector_methods.py | 6 +- core/dbt/graph/selector_spec.py | 4 +- core/dbt/hooks.py | 2 +- core/dbt/internal_deprecations.py | 2 +- core/dbt/logger.py | 536 ++++++++++++++++ core/dbt/node_types.py | 2 +- core/dbt/parser/base.py | 2 +- core/dbt/parser/common.py | 4 +- core/dbt/parser/generic_test.py | 4 +- core/dbt/parser/generic_test_builders.py | 2 +- core/dbt/parser/hooks.py | 2 +- core/dbt/parser/macros.py | 4 +- core/dbt/parser/manifest.py | 75 +-- core/dbt/parser/models.py | 10 +- core/dbt/parser/partial.py | 4 +- core/dbt/parser/read_files.py | 6 +- core/dbt/parser/schema_generic_tests.py | 2 +- core/dbt/parser/schema_yaml_readers.py | 4 +- core/dbt/parser/schemas.py | 10 +- core/dbt/parser/search.py | 4 +- core/dbt/parser/snapshots.py | 2 +- core/dbt/parser/sources.py | 4 +- core/dbt/parser/sql.py | 2 +- core/dbt/plugins/contracts.py | 2 +- core/dbt/plugins/manager.py | 2 +- core/dbt/task/base.py | 34 +- core/dbt/task/build.py | 6 +- core/dbt/task/clean.py | 4 +- core/dbt/task/clone.py | 4 +- core/dbt/task/compile.py | 8 +- core/dbt/task/debug.py | 28 +- core/dbt/task/deps.py | 6 +- core/dbt/task/docs/generate.py | 14 +- core/dbt/task/freshness.py | 8 +- core/dbt/task/init.py | 10 +- core/dbt/task/list.py | 6 +- core/dbt/task/printer.py | 50 +- core/dbt/task/retry.py | 2 +- core/dbt/task/run.py | 91 +-- core/dbt/task/run_operation.py | 8 +- core/dbt/task/runnable.py | 92 +-- core/dbt/task/seed.py | 15 +- core/dbt/task/show.py | 8 +- core/dbt/task/snapshot.py | 8 +- core/dbt/task/sql.py | 12 +- core/dbt/task/test.py | 6 +- core/dbt/tests/fixtures/project.py | 9 +- core/dbt/tests/util.py | 14 +- core/dbt/tracking.py | 19 +- core/dbt/utils.py | 6 +- core/dbt/version.py | 4 +- core/setup.py | 15 +- dev-requirements.txt | 2 + .../dbt/adapters/postgres/connections.py | 10 +- .../postgres/dbt/adapters/postgres/impl.py | 8 +- .../dbt/adapters/postgres/relation.py | 2 +- .../postgres/relation_configs/index.py | 4 +- .../relation_configs/materialized_view.py | 2 +- scripts/collect-artifact-schema.py | 2 +- scripts/collect-dbt-contexts.py | 2 +- .../tests/adapter/hooks/test_model_hooks.py | 2 +- .../adapter/materialized_view/changes.py | 2 +- .../tests/adapter/utils/test_validate_sql.py | 2 +- tests/functional/assertions/test_runner.py | 2 +- tests/functional/compile/test_compile.py | 2 +- tests/functional/configs/test_configs.py | 2 +- .../test_custom_node_colors_configs.py | 2 +- .../functional/configs/test_disabled_model.py | 2 +- .../dependencies/test_local_dependency.py | 7 +- .../deprecations/test_deprecations.py | 10 +- .../docs/test_duplicate_docs_block.py | 4 +- tests/functional/docs/test_invalid_doc_ref.py | 4 +- .../docs/test_missing_docs_blocks.py | 4 +- tests/functional/docs/test_static.py | 2 +- tests/functional/events/events.py | 2 +- .../exposures/test_exposure_configs.py | 2 +- tests/functional/list/test_list.py | 3 + tests/functional/logging/test_logging.py | 2 +- tests/functional/macros/test_macros.py | 8 +- .../functional/metrics/test_metric_configs.py | 2 +- .../run_operations/test_run_operations.py | 2 +- .../saved_queries/test_saved_query_parsing.py | 2 +- .../test_semantic_model_parsing.py | 2 +- tests/functional/show/test_show.py | 2 +- .../functional/sources/test_source_configs.py | 2 +- .../sources/test_source_fresher_state.py | 2 +- tests/unit/common/__init__.py | 0 tests/unit/common/test_agate_helper.py | 227 +++++++ tests/unit/common/test_jinja.py | 425 +++++++++++++ tests/unit/common/test_model_config.py | 92 +++ tests/unit/common/test_system_client.py | 271 +++++++++ tests/unit/common/test_utils.py | 143 +++++ tests/unit/test_cli_flags.py | 4 +- tests/unit/test_config.py | 5 +- tests/unit/test_connection_retries.py | 59 ++ tests/unit/test_context.py | 8 +- tests/unit/test_contracts_graph_parsed.py | 2 +- tests/unit/test_contracts_project.py | 2 +- tests/unit/test_core_dbt_utils.py | 73 +++ tests/unit/test_deps.py | 12 +- tests/unit/test_event_handler.py | 40 ++ tests/unit/test_events.py | 14 +- tests/unit/test_functions.py | 10 +- tests/unit/test_graph.py | 1 + tests/unit/test_graph_selection.py | 4 +- tests/unit/test_graph_selector_methods.py | 11 +- tests/unit/test_helper_types.py | 59 ++ tests/unit/test_jinja.py | 2 +- tests/unit/test_manifest.py | 9 +- tests/unit/test_postgres_adapter.py | 4 +- tests/unit/test_proto_events.py | 6 +- .../test_registry_get_request_exception.py | 2 +- tests/unit/test_semver.py | 4 +- tests/unit/test_version.py | 2 +- tests/unit/utils.py | 4 +- third-party-stubs/colorama/__init__.pyi | 16 + third-party-stubs/isodate/__init__.pyi | 4 + third-party-stubs/logbook/__init__.pyi | 65 ++ third-party-stubs/logbook/__version__.pyi | 5 + third-party-stubs/logbook/_fallback.pyi | 40 ++ third-party-stubs/logbook/_termcolors.pyi | 13 + third-party-stubs/logbook/base.pyi | 184 ++++++ third-party-stubs/logbook/compat.pyi | 60 ++ third-party-stubs/logbook/concurrency.pyi | 51 ++ third-party-stubs/logbook/handlers.pyi | 412 +++++++++++++ third-party-stubs/logbook/helpers.pyi | 42 ++ third-party-stubs/logbook/more.pyi | 148 +++++ third-party-stubs/logbook/notifiers.pyi | 123 ++++ third-party-stubs/logbook/queues.pyi | 154 +++++ third-party-stubs/logbook/ticketing.pyi | 110 ++++ third-party-stubs/logbook/utils.pyi | 39 ++ 269 files changed, 9291 insertions(+), 624 deletions(-) delete mode 100644 .changes/unreleased/Dependencies-20240109-120530.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240110-105734.yaml create mode 100644 core/dbt/common/__init__.py create mode 100644 core/dbt/common/clients/__init__.py create mode 100644 core/dbt/common/clients/_jinja_blocks.py create mode 100644 core/dbt/common/clients/agate_helper.py create mode 100644 core/dbt/common/clients/jinja.py create mode 100644 core/dbt/common/clients/system.py create mode 100644 core/dbt/common/constants.py create mode 100644 core/dbt/common/contracts/__init__.py create mode 100644 core/dbt/common/contracts/config/__init__.py create mode 100644 core/dbt/common/contracts/config/base.py create mode 100644 core/dbt/common/contracts/config/materialization.py create mode 100644 core/dbt/common/contracts/config/metadata.py create mode 100644 core/dbt/common/contracts/config/properties.py create mode 100644 core/dbt/common/contracts/connection.py create mode 100644 core/dbt/common/contracts/constraints.py create mode 100644 core/dbt/common/contracts/util.py create mode 100644 core/dbt/common/dataclass_schema.py create mode 100644 core/dbt/common/events/README.md create mode 100644 core/dbt/common/events/__init__.py create mode 100644 core/dbt/common/events/base_types.py create mode 100644 core/dbt/common/events/contextvars.py create mode 100644 core/dbt/common/events/event_handler.py create mode 100644 core/dbt/common/events/event_manager.py create mode 100644 core/dbt/common/events/event_manager_client.py create mode 100644 core/dbt/common/events/format.py create mode 100644 core/dbt/common/events/functions.py create mode 100644 core/dbt/common/events/helpers.py create mode 100644 core/dbt/common/events/interfaces.py create mode 100644 core/dbt/common/events/logger.py create mode 100644 core/dbt/common/events/types.proto create mode 100644 core/dbt/common/events/types.py create mode 100644 core/dbt/common/events/types_pb2.py create mode 100644 core/dbt/common/exceptions/__init__.py create mode 100644 core/dbt/common/exceptions/base.py create mode 100644 core/dbt/common/exceptions/connection.py create mode 100644 core/dbt/common/exceptions/contracts.py create mode 100644 core/dbt/common/exceptions/events.py create mode 100644 core/dbt/common/exceptions/jinja.py create mode 100644 core/dbt/common/exceptions/macros.py create mode 100644 core/dbt/common/exceptions/system.py create mode 100644 core/dbt/common/helper_types.py create mode 100644 core/dbt/common/invocation.py create mode 100644 core/dbt/common/semver.py create mode 100644 core/dbt/common/ui.py create mode 100644 core/dbt/common/utils/__init__.py create mode 100644 core/dbt/common/utils/casting.py create mode 100644 core/dbt/common/utils/connection.py create mode 100644 core/dbt/common/utils/dict.py create mode 100644 core/dbt/common/utils/encoding.py create mode 100644 core/dbt/common/utils/executor.py create mode 100644 core/dbt/common/utils/formatting.py create mode 100644 core/dbt/common/utils/jinja.py create mode 100644 core/dbt/logger.py create mode 100644 tests/unit/common/__init__.py create mode 100644 tests/unit/common/test_agate_helper.py create mode 100644 tests/unit/common/test_jinja.py create mode 100644 tests/unit/common/test_model_config.py create mode 100644 tests/unit/common/test_system_client.py create mode 100644 tests/unit/common/test_utils.py create mode 100644 tests/unit/test_connection_retries.py create mode 100644 tests/unit/test_core_dbt_utils.py create mode 100644 tests/unit/test_event_handler.py create mode 100644 tests/unit/test_helper_types.py create mode 100644 third-party-stubs/colorama/__init__.pyi create mode 100644 third-party-stubs/isodate/__init__.pyi create mode 100644 third-party-stubs/logbook/__init__.pyi create mode 100644 third-party-stubs/logbook/__version__.pyi create mode 100644 third-party-stubs/logbook/_fallback.pyi create mode 100644 third-party-stubs/logbook/_termcolors.pyi create mode 100644 third-party-stubs/logbook/base.pyi create mode 100644 third-party-stubs/logbook/compat.pyi create mode 100644 third-party-stubs/logbook/concurrency.pyi create mode 100644 third-party-stubs/logbook/handlers.pyi create mode 100644 third-party-stubs/logbook/helpers.pyi create mode 100644 third-party-stubs/logbook/more.pyi create mode 100644 third-party-stubs/logbook/notifiers.pyi create mode 100644 third-party-stubs/logbook/queues.pyi create mode 100644 third-party-stubs/logbook/ticketing.pyi create mode 100644 third-party-stubs/logbook/utils.pyi diff --git a/.changes/unreleased/Dependencies-20240109-120530.yaml b/.changes/unreleased/Dependencies-20240109-120530.yaml deleted file mode 100644 index cde20b74760..00000000000 --- a/.changes/unreleased/Dependencies-20240109-120530.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Remove logbook dependency -time: 2024-01-09T12:05:30.176656-06:00 -custom: - Author: emmyoop - PR: "9353" diff --git a/.changes/unreleased/Under the Hood-20240110-105734.yaml b/.changes/unreleased/Under the Hood-20240110-105734.yaml deleted file mode 100644 index 5c8b26e550e..00000000000 --- a/.changes/unreleased/Under the Hood-20240110-105734.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add dbt-common as a dependency and remove dbt/common -time: 2024-01-10T10:57:34.054908-05:00 -custom: - Author: michelleark emmyoop - Issue: "9357" diff --git a/core/dbt/README.md b/core/dbt/README.md index 6b545ceb888..79123a95f47 100644 --- a/core/dbt/README.md +++ b/core/dbt/README.md @@ -22,6 +22,8 @@ ### links.py +### logger.py + ### main.py ### node_types.py diff --git a/core/dbt/adapters/base/column.py b/core/dbt/adapters/base/column.py index 72e5576d408..7d08780c4f4 100644 --- a/core/dbt/adapters/base/column.py +++ b/core/dbt/adapters/base/column.py @@ -2,7 +2,7 @@ import re from typing import Dict, ClassVar, Any, Optional -from dbt_common.exceptions import DbtRuntimeError +from dbt.common.exceptions import DbtRuntimeError @dataclass diff --git a/core/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py index cdb3f7467e1..f347876f62e 100644 --- a/core/dbt/adapters/base/connections.py +++ b/core/dbt/adapters/base/connections.py @@ -25,7 +25,7 @@ import agate import dbt.adapters.exceptions -import dbt_common.exceptions.base +import dbt.common.exceptions.base from dbt.adapters.contracts.connection import ( Connection, Identifier, @@ -38,7 +38,7 @@ MacroQueryStringSetter, ) from dbt.adapters.events.logging import AdapterLogger -from dbt_common.events.functions import fire_event +from dbt.common.events.functions import fire_event from dbt.adapters.events.types import ( NewConnection, ConnectionReused, @@ -49,8 +49,8 @@ Rollback, RollbackFailed, ) -from dbt_common.events.contextvars import get_node_info -from dbt_common.utils import cast_to_str +from dbt.common.events.contextvars import get_node_info +from dbt.common.utils import cast_to_str SleepTime = Union[int, float] # As taken by time.sleep. AdapterHandle = Any # Adapter connection handle objects can be any class. @@ -99,7 +99,7 @@ def get_thread_connection(self) -> Connection: def set_thread_connection(self, conn: Connection) -> None: key = self.get_thread_identifier() if key in self.thread_connections: - raise dbt_common.exceptions.DbtInternalError( + raise dbt.common.exceptions.DbtInternalError( "In set_thread_connection, existing connection exists for {}" ) self.thread_connections[key] = conn @@ -139,7 +139,7 @@ def exception_handler(self, sql: str) -> ContextManager: :return: A context manager that handles exceptions raised by the underlying database. """ - raise dbt_common.exceptions.base.NotImplementedError( + raise dbt.common.exceptions.base.NotImplementedError( "`exception_handler` is not implemented for this adapter!" ) @@ -275,7 +275,7 @@ def retry_connection( @abc.abstractmethod def cancel_open(self) -> Optional[List[str]]: """Cancel all open connections on the adapter. (passable)""" - raise dbt_common.exceptions.base.NotImplementedError( + raise dbt.common.exceptions.base.NotImplementedError( "`cancel_open` is not implemented for this adapter!" ) @@ -290,7 +290,7 @@ def open(cls, connection: Connection) -> Connection: This should be thread-safe, or hold the lock if necessary. The given connection should not be in either in_use or available. """ - raise dbt_common.exceptions.base.NotImplementedError( + raise dbt.common.exceptions.base.NotImplementedError( "`open` is not implemented for this adapter!" ) @@ -324,14 +324,14 @@ def cleanup_all(self) -> None: @abc.abstractmethod def begin(self) -> None: """Begin a transaction. (passable)""" - raise dbt_common.exceptions.base.NotImplementedError( + raise dbt.common.exceptions.base.NotImplementedError( "`begin` is not implemented for this adapter!" ) @abc.abstractmethod def commit(self) -> None: """Commit a transaction. (passable)""" - raise dbt_common.exceptions.base.NotImplementedError( + raise dbt.common.exceptions.base.NotImplementedError( "`commit` is not implemented for this adapter!" ) @@ -369,7 +369,7 @@ def _close_handle(cls, connection: Connection) -> None: def _rollback(cls, connection: Connection) -> None: """Roll back the given connection.""" if connection.transaction_open is False: - raise dbt_common.exceptions.DbtInternalError( + raise dbt.common.exceptions.DbtInternalError( f"Tried to rollback transaction on connection " f'"{connection.name}", but it does not have one open!' ) @@ -420,7 +420,7 @@ def execute( :return: A tuple of the query status and results (empty if fetch=False). :rtype: Tuple[AdapterResponse, agate.Table] """ - raise dbt_common.exceptions.base.NotImplementedError( + raise dbt.common.exceptions.base.NotImplementedError( "`execute` is not implemented for this adapter!" ) @@ -432,7 +432,7 @@ def add_select_query(self, sql: str) -> Tuple[Connection, Any]: See https://github.com/dbt-labs/dbt-core/issues/8396 for more information. """ - raise dbt_common.exceptions.base.NotImplementedError( + raise dbt.common.exceptions.base.NotImplementedError( "`add_select_query` is not implemented for this adapter!" ) @@ -440,6 +440,6 @@ def add_select_query(self, sql: str) -> Tuple[Connection, Any]: def data_type_code_to_name(cls, type_code: Union[int, str]) -> str: """Get the string representation of the data type from the type_code.""" # https://peps.python.org/pep-0249/#type-objects - raise dbt_common.exceptions.base.NotImplementedError( + raise dbt.common.exceptions.base.NotImplementedError( "`data_type_code_to_name` is not implemented for this adapter!" ) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 6849fca7c70..6aa6c587316 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -23,7 +23,7 @@ from multiprocessing.context import SpawnContext from dbt.adapters.capability import Capability, CapabilityDict -from dbt_common.contracts.constraints import ( +from dbt.common.contracts.constraints import ( ColumnLevelConstraint, ConstraintType, ModelLevelConstraint, @@ -44,7 +44,7 @@ QuoteConfigTypeError, ) -from dbt_common.exceptions import ( +from dbt.common.exceptions import ( NotImplementedError, DbtInternalError, DbtRuntimeError, @@ -58,15 +58,15 @@ AdapterConfig, MacroContextGeneratorCallable, ) -from dbt_common.clients.agate_helper import ( +from dbt.common.clients.agate_helper import ( empty_table, get_column_value_uncased, merge_tables, table_from_rows, Integer, ) -from dbt_common.clients.jinja import CallableMacroGenerator -from dbt_common.events.functions import fire_event, warn_or_error +from dbt.common.clients.jinja import CallableMacroGenerator +from dbt.common.events.functions import fire_event, warn_or_error from dbt.adapters.events.types import ( CacheMiss, ListRelations, @@ -76,7 +76,7 @@ ConstraintNotSupported, ConstraintNotEnforced, ) -from dbt_common.utils import filter_null_values, executor, cast_to_str, AttrDict +from dbt.common.utils import filter_null_values, executor, cast_to_str, AttrDict from dbt.adapters.contracts.relation import RelationConfig from dbt.adapters.base.connections import ( diff --git a/core/dbt/adapters/base/meta.py b/core/dbt/adapters/base/meta.py index bf887a407ab..12f318d0c18 100644 --- a/core/dbt/adapters/base/meta.py +++ b/core/dbt/adapters/base/meta.py @@ -1,7 +1,7 @@ import abc from functools import wraps from typing import Callable, Optional, Any, FrozenSet, Dict, Set -from dbt_common.events.functions import warn_or_error +from dbt.common.events.functions import warn_or_error from dbt.adapters.events.types import AdapterDeprecationWarning Decorator = Callable[[Any], Callable] diff --git a/core/dbt/adapters/base/query_headers.py b/core/dbt/adapters/base/query_headers.py index 8ab8088aa3c..b5f64d6214c 100644 --- a/core/dbt/adapters/base/query_headers.py +++ b/core/dbt/adapters/base/query_headers.py @@ -3,7 +3,7 @@ from dbt.adapters.clients.jinja import QueryStringGenerator from dbt.adapters.contracts.connection import AdapterRequiredConfig, QueryComment -from dbt_common.exceptions import DbtRuntimeError +from dbt.common.exceptions import DbtRuntimeError class QueryHeaderContextWrapper: diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index 060dcb0a3ef..d68b2ce3c25 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -12,10 +12,10 @@ Path, ) from dbt.adapters.exceptions import MultipleDatabasesNotAllowedError, ApproximateMatchError -from dbt_common.utils import filter_null_values, deep_merge +from dbt.common.utils import filter_null_values, deep_merge from dbt.adapters.utils import classproperty -import dbt_common.exceptions +import dbt.common.exceptions Self = TypeVar("Self", bound="BaseRelation") @@ -97,7 +97,7 @@ def matches( if not search: # nothing was passed in - raise dbt_common.exceptions.DbtRuntimeError( + raise dbt.common.exceptions.DbtRuntimeError( "Tried to match relation, but no search path was passed!" ) @@ -360,7 +360,7 @@ class InformationSchema(BaseRelation): def __post_init__(self): if not isinstance(self.information_schema_view, (type(None), str)): - raise dbt_common.exceptions.CompilationError( + raise dbt.common.exceptions.CompilationError( "Got an invalid name: {}".format(self.information_schema_view) ) diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index bc730aa158e..01a8dc70d21 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -7,7 +7,6 @@ _make_ref_key_dict, _ReferenceKey, ) - from dbt.adapters.exceptions.cache import ( NewNameAlreadyInCacheError, ReferencedLinkNotCachedError, @@ -15,9 +14,9 @@ TruncatedModelNameCausedCollisionError, NoneRelationFoundError, ) -from dbt_common.events.functions import fire_event, fire_event_if +from dbt.common.events.functions import fire_event, fire_event_if from dbt.adapters.events.types import CacheAction, CacheDumpGraph -from dbt_common.utils.formatting import lowercase +from dbt.common.utils.formatting import lowercase def dot_separated(key: _ReferenceKey) -> str: diff --git a/core/dbt/adapters/clients/jinja.py b/core/dbt/adapters/clients/jinja.py index c2b6edbbfa7..ace89c0d1d4 100644 --- a/core/dbt/adapters/clients/jinja.py +++ b/core/dbt/adapters/clients/jinja.py @@ -1,5 +1,5 @@ from typing import Dict, Any -from dbt_common.clients.jinja import BaseMacroGenerator, get_environment +from dbt.common.clients.jinja import BaseMacroGenerator, get_environment class QueryStringGenerator(BaseMacroGenerator): diff --git a/core/dbt/adapters/contracts/connection.py b/core/dbt/adapters/contracts/connection.py index 14e9f07e71d..0d6d36aa0cb 100644 --- a/core/dbt/adapters/contracts/connection.py +++ b/core/dbt/adapters/contracts/connection.py @@ -16,21 +16,21 @@ from mashumaro.jsonschema.annotations import Pattern from dbt.adapters.utils import translate_aliases -from dbt_common.exceptions import DbtInternalError -from dbt_common.dataclass_schema import ( +from dbt.common.exceptions import DbtInternalError +from dbt.common.dataclass_schema import ( dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, ValidatedStringMixin, ) -from dbt_common.contracts.util import Replaceable -from dbt_common.utils import md5 +from dbt.common.contracts.util import Replaceable +from dbt.common.utils import md5 -from dbt_common.events.functions import fire_event +from dbt.common.events.functions import fire_event from dbt.adapters.events.types import NewConnectionOpening # TODO: this is a very bad dependency - shared global state -from dbt_common.events.contextvars import get_node_info +from dbt.common.events.contextvars import get_node_info class Identifier(ValidatedStringMixin): diff --git a/core/dbt/adapters/contracts/macros.py b/core/dbt/adapters/contracts/macros.py index 6ffe58c1be2..151c9c44dde 100644 --- a/core/dbt/adapters/contracts/macros.py +++ b/core/dbt/adapters/contracts/macros.py @@ -1,7 +1,7 @@ from typing import Optional from typing_extensions import Protocol -from dbt_common.clients.jinja import MacroProtocol +from dbt.common.clients.jinja import MacroProtocol class MacroResolverProtocol(Protocol): diff --git a/core/dbt/adapters/contracts/relation.py b/core/dbt/adapters/contracts/relation.py index 8fcb8b43edc..aea294e922c 100644 --- a/core/dbt/adapters/contracts/relation.py +++ b/core/dbt/adapters/contracts/relation.py @@ -6,11 +6,11 @@ ) from typing_extensions import Protocol -from dbt_common.dataclass_schema import dbtClassMixin, StrEnum +from dbt.common.dataclass_schema import dbtClassMixin, StrEnum -from dbt_common.contracts.util import Replaceable -from dbt_common.exceptions import CompilationError, DataclassNotDictError -from dbt_common.utils import deep_merge +from dbt.common.contracts.util import Replaceable +from dbt.common.exceptions import CompilationError, DataclassNotDictError +from dbt.common.utils import deep_merge class RelationType(StrEnum): diff --git a/core/dbt/adapters/events/base_types.py b/core/dbt/adapters/events/base_types.py index 23de6ab2b73..3717fb44071 100644 --- a/core/dbt/adapters/events/base_types.py +++ b/core/dbt/adapters/events/base_types.py @@ -1,5 +1,5 @@ # Aliasing common Level classes in order to make custom, but not overly-verbose versions that have PROTO_TYPES_MODULE set to the adapter-specific generated types_pb2 module -from dbt_common.events.base_types import ( +from dbt.common.events.base_types import ( BaseEvent, DynamicLevel as CommonDyanicLevel, TestLevel as CommonTestLevel, diff --git a/core/dbt/adapters/events/logging.py b/core/dbt/adapters/events/logging.py index 93f9d15fce1..f85b3358520 100644 --- a/core/dbt/adapters/events/logging.py +++ b/core/dbt/adapters/events/logging.py @@ -7,10 +7,10 @@ AdapterEventWarning, AdapterEventError, ) -from dbt_common.events import get_event_manager -from dbt_common.events.contextvars import get_node_info -from dbt_common.events.event_handler import set_package_logging -from dbt_common.events.functions import fire_event +from dbt.common.events import get_event_manager +from dbt.common.events.contextvars import get_node_info +from dbt.common.events.event_handler import set_package_logging +from dbt.common.events.functions import fire_event @dataclass diff --git a/core/dbt/adapters/events/types.py b/core/dbt/adapters/events/types.py index 99fe1c1bf36..d3aa0a87214 100644 --- a/core/dbt/adapters/events/types.py +++ b/core/dbt/adapters/events/types.py @@ -1,5 +1,5 @@ from dbt.adapters.events.base_types import WarnLevel, InfoLevel, ErrorLevel, DebugLevel -from dbt_common.ui import line_wrap_message, warning_tag +from dbt.common.ui import line_wrap_message, warning_tag def format_adapter_message(name, base_msg, args) -> str: diff --git a/core/dbt/adapters/exceptions/alias.py b/core/dbt/adapters/exceptions/alias.py index 60426a4cb5d..68a677088d2 100644 --- a/core/dbt/adapters/exceptions/alias.py +++ b/core/dbt/adapters/exceptions/alias.py @@ -1,6 +1,6 @@ from typing import Mapping, Any -from dbt_common.exceptions import DbtValidationError +from dbt.common.exceptions import DbtValidationError class AliasError(DbtValidationError): diff --git a/core/dbt/adapters/exceptions/cache.py b/core/dbt/adapters/exceptions/cache.py index 9b51f26f9fb..d557be07d6c 100644 --- a/core/dbt/adapters/exceptions/cache.py +++ b/core/dbt/adapters/exceptions/cache.py @@ -1,7 +1,7 @@ import re from typing import Dict -from dbt_common.exceptions import DbtInternalError +from dbt.common.exceptions import DbtInternalError class CacheInconsistencyError(DbtInternalError): diff --git a/core/dbt/adapters/exceptions/compilation.py b/core/dbt/adapters/exceptions/compilation.py index a66610d54d5..c87e74b05e7 100644 --- a/core/dbt/adapters/exceptions/compilation.py +++ b/core/dbt/adapters/exceptions/compilation.py @@ -1,7 +1,7 @@ from typing import List, Mapping, Any -from dbt_common.exceptions import CompilationError, DbtDatabaseError -from dbt_common.ui import line_wrap_message +from dbt.common.exceptions import CompilationError, DbtDatabaseError +from dbt.common.ui import line_wrap_message class MissingConfigError(CompilationError): diff --git a/core/dbt/adapters/exceptions/connection.py b/core/dbt/adapters/exceptions/connection.py index 870794fbe8d..aac55166407 100644 --- a/core/dbt/adapters/exceptions/connection.py +++ b/core/dbt/adapters/exceptions/connection.py @@ -1,6 +1,6 @@ from typing import List -from dbt_common.exceptions import DbtRuntimeError, DbtDatabaseError +from dbt.common.exceptions import DbtRuntimeError, DbtDatabaseError class InvalidConnectionError(DbtRuntimeError): diff --git a/core/dbt/adapters/exceptions/database.py b/core/dbt/adapters/exceptions/database.py index 066016636d6..ff177289a03 100644 --- a/core/dbt/adapters/exceptions/database.py +++ b/core/dbt/adapters/exceptions/database.py @@ -1,6 +1,6 @@ from typing import Any -from dbt_common.exceptions import NotImplementedError, CompilationError +from dbt.common.exceptions import NotImplementedError, CompilationError class UnexpectedDbReferenceError(NotImplementedError): diff --git a/core/dbt/adapters/factory.py b/core/dbt/adapters/factory.py index 8124c5047d8..17023d8bf64 100644 --- a/core/dbt/adapters/factory.py +++ b/core/dbt/adapters/factory.py @@ -10,12 +10,12 @@ from dbt.adapters.base.plugin import AdapterPlugin from dbt.adapters.protocol import AdapterConfig, AdapterProtocol, RelationProtocol from dbt.adapters.contracts.connection import AdapterRequiredConfig, Credentials -from dbt_common.events.functions import fire_event +from dbt.common.events.functions import fire_event from dbt.adapters.events.types import AdapterImportError, PluginLoadError, AdapterRegistered -from dbt_common.exceptions import DbtInternalError, DbtRuntimeError +from dbt.common.exceptions import DbtInternalError, DbtRuntimeError from dbt.adapters.include.global_project import PACKAGE_PATH as GLOBAL_PROJECT_PATH from dbt.adapters.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME -from dbt_common.semver import VersionSpecifier +from dbt.common.semver import VersionSpecifier Adapter = AdapterProtocol diff --git a/core/dbt/adapters/protocol.py b/core/dbt/adapters/protocol.py index f2de7de50ac..2b0399acfc0 100644 --- a/core/dbt/adapters/protocol.py +++ b/core/dbt/adapters/protocol.py @@ -18,8 +18,8 @@ from dbt.adapters.contracts.connection import Connection, AdapterRequiredConfig, AdapterResponse from dbt.adapters.contracts.macros import MacroResolverProtocol from dbt.adapters.contracts.relation import Policy, HasQuoting, RelationConfig -from dbt_common.contracts.config.base import BaseConfig -from dbt_common.clients.jinja import MacroProtocol +from dbt.common.contracts.config.base import BaseConfig +from dbt.common.clients.jinja import MacroProtocol @dataclass diff --git a/core/dbt/adapters/relation_configs/config_base.py b/core/dbt/adapters/relation_configs/config_base.py index 57b791939bf..5bfaa8de233 100644 --- a/core/dbt/adapters/relation_configs/config_base.py +++ b/core/dbt/adapters/relation_configs/config_base.py @@ -2,7 +2,7 @@ from typing import Union, Dict import agate -from dbt_common.utils import filter_null_values +from dbt.common.utils import filter_null_values """ diff --git a/core/dbt/adapters/relation_configs/config_change.py b/core/dbt/adapters/relation_configs/config_change.py index 10a6781a24b..94f7dd7e9ad 100644 --- a/core/dbt/adapters/relation_configs/config_change.py +++ b/core/dbt/adapters/relation_configs/config_change.py @@ -3,7 +3,7 @@ from typing import Hashable from dbt.adapters.relation_configs.config_base import RelationConfigBase -from dbt_common.dataclass_schema import StrEnum +from dbt.common.dataclass_schema import StrEnum class RelationConfigChangeAction(StrEnum): diff --git a/core/dbt/adapters/relation_configs/config_validation.py b/core/dbt/adapters/relation_configs/config_validation.py index 9442a60a091..ef7b18bb7bb 100644 --- a/core/dbt/adapters/relation_configs/config_validation.py +++ b/core/dbt/adapters/relation_configs/config_validation.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Set, Optional -from dbt_common.exceptions import DbtRuntimeError +from dbt.common.exceptions import DbtRuntimeError @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/core/dbt/adapters/sql/connections.py b/core/dbt/adapters/sql/connections.py index 324a84ff020..86de2dfbddc 100644 --- a/core/dbt/adapters/sql/connections.py +++ b/core/dbt/adapters/sql/connections.py @@ -5,13 +5,13 @@ import agate from dbt.adapters.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus -import dbt_common.clients.agate_helper -import dbt_common.exceptions +import dbt.common.clients.agate_helper +import dbt.common.exceptions from dbt.adapters.base import BaseConnectionManager from dbt.adapters.contracts.connection import Connection, ConnectionState, AdapterResponse -from dbt_common.events.functions import fire_event -from dbt_common.events.contextvars import get_node_info -from dbt_common.utils import cast_to_str +from dbt.common.events.functions import fire_event +from dbt.common.events.contextvars import get_node_info +from dbt.common.utils import cast_to_str class SQLConnectionManager(BaseConnectionManager): @@ -27,7 +27,7 @@ class SQLConnectionManager(BaseConnectionManager): @abc.abstractmethod def cancel(self, connection: Connection): """Cancel the given connection.""" - raise dbt_common.exceptions.base.NotImplementedError( + raise dbt.common.exceptions.base.NotImplementedError( "`cancel` is not implemented for this adapter!" ) @@ -95,7 +95,7 @@ def add_query( @abc.abstractmethod def get_response(cls, cursor: Any) -> AdapterResponse: """Get the status of the cursor.""" - raise dbt_common.exceptions.base.NotImplementedError( + raise dbt.common.exceptions.base.NotImplementedError( "`get_response` is not implemented for this adapter!" ) @@ -131,7 +131,7 @@ def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> agate.Tabl rows = cursor.fetchall() data = cls.process_results(column_names, rows) - return dbt_common.clients.agate_helper.table_from_data_flat(data, column_names) + return dbt.common.clients.agate_helper.table_from_data_flat(data, column_names) def execute( self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None @@ -142,7 +142,7 @@ def execute( if fetch: table = self.get_result_from_cursor(cursor, limit) else: - table = dbt_common.clients.agate_helper.empty_table() + table = dbt.common.clients.agate_helper.empty_table() return response, table def add_begin_query(self): @@ -158,7 +158,7 @@ def add_select_query(self, sql: str) -> Tuple[Connection, Any]: def begin(self): connection = self.get_thread_connection() if connection.transaction_open is True: - raise dbt_common.exceptions.DbtInternalError( + raise dbt.common.exceptions.DbtInternalError( 'Tried to begin a new transaction on connection "{}", but ' "it already had one open!".format(connection.name) ) @@ -171,7 +171,7 @@ def begin(self): def commit(self): connection = self.get_thread_connection() if connection.transaction_open is False: - raise dbt_common.exceptions.DbtInternalError( + raise dbt.common.exceptions.DbtInternalError( 'Tried to commit transaction on connection "{}", but ' "it does not have one open!".format(connection.name) ) diff --git a/core/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py index e8cf8b45677..e8a712774de 100644 --- a/core/dbt/adapters/sql/impl.py +++ b/core/dbt/adapters/sql/impl.py @@ -7,7 +7,7 @@ from dbt.adapters.base import BaseAdapter, available from dbt.adapters.cache import _make_ref_key_dict from dbt.adapters.sql import SQLConnectionManager -from dbt_common.events.functions import fire_event +from dbt.common.events.functions import fire_event from dbt.adapters.base.relation import BaseRelation diff --git a/core/dbt/artifacts/base.py b/core/dbt/artifacts/base.py index ad94aa64e68..69806915c43 100644 --- a/core/dbt/artifacts/base.py +++ b/core/dbt/artifacts/base.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import ClassVar, Type, TypeVar, Dict, Any, Optional -from dbt_common.clients.system import write_json, read_json +from dbt.common.clients.system import write_json, read_json from dbt.exceptions import ( DbtInternalError, DbtRuntimeError, @@ -10,9 +10,9 @@ ) from dbt.version import __version__ -from dbt_common.events.functions import get_metadata_vars -from dbt_common.invocation import get_invocation_id -from dbt_common.dataclass_schema import dbtClassMixin +from dbt.common.events.functions import get_metadata_vars +from dbt.common.invocation import get_invocation_id +from dbt.common.dataclass_schema import dbtClassMixin from mashumaro.jsonschema import build_json_schema from mashumaro.jsonschema.dialects import DRAFT_2020_12 diff --git a/core/dbt/artifacts/catalog.py b/core/dbt/artifacts/catalog.py index b5307576e14..768992ddd1b 100644 --- a/core/dbt/artifacts/catalog.py +++ b/core/dbt/artifacts/catalog.py @@ -2,9 +2,9 @@ from dataclasses import dataclass, field from datetime import datetime -from dbt_common.dataclass_schema import dbtClassMixin -from dbt_common.utils.formatting import lowercase -from dbt_common.contracts.util import Replaceable +from dbt.common.dataclass_schema import dbtClassMixin +from dbt.common.utils.formatting import lowercase +from dbt.common.contracts.util import Replaceable from dbt.artifacts.base import ArtifactMixin, BaseArtifactMetadata, schema_version Primitive = Union[bool, str, float, None] diff --git a/core/dbt/artifacts/freshness.py b/core/dbt/artifacts/freshness.py index b77749bd8ac..ccef153c9ac 100644 --- a/core/dbt/artifacts/freshness.py +++ b/core/dbt/artifacts/freshness.py @@ -4,8 +4,8 @@ from dbt.artifacts.results import ExecutionResult, FreshnessStatus, NodeResult, TimingInfo from dbt.artifacts.base import ArtifactMixin, VersionedSchema, schema_version, BaseArtifactMetadata -from dbt_common.dataclass_schema import dbtClassMixin, StrEnum -from dbt_common.exceptions import DbtInternalError +from dbt.common.dataclass_schema import dbtClassMixin, StrEnum +from dbt.common.exceptions import DbtInternalError from dbt.contracts.graph.unparsed import FreshnessThreshold from dbt.contracts.graph.nodes import SourceDefinition diff --git a/core/dbt/artifacts/results.py b/core/dbt/artifacts/results.py index f1cad47fac3..8bd5bf1fe54 100644 --- a/core/dbt/artifacts/results.py +++ b/core/dbt/artifacts/results.py @@ -1,10 +1,11 @@ from dbt.contracts.graph.nodes import ResultNode -from dbt_common.events.functions import fire_event +from dbt.common.events.functions import fire_event from dbt.events.types import TimingInfoCollected -from dbt_common.events.contextvars import get_node_info -from dbt_common.events.helpers import datetime_to_json_string -from dbt_common.utils import cast_to_str, cast_to_int -from dbt_common.dataclass_schema import dbtClassMixin, StrEnum +from dbt.common.events.contextvars import get_node_info +from dbt.common.events.helpers import datetime_to_json_string +from dbt.logger import TimingProcessor +from dbt.common.utils import cast_to_str, cast_to_int +from dbt.common.dataclass_schema import dbtClassMixin, StrEnum from dataclasses import dataclass from datetime import datetime @@ -44,11 +45,13 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self.timing_info.end() self.callback(self.timing_info) - fire_event( - TimingInfoCollected( - timing_info=self.timing_info.to_msg_dict(), node_info=get_node_info() + # Note: when legacy logger is removed, we can remove the following line + with TimingProcessor(self.timing_info): + fire_event( + TimingInfoCollected( + timing_info=self.timing_info.to_msg_dict(), node_info=get_node_info() + ) ) - ) class RunningStatus(StrEnum): diff --git a/core/dbt/artifacts/run.py b/core/dbt/artifacts/run.py index d6479bb2009..c3e55248230 100644 --- a/core/dbt/artifacts/run.py +++ b/core/dbt/artifacts/run.py @@ -19,7 +19,7 @@ ResultNode, ExecutionResult, ) -from dbt_common.clients.system import write_json +from dbt.common.clients.system import write_json @dataclass diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 90ee795afe2..ffc73323df8 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -13,12 +13,12 @@ from dbt.cli.types import Command as CliCommand from dbt.config.project import read_project_flags from dbt.contracts.project import ProjectFlags -from dbt_common import ui -from dbt_common.events import functions -from dbt_common.exceptions import DbtInternalError -from dbt_common.clients import jinja +from dbt.common import ui +from dbt.common.events import functions +from dbt.common.exceptions import DbtInternalError +from dbt.common.clients import jinja from dbt.deprecations import renamed_env_var -from dbt_common.helper_types import WarnErrorOptions +from dbt.common.helper_types import WarnErrorOptions from dbt.events import ALL_EVENT_NAMES if os.name != "nt": diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 714d7fd0f58..3525de95f96 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -19,7 +19,7 @@ from dbt.contracts.graph.manifest import Manifest from dbt.artifacts.catalog import CatalogArtifact from dbt.artifacts.run import RunExecutionResult -from dbt_common.events.base_types import EventMsg +from dbt.common.events.base_types import EventMsg from dbt.task.build import BuildTask from dbt.task.clean import CleanTask from dbt.task.clone import CloneTask @@ -122,6 +122,7 @@ def global_flags(func): @p.cache_selected_only @p.debug @p.deprecated_print + @p.enable_legacy_logger @p.fail_fast @p.log_cache_events @p.log_file_max_bytes diff --git a/core/dbt/cli/option_types.py b/core/dbt/cli/option_types.py index 7dc725f6b52..f673ac279dc 100644 --- a/core/dbt/cli/option_types.py +++ b/core/dbt/cli/option_types.py @@ -3,9 +3,9 @@ from dbt.config.utils import parse_cli_yaml_string from dbt.events import ALL_EVENT_NAMES from dbt.exceptions import ValidationError, OptionNotYamlDictError -from dbt_common.exceptions import DbtValidationError +from dbt.common.exceptions import DbtValidationError -from dbt_common.helper_types import WarnErrorOptions +from dbt.common.helper_types import WarnErrorOptions class YAML(ParamType): diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index a15453e66a9..3e03376f890 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -90,6 +90,12 @@ is_flag=True, ) +enable_legacy_logger = click.option( + "--enable-legacy-logger/--no-enable-legacy-logger", + envvar="DBT_ENABLE_LEGACY_LOGGER", + hidden=True, +) + exclude = click.option( "--exclude", envvar=None, diff --git a/core/dbt/cli/requires.py b/core/dbt/cli/requires.py index 4667e583fa7..c00daddba9c 100644 --- a/core/dbt/cli/requires.py +++ b/core/dbt/cli/requires.py @@ -1,5 +1,5 @@ import dbt.tracking -from dbt_common.invocation import reset_invocation_id +from dbt.common.invocation import reset_invocation_id from dbt.version import installed as installed_version from dbt.adapters.factory import adapter_management from dbt.flags import set_flags, get_flag_dict @@ -11,8 +11,8 @@ from dbt.config import RuntimeConfig from dbt.config.runtime import load_project, load_profile, UnsetProfile -from dbt_common.events.base_types import EventLevel -from dbt_common.events.functions import ( +from dbt.common.events.base_types import EventLevel +from dbt.common.events.functions import ( fire_event, LOG_VERSION, ) @@ -22,14 +22,14 @@ MainReportArgs, MainTrackingUserState, ) -from dbt_common.events.helpers import get_json_string_utcnow +from dbt.common.events.helpers import get_json_string_utcnow from dbt.events.types import CommandCompleted, MainEncounteredError, MainStackTrace, ResourceReport -from dbt_common.exceptions import DbtBaseException as DbtException +from dbt.common.exceptions import DbtBaseException as DbtException from dbt.exceptions import DbtProjectError, FailFastError from dbt.parser.manifest import parse_manifest from dbt.profiler import profiler from dbt.tracking import active_user, initialize_from_flags, track_run -from dbt_common.utils import cast_dict_to_dict_of_strings +from dbt.common.utils import cast_dict_to_dict_of_strings from dbt.plugins import set_up_plugin_manager from click import Context diff --git a/core/dbt/cli/types.py b/core/dbt/cli/types.py index 1da078df902..f43314c873f 100644 --- a/core/dbt/cli/types.py +++ b/core/dbt/cli/types.py @@ -1,7 +1,7 @@ from enum import Enum from typing import List -from dbt_common.exceptions import DbtInternalError +from dbt.common.exceptions import DbtInternalError class Command(Enum): diff --git a/core/dbt/clients/git.py b/core/dbt/clients/git.py index 4da1c323327..2c033ae0c41 100644 --- a/core/dbt/clients/git.py +++ b/core/dbt/clients/git.py @@ -1,8 +1,8 @@ import re import os.path -from dbt_common.clients.system import run_cmd, rmdir -from dbt_common.events.functions import fire_event +from dbt.common.clients.system import run_cmd, rmdir +from dbt.common.events.functions import fire_event from dbt.events.types import ( GitSparseCheckoutSubdirectory, GitProgressCheckoutRevision, diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index 37253e0ea92..1c6e4ae4570 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -10,13 +10,13 @@ import jinja2.parser import jinja2.sandbox -from dbt_common.clients.jinja import ( +from dbt.common.clients.jinja import ( render_template, get_template, CallableMacroGenerator, MacroProtocol, ) -from dbt_common.utils import deep_map_render +from dbt.common.utils import deep_map_render from dbt.contracts.graph.nodes import GenericTestNode from dbt.exceptions import ( diff --git a/core/dbt/clients/jinja_static.py b/core/dbt/clients/jinja_static.py index a26710f01de..2112503d9f9 100644 --- a/core/dbt/clients/jinja_static.py +++ b/core/dbt/clients/jinja_static.py @@ -1,7 +1,7 @@ import jinja2 -from dbt_common.clients.jinja import get_environment +from dbt.common.clients.jinja import get_environment from dbt.exceptions import MacroNamespaceNotStringError -from dbt_common.exceptions.macros import MacroNameNotStringError +from dbt.common.exceptions.macros import MacroNameNotStringError def statically_extract_macro_calls(string, ctx, db_wrapper=None): diff --git a/core/dbt/clients/registry.py b/core/dbt/clients/registry.py index 3a160c7f791..b22375fd8f4 100644 --- a/core/dbt/clients/registry.py +++ b/core/dbt/clients/registry.py @@ -1,7 +1,7 @@ import functools from typing import Any, Dict, List import requests -from dbt_common.events.functions import fire_event +from dbt.common.events.functions import fire_event from dbt.events.types import ( RegistryProgressGETRequest, RegistryProgressGETResponse, @@ -13,9 +13,9 @@ RegistryResponseExtraNestedKeys, ) from dbt.utils import memoized -from dbt_common.utils.connection import connection_exception_retry +from dbt.common.utils.connection import connection_exception_retry from dbt import deprecations -from dbt_common import semver +from dbt.common import semver import os if os.getenv("DBT_PACKAGE_HUB_URL"): diff --git a/core/dbt/clients/yaml_helper.py b/core/dbt/clients/yaml_helper.py index 95f65e397ae..bfdf0ff189b 100644 --- a/core/dbt/clients/yaml_helper.py +++ b/core/dbt/clients/yaml_helper.py @@ -1,5 +1,5 @@ -import dbt_common.exceptions.base -import dbt_common.exceptions +import dbt.common.exceptions.base +import dbt.exceptions from typing import Any, Dict, Optional import yaml @@ -61,4 +61,4 @@ def load_yaml_text(contents, path=None): else: error = str(e) - raise dbt_common.exceptions.base.DbtValidationError(error) + raise dbt.common.exceptions.base.DbtValidationError(error) diff --git a/core/dbt/common/__init__.py b/core/dbt/common/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/common/clients/__init__.py b/core/dbt/common/clients/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/common/clients/_jinja_blocks.py b/core/dbt/common/clients/_jinja_blocks.py new file mode 100644 index 00000000000..9eb85df13f2 --- /dev/null +++ b/core/dbt/common/clients/_jinja_blocks.py @@ -0,0 +1,360 @@ +import re +from collections import namedtuple + +from dbt.common.exceptions import ( + BlockDefinitionNotAtTopError, + DbtInternalError, + MissingCloseTagError, + MissingControlFlowStartTagError, + NestedTagsError, + UnexpectedControlFlowEndTagError, + UnexpectedMacroEOFError, +) + + +def regex(pat): + return re.compile(pat, re.DOTALL | re.MULTILINE) + + +class BlockData: + """raw plaintext data from the top level of the file.""" + + def __init__(self, contents): + self.block_type_name = "__dbt__data" + self.contents = contents + self.full_block = contents + + +class BlockTag: + def __init__(self, block_type_name, block_name, contents=None, full_block=None, **kw): + self.block_type_name = block_type_name + self.block_name = block_name + self.contents = contents + self.full_block = full_block + + def __str__(self): + return "BlockTag({!r}, {!r})".format(self.block_type_name, self.block_name) + + def __repr__(self): + return str(self) + + @property + def end_block_type_name(self): + return "end{}".format(self.block_type_name) + + def end_pat(self): + # we don't want to use string formatting here because jinja uses most + # of the string formatting operators in its syntax... + pattern = "".join( + ( + r"(?P((?:\s*\{\%\-|\{\%)\s*", + self.end_block_type_name, + r"\s*(?:\-\%\}\s*|\%\})))", + ) + ) + return regex(pattern) + + +Tag = namedtuple("Tag", "block_type_name block_name start end") + + +_NAME_PATTERN = r"[A-Za-z_][A-Za-z_0-9]*" + +COMMENT_START_PATTERN = regex(r"(?:(?P(\s*\{\#)))") +COMMENT_END_PATTERN = regex(r"(.*?)(\s*\#\})") +RAW_START_PATTERN = regex(r"(?:\s*\{\%\-|\{\%)\s*(?P(raw))\s*(?:\-\%\}\s*|\%\})") +EXPR_START_PATTERN = regex(r"(?P(\{\{\s*))") +EXPR_END_PATTERN = regex(r"(?P(\s*\}\}))") + +BLOCK_START_PATTERN = regex( + "".join( + ( + r"(?:\s*\{\%\-|\{\%)\s*", + r"(?P({}))".format(_NAME_PATTERN), + # some blocks have a 'block name'. + r"(?:\s+(?P({})))?".format(_NAME_PATTERN), + ) + ) +) + + +RAW_BLOCK_PATTERN = regex( + "".join( + ( + r"(?:\s*\{\%\-|\{\%)\s*raw\s*(?:\-\%\}\s*|\%\})", + r"(?:.*?)", + r"(?:\s*\{\%\-|\{\%)\s*endraw\s*(?:\-\%\}\s*|\%\})", + ) + ) +) + +TAG_CLOSE_PATTERN = regex(r"(?:(?P(\-\%\}\s*|\%\})))") + +# stolen from jinja's lexer. Note that we've consumed all prefix whitespace by +# the time we want to use this. +STRING_PATTERN = regex(r"(?P('([^'\\]*(?:\\.[^'\\]*)*)'|" r'"([^"\\]*(?:\\.[^"\\]*)*)"))') + +QUOTE_START_PATTERN = regex(r"""(?P(['"]))""") + + +class TagIterator: + def __init__(self, data): + self.data = data + self.blocks = [] + self._parenthesis_stack = [] + self.pos = 0 + + def linepos(self, end=None) -> str: + """Given an absolute position in the input data, return a pair of + line number + relative position to the start of the line. + """ + end_val: int = self.pos if end is None else end + data = self.data[:end_val] + # if not found, rfind returns -1, and -1+1=0, which is perfect! + last_line_start = data.rfind("\n") + 1 + # it's easy to forget this, but line numbers are 1-indexed + line_number = data.count("\n") + 1 + return f"{line_number}:{end_val - last_line_start}" + + def advance(self, new_position): + self.pos = new_position + + def rewind(self, amount=1): + self.pos -= amount + + def _search(self, pattern): + return pattern.search(self.data, self.pos) + + def _match(self, pattern): + return pattern.match(self.data, self.pos) + + def _first_match(self, *patterns, **kwargs): + matches = [] + for pattern in patterns: + # default to 'search', but sometimes we want to 'match'. + if kwargs.get("method", "search") == "search": + match = self._search(pattern) + else: + match = self._match(pattern) + if match: + matches.append(match) + if not matches: + return None + # if there are multiple matches, pick the least greedy match + # TODO: do I need to account for m.start(), or is this ok? + return min(matches, key=lambda m: m.end()) + + def _expect_match(self, expected_name, *patterns, **kwargs): + match = self._first_match(*patterns, **kwargs) + if match is None: + raise UnexpectedMacroEOFError(expected_name, self.data[self.pos :]) + return match + + def handle_expr(self, match): + """Handle an expression. At this point we're at a string like: + {{ 1 + 2 }} + ^ right here + + And the match contains "{{ " + + We expect to find a `}}`, but we might find one in a string before + that. Imagine the case of `{{ 2 * "}}" }}`... + + You're not allowed to have blocks or comments inside an expr so it is + pretty straightforward, I hope: only strings can get in the way. + """ + self.advance(match.end()) + while True: + match = self._expect_match("}}", EXPR_END_PATTERN, QUOTE_START_PATTERN) + if match.groupdict().get("expr_end") is not None: + break + else: + # it's a quote. we haven't advanced for this match yet, so + # just slurp up the whole string, no need to rewind. + match = self._expect_match("string", STRING_PATTERN) + self.advance(match.end()) + + self.advance(match.end()) + + def handle_comment(self, match): + self.advance(match.end()) + match = self._expect_match("#}", COMMENT_END_PATTERN) + self.advance(match.end()) + + def _expect_block_close(self): + """Search for the tag close marker. + To the right of the type name, there are a few possiblities: + - a name (handled by the regex's 'block_name') + - any number of: `=`, `(`, `)`, strings, etc (arguments) + - nothing + + followed eventually by a %} + + So the only characters we actually have to worry about in this context + are quote and `%}` - nothing else can hide the %} and be valid jinja. + """ + while True: + end_match = self._expect_match( + 'tag close ("%}")', QUOTE_START_PATTERN, TAG_CLOSE_PATTERN + ) + self.advance(end_match.end()) + if end_match.groupdict().get("tag_close") is not None: + return + # must be a string. Rewind to its start and advance past it. + self.rewind() + string_match = self._expect_match("string", STRING_PATTERN) + self.advance(string_match.end()) + + def handle_raw(self): + # raw blocks are super special, they are a single complete regex + match = self._expect_match("{% raw %}...{% endraw %}", RAW_BLOCK_PATTERN) + self.advance(match.end()) + return match.end() + + def handle_tag(self, match): + """The tag could be one of a few things: + + {% mytag %} + {% mytag x = y %} + {% mytag x = "y" %} + {% mytag x.y() %} + {% mytag foo("a", "b", c="d") %} + + But the key here is that it's always going to be `{% mytag`! + """ + groups = match.groupdict() + # always a value + block_type_name = groups["block_type_name"] + # might be None + block_name = groups.get("block_name") + start_pos = self.pos + if block_type_name == "raw": + match = self._expect_match("{% raw %}...{% endraw %}", RAW_BLOCK_PATTERN) + self.advance(match.end()) + else: + self.advance(match.end()) + self._expect_block_close() + return Tag( + block_type_name=block_type_name, block_name=block_name, start=start_pos, end=self.pos + ) + + def find_tags(self): + while True: + match = self._first_match( + BLOCK_START_PATTERN, COMMENT_START_PATTERN, EXPR_START_PATTERN + ) + if match is None: + break + + self.advance(match.start()) + # start = self.pos + + groups = match.groupdict() + comment_start = groups.get("comment_start") + expr_start = groups.get("expr_start") + block_type_name = groups.get("block_type_name") + + if comment_start is not None: + self.handle_comment(match) + elif expr_start is not None: + self.handle_expr(match) + elif block_type_name is not None: + yield self.handle_tag(match) + else: + raise DbtInternalError( + "Invalid regex match in next_block, expected block start, " + "expr start, or comment start" + ) + + def __iter__(self): + return self.find_tags() + + +_CONTROL_FLOW_TAGS = { + "if": "endif", + "for": "endfor", +} + +_CONTROL_FLOW_END_TAGS = {v: k for k, v in _CONTROL_FLOW_TAGS.items()} + + +class BlockIterator: + def __init__(self, data): + self.tag_parser = TagIterator(data) + self.current = None + self.stack = [] + self.last_position = 0 + + @property + def current_end(self): + if self.current is None: + return 0 + else: + return self.current.end + + @property + def data(self): + return self.tag_parser.data + + def is_current_end(self, tag): + return ( + tag.block_type_name.startswith("end") + and self.current is not None + and tag.block_type_name[3:] == self.current.block_type_name + ) + + def find_blocks(self, allowed_blocks=None, collect_raw_data=True): + """Find all top-level blocks in the data.""" + if allowed_blocks is None: + allowed_blocks = {"snapshot", "macro", "materialization", "docs"} + + for tag in self.tag_parser.find_tags(): + if tag.block_type_name in _CONTROL_FLOW_TAGS: + self.stack.append(tag.block_type_name) + elif tag.block_type_name in _CONTROL_FLOW_END_TAGS: + found = None + if self.stack: + found = self.stack.pop() + else: + expected = _CONTROL_FLOW_END_TAGS[tag.block_type_name] + raise UnexpectedControlFlowEndTagError(tag, expected, self.tag_parser) + expected = _CONTROL_FLOW_TAGS[found] + if expected != tag.block_type_name: + raise MissingControlFlowStartTagError(tag, expected, self.tag_parser) + + if tag.block_type_name in allowed_blocks: + if self.stack: + raise BlockDefinitionNotAtTopError(self.tag_parser, tag.start) + if self.current is not None: + raise NestedTagsError(outer=self.current, inner=tag) + if collect_raw_data: + raw_data = self.data[self.last_position : tag.start] + self.last_position = tag.start + if raw_data: + yield BlockData(raw_data) + self.current = tag + + elif self.is_current_end(tag): + self.last_position = tag.end + assert self.current is not None + yield BlockTag( + block_type_name=self.current.block_type_name, + block_name=self.current.block_name, + contents=self.data[self.current.end : tag.start], + full_block=self.data[self.current.start : tag.end], + ) + self.current = None + + if self.current: + linecount = self.data[: self.current.end].count("\n") + 1 + raise MissingCloseTagError(self.current.block_type_name, linecount) + + if collect_raw_data: + raw_data = self.data[self.last_position :] + if raw_data: + yield BlockData(raw_data) + + def lex_for_blocks(self, allowed_blocks=None, collect_raw_data=True): + return list( + self.find_blocks(allowed_blocks=allowed_blocks, collect_raw_data=collect_raw_data) + ) diff --git a/core/dbt/common/clients/agate_helper.py b/core/dbt/common/clients/agate_helper.py new file mode 100644 index 00000000000..d7ac0916cdb --- /dev/null +++ b/core/dbt/common/clients/agate_helper.py @@ -0,0 +1,251 @@ +from codecs import BOM_UTF8 + +import agate +import datetime +import isodate +import json +from typing import Iterable, List, Dict, Union, Optional, Any + +from dbt.common.exceptions import DbtRuntimeError +from dbt.common.utils import ForgivingJSONEncoder + +BOM = BOM_UTF8.decode("utf-8") # '\ufeff' + + +class Integer(agate.data_types.DataType): + def cast(self, d): + # by default agate will cast none as a Number + # but we need to cast it as an Integer to preserve + # the type when merging and unioning tables + if type(d) == int or d is None: + return d + else: + raise agate.exceptions.CastError('Can not parse value "%s" as Integer.' % d) + + def jsonify(self, d): + return d + + +class Number(agate.data_types.Number): + # undo the change in https://github.com/wireservice/agate/pull/733 + # i.e. do not cast True and False to numeric 1 and 0 + def cast(self, d): + if type(d) == bool: + raise agate.exceptions.CastError("Do not cast True to 1 or False to 0.") + else: + return super().cast(d) + + +class ISODateTime(agate.data_types.DateTime): + def cast(self, d): + # this is agate.data_types.DateTime.cast with the "clever" bits removed + # so we only handle ISO8601 stuff + if isinstance(d, datetime.datetime) or d is None: + return d + elif isinstance(d, datetime.date): + return datetime.datetime.combine(d, datetime.time(0, 0, 0)) + elif isinstance(d, str): + d = d.strip() + if d.lower() in self.null_values: + return None + try: + return isodate.parse_datetime(d) + except: # noqa + pass + + raise agate.exceptions.CastError('Can not parse value "%s" as datetime.' % d) + + +def build_type_tester( + text_columns: Iterable[str], string_null_values: Optional[Iterable[str]] = ("null", "") +) -> agate.TypeTester: + + types = [ + Integer(null_values=("null", "")), + Number(null_values=("null", "")), + agate.data_types.Date(null_values=("null", ""), date_format="%Y-%m-%d"), + agate.data_types.DateTime(null_values=("null", ""), datetime_format="%Y-%m-%d %H:%M:%S"), + ISODateTime(null_values=("null", "")), + agate.data_types.Boolean( + true_values=("true",), false_values=("false",), null_values=("null", "") + ), + agate.data_types.Text(null_values=string_null_values), + ] + force = {k: agate.data_types.Text(null_values=string_null_values) for k in text_columns} + return agate.TypeTester(force=force, types=types) + + +DEFAULT_TYPE_TESTER = build_type_tester(()) + + +def table_from_rows( + rows: List[Any], + column_names: Iterable[str], + text_only_columns: Optional[Iterable[str]] = None, +) -> agate.Table: + if text_only_columns is None: + column_types = DEFAULT_TYPE_TESTER + else: + # If text_only_columns are present, prevent coercing empty string or + # literal 'null' strings to a None representation. + column_types = build_type_tester(text_only_columns, string_null_values=()) + + return agate.Table(rows, column_names, column_types=column_types) + + +def table_from_data(data, column_names: Iterable[str]) -> agate.Table: + "Convert a list of dictionaries into an Agate table" + + # The agate table is generated from a list of dicts, so the column order + # from `data` is not preserved. We can use `select` to reorder the columns + # + # If there is no data, create an empty table with the specified columns + + if len(data) == 0: + return agate.Table([], column_names=column_names) + else: + table = agate.Table.from_object(data, column_types=DEFAULT_TYPE_TESTER) + return table.select(column_names) + + +def table_from_data_flat(data, column_names: Iterable[str]) -> agate.Table: + """ + Convert a list of dictionaries into an Agate table. This method does not + coerce string values into more specific types (eg. '005' will not be + coerced to '5'). Additionally, this method does not coerce values to + None (eg. '' or 'null' will retain their string literal representations). + """ + + rows = [] + text_only_columns = set() + for _row in data: + row = [] + for col_name in column_names: + value = _row[col_name] + if isinstance(value, (dict, list, tuple)): + # Represent container types as json strings + value = json.dumps(value, cls=ForgivingJSONEncoder) + text_only_columns.add(col_name) + elif isinstance(value, str): + text_only_columns.add(col_name) + row.append(value) + + rows.append(row) + + return table_from_rows( + rows=rows, column_names=column_names, text_only_columns=text_only_columns + ) + + +def empty_table(): + "Returns an empty Agate table. To be used in place of None" + + return agate.Table(rows=[]) + + +def as_matrix(table): + "Return an agate table as a matrix of data sans columns" + + return [r.values() for r in table.rows.values()] + + +def from_csv(abspath, text_columns, delimiter=","): + type_tester = build_type_tester(text_columns=text_columns) + with open(abspath, encoding="utf-8") as fp: + if fp.read(1) != BOM: + fp.seek(0) + return agate.Table.from_csv(fp, column_types=type_tester, delimiter=delimiter) + + +class _NullMarker: + pass + + +NullableAgateType = Union[agate.data_types.DataType, _NullMarker] + + +class ColumnTypeBuilder(Dict[str, NullableAgateType]): + def __init__(self) -> None: + super().__init__() + + def __setitem__(self, key, value): + if key not in self: + super().__setitem__(key, value) + return + + existing_type = self[key] + if isinstance(existing_type, _NullMarker): + # overwrite + super().__setitem__(key, value) + elif isinstance(value, _NullMarker): + # use the existing value + return + # when one table column is Number while another is Integer, force the column to Number on merge + elif isinstance(value, Integer) and isinstance(existing_type, agate.data_types.Number): + # use the existing value + return + elif isinstance(existing_type, Integer) and isinstance(value, agate.data_types.Number): + # overwrite + super().__setitem__(key, value) + elif not isinstance(value, type(existing_type)): + # actual type mismatch! + raise DbtRuntimeError( + f"Tables contain columns with the same names ({key}), " + f"but different types ({value} vs {existing_type})" + ) + + def finalize(self) -> Dict[str, agate.data_types.DataType]: + result: Dict[str, agate.data_types.DataType] = {} + for key, value in self.items(): + if isinstance(value, _NullMarker): + # agate would make it a Number but we'll make it Integer so that if this column + # gets merged with another Integer column, it won't get forced to a Number + result[key] = Integer() + else: + result[key] = value + return result + + +def _merged_column_types(tables: List[agate.Table]) -> Dict[str, agate.data_types.DataType]: + # this is a lot like agate.Table.merge, but with handling for all-null + # rows being "any type". + new_columns: ColumnTypeBuilder = ColumnTypeBuilder() + for table in tables: + for i in range(len(table.columns)): + column_name: str = table.column_names[i] + column_type: NullableAgateType = table.column_types[i] + # avoid over-sensitive type inference + if all(x is None for x in table.columns[column_name]): + column_type = _NullMarker() + new_columns[column_name] = column_type + + return new_columns.finalize() + + +def merge_tables(tables: List[agate.Table]) -> agate.Table: + """This is similar to agate.Table.merge, but it handles rows of all 'null' + values more gracefully during merges. + """ + new_columns = _merged_column_types(tables) + column_names = tuple(new_columns.keys()) + column_types = tuple(new_columns.values()) + + rows: List[agate.Row] = [] + for table in tables: + if table.column_names == column_names and table.column_types == column_types: + rows.extend(table.rows) + else: + for row in table.rows: + data = [row.get(name, None) for name in column_names] + rows.append(agate.Row(data, column_names)) + # _is_fork to tell agate that we already made things into `Row`s. + return agate.Table(rows, column_names, column_types, _is_fork=True) + + +def get_column_value_uncased(column_name: str, row: agate.Row) -> Any: + """Get the value of a column in this row, ignoring the casing of the column name.""" + for key, value in row.items(): + if key.casefold() == column_name.casefold(): + return value + + raise KeyError diff --git a/core/dbt/common/clients/jinja.py b/core/dbt/common/clients/jinja.py new file mode 100644 index 00000000000..e01ad57093c --- /dev/null +++ b/core/dbt/common/clients/jinja.py @@ -0,0 +1,505 @@ +import codecs +import linecache +import os +import tempfile +from ast import literal_eval +from contextlib import contextmanager +from itertools import chain, islice +from typing import List, Union, Set, Optional, Dict, Any, Iterator, Type, Callable +from typing_extensions import Protocol + +import jinja2 +import jinja2.ext +import jinja2.nativetypes # type: ignore +import jinja2.nodes +import jinja2.parser +import jinja2.sandbox + +from dbt.common.utils import ( + get_dbt_macro_name, + get_docs_macro_name, + get_materialization_macro_name, + get_test_macro_name, +) +from dbt.common.clients._jinja_blocks import BlockIterator, BlockData, BlockTag + +from dbt.common.exceptions import ( + CompilationError, + DbtInternalError, + CaughtMacroErrorWithNodeError, + MaterializationArgError, + JinjaRenderingError, + UndefinedCompilationError, +) +from dbt.common.exceptions.macros import MacroReturn, UndefinedMacroError, CaughtMacroError + + +SUPPORTED_LANG_ARG = jinja2.nodes.Name("supported_languages", "param") + +# Global which can be set by dependents of dbt-common (e.g. core via flag parsing) +MACRO_DEBUGGING = False + + +def _linecache_inject(source, write): + if write: + # this is the only reliable way to accomplish this. Obviously, it's + # really darn noisy and will fill your temporary directory + tmp_file = tempfile.NamedTemporaryFile( + prefix="dbt-macro-compiled-", + suffix=".py", + delete=False, + mode="w+", + encoding="utf-8", + ) + tmp_file.write(source) + filename = tmp_file.name + else: + # `codecs.encode` actually takes a `bytes` as the first argument if + # the second argument is 'hex' - mypy does not know this. + rnd = codecs.encode(os.urandom(12), "hex") # type: ignore + filename = rnd.decode("ascii") + + # put ourselves in the cache + cache_entry = (len(source), None, [line + "\n" for line in source.splitlines()], filename) + # linecache does in fact have an attribute `cache`, thanks + linecache.cache[filename] = cache_entry # type: ignore + return filename + + +class MacroFuzzParser(jinja2.parser.Parser): + def parse_macro(self): + node = jinja2.nodes.Macro(lineno=next(self.stream).lineno) + + # modified to fuzz macros defined in the same file. this way + # dbt can understand the stack of macros being called. + # - @cmcarthur + node.name = get_dbt_macro_name(self.parse_assign_target(name_only=True).name) + + self.parse_signature(node) + node.body = self.parse_statements(("name:endmacro",), drop_needle=True) + return node + + +class MacroFuzzEnvironment(jinja2.sandbox.SandboxedEnvironment): + def _parse(self, source, name, filename): + return MacroFuzzParser(self, source, name, filename).parse() + + def _compile(self, source, filename): + """Override jinja's compilation to stash the rendered source inside + the python linecache for debugging when the appropriate environment + variable is set. + + If the value is 'write', also write the files to disk. + WARNING: This can write a ton of data if you aren't careful. + """ + if filename == "