From 083597a803db83aa4b950925527aac85ba1b1b86 Mon Sep 17 00:00:00 2001 From: slothkong Date: Sat, 30 Mar 2024 00:15:02 +0100 Subject: [PATCH 1/3] [#9791] Fix datetime.datetime.utcnow() is deprecated as of Python 3.12 --- .../unreleased/Fixes-20240329-223653.yaml | 6 + core/dbt/artifacts/schemas/base.py | 2 +- core/dbt/artifacts/schemas/results.py | 6 +- core/dbt/artifacts/schemas/run/v5/run.py | 2 +- core/dbt/contracts/graph/node_args.py | 2 +- core/dbt/contracts/sql.py | 6 +- core/dbt/graph/selector_methods.py | 158 +++++++++--------- core/dbt/logger.py | 4 +- core/dbt/parser/manifest.py | 21 ++- core/dbt/task/base.py | 5 +- core/dbt/task/docs/generate.py | 6 +- core/dbt/task/run.py | 9 +- core/dbt/task/run_operation.py | 6 +- core/dbt/task/runnable.py | 13 +- core/dbt/task/sql.py | 10 +- core/dbt/tests/fixtures/project.py | 6 +- core/dbt/tests/util.py | 4 +- core/dbt/utils.py | 6 +- .../adapter/basic/test_docs_generate.py | 4 +- .../adapter/utils/test_current_timestamp.py | 6 +- tests/functional/artifacts/test_artifacts.py | 8 +- tests/functional/configs/test_configs.py | 2 +- .../test_simple_source_override.py | 4 +- .../sources/test_source_fresher_state.py | 8 +- .../sources/test_source_freshness.py | 8 +- tests/unit/test_manifest.py | 19 ++- 26 files changed, 177 insertions(+), 154 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240329-223653.yaml diff --git a/.changes/unreleased/Fixes-20240329-223653.yaml b/.changes/unreleased/Fixes-20240329-223653.yaml new file mode 100644 index 00000000000..7eac4c63827 --- /dev/null +++ b/.changes/unreleased/Fixes-20240329-223653.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: datetime.datetime.utcnow() is deprecated as of Python 3.12 +time: 2024-03-29T22:36:53.504295426+01:00 +custom: + Author: slothkong + Issue: "9791" diff --git a/core/dbt/artifacts/schemas/base.py b/core/dbt/artifacts/schemas/base.py index ad94aa64e68..a7f9b5ec7a2 100644 --- a/core/dbt/artifacts/schemas/base.py +++ b/core/dbt/artifacts/schemas/base.py @@ -60,7 +60,7 @@ def read(cls, path: str): class BaseArtifactMetadata(dbtClassMixin): dbt_schema_version: str dbt_version: str = __version__ - generated_at: datetime = dataclasses.field(default_factory=datetime.utcnow) + generated_at: datetime = dataclasses.field(default_factory=datetime.now) invocation_id: Optional[str] = dataclasses.field(default_factory=get_invocation_id) env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_vars) diff --git a/core/dbt/artifacts/schemas/results.py b/core/dbt/artifacts/schemas/results.py index 2e452f44678..55f9099e3ef 100644 --- a/core/dbt/artifacts/schemas/results.py +++ b/core/dbt/artifacts/schemas/results.py @@ -8,7 +8,7 @@ from dbt_common.dataclass_schema import dbtClassMixin, StrEnum from dataclasses import dataclass -from datetime import datetime +from datetime import timezone, datetime from typing import Any, Callable, Dict, List, Optional, Sequence, Union @@ -19,10 +19,10 @@ class TimingInfo(dbtClassMixin): completed_at: Optional[datetime] = None def begin(self): - self.started_at = datetime.utcnow() + self.started_at = datetime.now(timezone.utc).replace(tzinfo=None) def end(self): - self.completed_at = datetime.utcnow() + self.completed_at = datetime.now(timezone.utc).replace(tzinfo=None) def to_msg_dict(self): msg_dict = {"name": self.name} diff --git a/core/dbt/artifacts/schemas/run/v5/run.py b/core/dbt/artifacts/schemas/run/v5/run.py index e8b5d1ddf36..3bc61406989 100644 --- a/core/dbt/artifacts/schemas/run/v5/run.py +++ b/core/dbt/artifacts/schemas/run/v5/run.py @@ -90,7 +90,7 @@ class RunExecutionResult( ): results: Sequence[RunResult] args: Dict[str, Any] = field(default_factory=dict) - generated_at: datetime = field(default_factory=datetime.utcnow) + generated_at: datetime = field(default_factory=datetime.now) def write(self, path: str): writable = RunResultsArtifact.from_execution_results( diff --git a/core/dbt/contracts/graph/node_args.py b/core/dbt/contracts/graph/node_args.py index cd19252275c..5b1864cbc44 100644 --- a/core/dbt/contracts/graph/node_args.py +++ b/core/dbt/contracts/graph/node_args.py @@ -18,7 +18,7 @@ class ModelNodeArgs: latest_version: Optional[NodeVersion] = None deprecation_date: Optional[datetime] = None access: Optional[str] = AccessType.Protected.value - generated_at: datetime = field(default_factory=datetime.utcnow) + generated_at: datetime = field(default_factory=datetime.now) depends_on_nodes: List[str] = field(default_factory=list) enabled: bool = True diff --git a/core/dbt/contracts/sql.py b/core/dbt/contracts/sql.py index ec1033ef831..0a9f8a2d1df 100644 --- a/core/dbt/contracts/sql.py +++ b/core/dbt/contracts/sql.py @@ -34,7 +34,7 @@ class RemoteCompileResultMixin(RemoteResult): @dataclass @schema_version("remote-compile-result", 1) class RemoteCompileResult(RemoteCompileResultMixin): - generated_at: datetime = field(default_factory=datetime.utcnow) + generated_at: datetime = field(default_factory=datetime.now) @property def error(self): @@ -46,7 +46,7 @@ def error(self): class RemoteExecutionResult(ExecutionResult, RemoteResult): results: Sequence[RunResult] args: Dict[str, Any] = field(default_factory=dict) - generated_at: datetime = field(default_factory=datetime.utcnow) + generated_at: datetime = field(default_factory=datetime.now) def write(self, path: str): writable = RunResultsArtifact.from_execution_results( @@ -82,4 +82,4 @@ class ResultTable(dbtClassMixin): @schema_version("remote-run-result", 1) class RemoteRunResult(RemoteCompileResultMixin): table: ResultTable - generated_at: datetime = field(default_factory=datetime.utcnow) + generated_at: datetime = field(default_factory=datetime.now) diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index a88a4be832f..bc51ddba22a 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -257,37 +257,35 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu :param str selector: The selector or node name """ non_source_nodes = list(self.non_source_nodes(included_nodes)) - for node, real_node in non_source_nodes: - if self.node_is_match(selector, real_node.fqn, real_node.is_versioned): - yield node + for unique_id, node in non_source_nodes: + if self.node_is_match(selector, node.fqn, node.is_versioned): + yield unique_id class TagSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """yields nodes from included that have the specified tag""" - for node, real_node in self.all_nodes(included_nodes): - if hasattr(real_node, "tags") and any( - fnmatch(tag, selector) for tag in real_node.tags - ): - yield node + for unique_id, node in self.all_nodes(included_nodes): + if hasattr(node, "tags") and any(fnmatch(tag, selector) for tag in node.tags): + yield unique_id class GroupSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """yields nodes from included in the specified group""" - for node, real_node in self.groupable_nodes(included_nodes): - if selector == real_node.config.get("group"): - yield node + for unique_id, node in self.groupable_nodes(included_nodes): + if selector == node.config.get("group"): + yield unique_id class AccessSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """yields model nodes matching the specified access level""" - for node, real_node in self.parsed_nodes(included_nodes): - if not isinstance(real_node, ModelNode): + for unique_id, node in self.parsed_nodes(included_nodes): + if not isinstance(node, ModelNode): continue - if selector == real_node.access: - yield node + if selector == node.access: + yield unique_id class SourceSelectorMethod(SelectorMethod): @@ -310,14 +308,14 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu ).format(selector) raise DbtRuntimeError(msg) - for node, real_node in self.source_nodes(included_nodes): - if not fnmatch(real_node.package_name, target_package): + for unique_id, node in self.source_nodes(included_nodes): + if not fnmatch(node.package_name, target_package): continue - if not fnmatch(real_node.source_name, target_source): + if not fnmatch(node.source_name, target_source): continue - if not fnmatch(real_node.name, target_table): + if not fnmatch(node.name, target_table): continue - yield node + yield unique_id class ExposureSelectorMethod(SelectorMethod): @@ -336,13 +334,13 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu ).format(selector) raise DbtRuntimeError(msg) - for node, real_node in self.exposure_nodes(included_nodes): - if not fnmatch(real_node.package_name, target_package): + for unique_id, node in self.exposure_nodes(included_nodes): + if not fnmatch(node.package_name, target_package): continue - if not fnmatch(real_node.name, target_name): + if not fnmatch(node.name, target_name): continue - yield node + yield unique_id class MetricSelectorMethod(SelectorMethod): @@ -361,13 +359,13 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu ).format(selector) raise DbtRuntimeError(msg) - for node, real_node in self.metric_nodes(included_nodes): - if not fnmatch(real_node.package_name, target_package): + for unique_id, node in self.metric_nodes(included_nodes): + if not fnmatch(node.package_name, target_package): continue - if not fnmatch(real_node.name, target_name): + if not fnmatch(node.name, target_name): continue - yield node + yield unique_id class SemanticModelSelectorMethod(SelectorMethod): @@ -386,13 +384,13 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu ).format(selector) raise DbtRuntimeError(msg) - for node, real_node in self.semantic_model_nodes(included_nodes): - if not fnmatch(real_node.package_name, target_package): + for unique_id, node in self.semantic_model_nodes(included_nodes): + if not fnmatch(node.package_name, target_package): continue - if not fnmatch(real_node.name, target_name): + if not fnmatch(node.name, target_name): continue - yield node + yield unique_id class SavedQuerySelectorMethod(SelectorMethod): @@ -411,13 +409,13 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu ).format(selector) raise DbtRuntimeError(msg) - for node, real_node in self.saved_query_nodes(included_nodes): - if not fnmatch(real_node.package_name, target_package): + for unique_id, node in self.saved_query_nodes(included_nodes): + if not fnmatch(node.package_name, target_package): continue - if not fnmatch(real_node.name, target_name): + if not fnmatch(node.name, target_name): continue - yield node + yield unique_id class PathSelectorMethod(SelectorMethod): @@ -430,35 +428,35 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu else: root = Path.cwd() paths = set(p.relative_to(root) for p in root.glob(selector)) - for node, real_node in self.all_nodes(included_nodes): - ofp = Path(real_node.original_file_path) + for unique_id, node in self.all_nodes(included_nodes): + ofp = Path(node.original_file_path) if ofp in paths: - yield node - if hasattr(real_node, "patch_path") and real_node.patch_path: # type: ignore - pfp = real_node.patch_path.split("://")[1] # type: ignore + yield unique_id + if hasattr(node, "patch_path") and node.patch_path: # type: ignore + pfp = node.patch_path.split("://")[1] # type: ignore ymlfp = Path(pfp) if ymlfp in paths: - yield node + yield unique_id if any(parent in paths for parent in ofp.parents): - yield node + yield unique_id class FileSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """Yields nodes from included that match the given file name.""" - for node, real_node in self.all_nodes(included_nodes): - if fnmatch(Path(real_node.original_file_path).name, selector): - yield node - elif fnmatch(Path(real_node.original_file_path).stem, selector): - yield node + for unique_id, node in self.all_nodes(included_nodes): + if fnmatch(Path(node.original_file_path).name, selector): + yield unique_id + elif fnmatch(Path(node.original_file_path).stem, selector): + yield unique_id class PackageSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: """Yields nodes from included that have the specified package""" - for node, real_node in self.all_nodes(included_nodes): - if fnmatch(real_node.package_name, selector): - yield node + for unique_id, node in self.all_nodes(included_nodes): + if fnmatch(node.package_name, selector): + yield unique_id def _getattr_descend(obj: Any, attrs: List[str]) -> Any: @@ -500,9 +498,9 @@ def search( # search sources is kind of useless now source configs only have # 'enabled', which you can't really filter on anyway, but maybe we'll # add more someday, so search them anyway. - for node, real_node in self.configurable_nodes(included_nodes): + for unique_id, node in self.configurable_nodes(included_nodes): try: - value = _getattr_descend(real_node.config, parts) + value = _getattr_descend(node.config, parts) except AttributeError: continue else: @@ -512,7 +510,7 @@ def search( or (CaseInsensitive(selector) == "true" and True in value) or (CaseInsensitive(selector) == "false" and False in value) ): - yield node + yield unique_id else: if ( (selector == value) @@ -520,7 +518,7 @@ def search( or (CaseInsensitive(selector) == "false") and value is False ): - yield node + yield unique_id class ResourceTypeSelectorMethod(SelectorMethod): @@ -529,9 +527,9 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu resource_type = NodeType(selector) except ValueError as exc: raise DbtRuntimeError(f'Invalid resource_type selector "{selector}"') from exc - for node, real_node in self.all_nodes(included_nodes): - if real_node.resource_type == resource_type: - yield node + for unique_id, node in self.all_nodes(included_nodes): + if node.resource_type == resource_type: + yield unique_id class TestNameSelectorMethod(SelectorMethod): @@ -761,9 +759,9 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu matches = set( result.unique_id for result in self.previous_state.results if result.status == selector ) - for node, real_node in self.all_nodes(included_nodes): - if node in matches: - yield node + for unique_id, node in self.all_nodes(included_nodes): + if unique_id in matches: + yield unique_id class SourceStatusSelectorMethod(SelectorMethod): @@ -815,37 +813,37 @@ def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[Uniqu ): matches.remove(unique_id) - for node, real_node in self.all_nodes(included_nodes): - if node in matches: - yield node + for unique_id, node in self.all_nodes(included_nodes): + if unique_id in matches: + yield unique_id class VersionSelectorMethod(SelectorMethod): def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]: - for node, real_node in self.parsed_nodes(included_nodes): - if isinstance(real_node, ModelNode): + for unique_id, node in self.parsed_nodes(included_nodes): + if isinstance(node, ModelNode): if selector == "latest": - if real_node.is_latest_version: - yield node + if node.is_latest_version: + yield unique_id elif selector == "prerelease": if ( - real_node.version - and real_node.latest_version - and UnparsedVersion(v=real_node.version) - > UnparsedVersion(v=real_node.latest_version) + node.version + and node.latest_version + and UnparsedVersion(v=node.version) + > UnparsedVersion(v=node.latest_version) ): - yield node + yield unique_id elif selector == "old": if ( - real_node.version - and real_node.latest_version - and UnparsedVersion(v=real_node.version) - < UnparsedVersion(v=real_node.latest_version) + node.version + and node.latest_version + and UnparsedVersion(v=node.version) + < UnparsedVersion(v=node.latest_version) ): - yield node + yield unique_id elif selector == "none": - if real_node.version is None: - yield node + if node.version is None: + yield unique_id else: raise DbtRuntimeError( f'Invalid version type selector {selector}: expected one of: "latest", "prerelease", "old", or "none"' diff --git a/core/dbt/logger.py b/core/dbt/logger.py index 33332417f2b..889368218ec 100644 --- a/core/dbt/logger.py +++ b/core/dbt/logger.py @@ -7,7 +7,7 @@ import time import warnings from dataclasses import dataclass -from datetime import datetime +from datetime import timezone, datetime from typing import Optional, List, ContextManager, Callable, Dict, Any, Set import logbook @@ -295,7 +295,7 @@ def __init__(self, name: str) -> None: def process(self, record): super().process(record) - record.extra[self.name] = datetime.utcnow().isoformat() + record.extra[self.name] = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() class ScrubSecrets(logbook.Processor): diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 0dd3ec08423..1814f32d9f2 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1,7 +1,7 @@ from copy import deepcopy from dataclasses import dataclass from dataclasses import field -import datetime +from datetime import timezone, datetime, date import os import traceback from typing import ( @@ -146,10 +146,10 @@ def extended_mashumaro_encoder(data): def extended_msgpack_encoder(obj): - if type(obj) is datetime.date: + if type(obj) is date: date_bytes = msgpack.ExtType(1, obj.isoformat().encode()) return date_bytes - elif type(obj) is datetime.datetime: + elif type(obj) is datetime: datetime_bytes = msgpack.ExtType(2, obj.isoformat().encode()) return datetime_bytes @@ -162,10 +162,10 @@ def extended_mashumuro_decoder(data): def extended_msgpack_decoder(code, data): if code == 1: - d = datetime.date.fromisoformat(data.decode()) + d = date.fromisoformat(data.decode()) return d elif code == 2: - dt = datetime.datetime.fromisoformat(data.decode()) + dt = datetime.fromisoformat(data.decode()) return dt else: return msgpack.ExtType(code, data) @@ -588,10 +588,7 @@ def safe_update_project_parser_files_partially(self, project_parser_files: Dict) def check_for_model_deprecations(self): for node in self.manifest.nodes.values(): if isinstance(node, ModelNode): - if ( - node.deprecation_date - and node.deprecation_date < datetime.datetime.now().astimezone() - ): + if node.deprecation_date and node.deprecation_date < datetime.now().astimezone(): warn_or_error( DeprecatedModel( model_name=node.name, @@ -605,7 +602,7 @@ def check_for_model_deprecations(self): node.depends_on for resolved_ref in resolved_model_refs: if resolved_ref.deprecation_date: - if resolved_ref.deprecation_date < datetime.datetime.now().astimezone(): + if resolved_ref.deprecation_date < datetime.now().astimezone(): event_cls = DeprecatedReference else: event_cls = UpcomingReferenceDeprecation @@ -901,7 +898,9 @@ def read_manifest_for_partial_parse(self) -> Optional[Manifest]: is_partial_parsable, reparse_reason = self.is_partial_parsable(manifest) if is_partial_parsable: # We don't want to have stale generated_at dates - manifest.metadata.generated_at = datetime.datetime.utcnow() + manifest.metadata.generated_at = datetime.now(timezone.utc).replace( + tzinfo=None + ) # or invocation_ids manifest.metadata.invocation_id = get_invocation_id() return manifest diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 2ca6cb2e978..ca0e45a4689 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -4,7 +4,7 @@ import traceback from abc import ABCMeta, abstractmethod from contextlib import nullcontext -from datetime import datetime +from datetime import timezone, datetime from pathlib import Path from typing import Any, Dict, List, Optional, Type, Union, Set @@ -232,7 +232,8 @@ def run_with_hooks(self, manifest): result = self.safe_run(manifest) self.node.update_event_status( - node_status=result.status, finished_at=datetime.utcnow().isoformat() + node_status=result.status, + finished_at=datetime.now(timezone.utc).replace(tzinfo=None).isoformat(), ) if not self.node.is_ephemeral_model: diff --git a/core/dbt/task/docs/generate.py b/core/dbt/task/docs/generate.py index 800f997268d..590dab97671 100644 --- a/core/dbt/task/docs/generate.py +++ b/core/dbt/task/docs/generate.py @@ -1,7 +1,7 @@ import os import shutil from dataclasses import replace -from datetime import datetime +from datetime import timezone, datetime from typing import Dict, List, Any, Optional, Tuple, Set, Iterable import agate from itertools import chain @@ -226,7 +226,7 @@ def run(self) -> CatalogArtifact: return CatalogArtifact.from_results( nodes={}, sources={}, - generated_at=datetime.utcnow(), + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), errors=None, compile_results=compile_results, ) @@ -306,7 +306,7 @@ def run(self) -> CatalogArtifact: results = self.get_catalog_results( nodes=nodes, sources=sources, - generated_at=datetime.utcnow(), + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), compile_results=compile_results, errors=errors, ) diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index 2a6031ad45c..b5a850ac512 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -11,7 +11,7 @@ print_run_end_messages, get_counts, ) -from datetime import datetime +from datetime import timezone, datetime from dbt import tracking from dbt import utils from dbt.adapters.base import BaseRelation @@ -366,7 +366,8 @@ def run_hooks(self, adapter, hook_type: RunHookType, extra_context) -> None: # log_contextvars with log_contextvars(node_info=hook.node_info): hook.update_event_status( - started_at=datetime.utcnow().isoformat(), node_status=RunningStatus.Started + started_at=datetime.now(timezone.utc).replace(tzinfo=None).isoformat(), + node_status=RunningStatus.Started, ) sql = self.get_hook_sql(adapter, hook, idx, num_hooks, extra_context) @@ -391,7 +392,9 @@ def run_hooks(self, adapter, hook_type: RunHookType, extra_context) -> None: status = "OK" self.ran_hooks.append(hook) - hook.update_event_status(finished_at=datetime.utcnow().isoformat()) + hook.update_event_status( + finished_at=datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + ) with finishctx, DbtModelState({"node_status": "passed"}): hook.update_event_status(node_status=RunStatus.Success) fire_event( diff --git a/core/dbt/task/run_operation.py b/core/dbt/task/run_operation.py index 1c6c5002e27..9abee69ad3d 100644 --- a/core/dbt/task/run_operation.py +++ b/core/dbt/task/run_operation.py @@ -1,7 +1,7 @@ import os import threading import traceback -from datetime import datetime +from datetime import timezone, datetime from typing import TYPE_CHECKING import dbt_common.exceptions @@ -51,7 +51,7 @@ def _run_unsafe(self, package_name, macro_name) -> "agate.Table": return res def run(self) -> RunResultsArtifact: - start = datetime.utcnow() + start = datetime.now(timezone.utc).replace(tzinfo=None) self.compile_manifest() success = True @@ -69,7 +69,7 @@ def run(self) -> RunResultsArtifact: fire_event(LogDebugStackTrace(exc_info=traceback.format_exc())) success = False - end = datetime.utcnow() + end = datetime.now(timezone.utc).replace(tzinfo=None) macro = ( self.manifest.find_macro_by_name(macro_name, self.config.project_name, package_name) diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index cff69e23e80..a89dfa7d15b 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -2,7 +2,7 @@ import time from abc import abstractmethod from concurrent.futures import as_completed -from datetime import datetime +from datetime import timezone, datetime from multiprocessing.dummy import Pool as ThreadPool from pathlib import Path from typing import AbstractSet, Optional, Dict, List, Set, Tuple, Iterable @@ -210,7 +210,8 @@ def call_runner(self, runner: BaseRunner) -> RunResult: startctx = TimestampNamed("node_started_at") index = self.index_offset(runner.node_index) runner.node.update_event_status( - started_at=datetime.utcnow().isoformat(), node_status=RunningStatus.Started + started_at=datetime.now(timezone.utc).replace(tzinfo=None).isoformat(), + node_status=RunningStatus.Started, ) extended_metadata = ModelMetadata(runner.node, index) @@ -424,7 +425,7 @@ def execute_nodes(self): run_result = self.get_result( results=self.node_results, elapsed_time=time.time() - self.started_at, - generated_at=datetime.utcnow(), + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) if self.args.write_json and hasattr(run_result, "write"): @@ -502,7 +503,9 @@ def execute_with_hooks(self, selected_uids: AbstractSet[str]): elapsed = time.time() - self.started_at self.print_results_line(self.node_results, elapsed) result = self.get_result( - results=self.node_results, elapsed_time=elapsed, generated_at=datetime.utcnow() + results=self.node_results, + elapsed_time=elapsed, + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) return result @@ -527,7 +530,7 @@ def run(self): warn_or_error(NothingToDo()) result = self.get_result( results=[], - generated_at=datetime.utcnow(), + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), elapsed_time=0.0, ) else: diff --git a/core/dbt/task/sql.py b/core/dbt/task/sql.py index 6a9522133d0..8aad9798ecd 100644 --- a/core/dbt/task/sql.py +++ b/core/dbt/task/sql.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from datetime import datetime +from datetime import timezone, datetime from typing import Generic, TypeVar import traceback @@ -68,7 +68,7 @@ def execute(self, compiled_node, manifest) -> RemoteCompileResult: node=compiled_node, timing=[], # this will get added later logs=[], - generated_at=datetime.utcnow(), + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) def from_run_result(self, result, start_time, timing_info) -> RemoteCompileResult: @@ -78,7 +78,7 @@ def from_run_result(self, result, start_time, timing_info) -> RemoteCompileResul node=result.node, timing=timing_info, logs=[], - generated_at=datetime.utcnow(), + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) @@ -98,7 +98,7 @@ def execute(self, compiled_node, manifest) -> RemoteRunResult: table=table, timing=[], logs=[], - generated_at=datetime.utcnow(), + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) def from_run_result(self, result, start_time, timing_info) -> RemoteRunResult: @@ -109,5 +109,5 @@ def from_run_result(self, result, start_time, timing_info) -> RemoteRunResult: table=result.table, timing=timing_info, logs=[], - generated_at=datetime.utcnow(), + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), ) diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index 3ef25c55789..70fc053ed67 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -3,7 +3,7 @@ import pytest # type: ignore import random from argparse import Namespace -from datetime import datetime +from datetime import timezone, datetime import warnings import yaml @@ -70,7 +70,9 @@ def prefix(): # create a directory name that will be unique per test session _randint = random.randint(0, 9999) - _runtime_timedelta = datetime.utcnow() - datetime(1970, 1, 1, 0, 0, 0) + _runtime_timedelta = datetime.now(timezone.utc).replace(tzinfo=None) - datetime( + 1970, 1, 1, 0, 0, 0 + ) _runtime = (int(_runtime_timedelta.total_seconds() * 1e6)) + _runtime_timedelta.microseconds prefix = f"test{_runtime}{_randint:04}" return prefix diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index 437f25aa2e2..31e9766384f 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -5,7 +5,7 @@ import yaml import json import warnings -from datetime import datetime +from datetime import timezone, datetime from typing import Any, Dict, List, Optional from contextlib import contextmanager from dbt.adapters.factory import Adapter @@ -287,7 +287,7 @@ def check_result_nodes_by_unique_id(results, unique_ids): def check_datetime_between(timestr, start, end=None): datefmt = "%Y-%m-%dT%H:%M:%S.%fZ" if end is None: - end = datetime.utcnow() + end = datetime.now(timezone.utc).replace(tzinfo=None) parsed = datetime.strptime(timestr, datefmt) assert start <= parsed assert end >= parsed diff --git a/core/dbt/utils.py b/core/dbt/utils.py index 8f7509a5dec..1ab12fddcf7 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -1,11 +1,11 @@ import collections -import datetime import decimal import functools import itertools import jinja2 import json import os +from datetime import timezone, datetime, date, time from pathlib import PosixPath, WindowsPath from dbt_common.utils import md5 @@ -144,7 +144,7 @@ def add_ephemeral_model_prefix(s: str) -> str: def timestring() -> str: """Get the current datetime as an RFC 3339-compliant string""" # isoformat doesn't include the mandatory trailing 'Z' for UTC. - return datetime.datetime.utcnow().isoformat() + "Z" + return datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z" def humanize_execution_time(execution_time: int) -> str: @@ -163,7 +163,7 @@ class JSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, DECIMALS): return float(obj) - elif isinstance(obj, (datetime.datetime, datetime.date, datetime.time)): + elif isinstance(obj, (datetime, date, time)): return obj.isoformat() elif isinstance(obj, jinja2.Undefined): return "" diff --git a/tests/functional/adapter/basic/test_docs_generate.py b/tests/functional/adapter/basic/test_docs_generate.py index 9b849160564..5503edd9e7c 100644 --- a/tests/functional/adapter/basic/test_docs_generate.py +++ b/tests/functional/adapter/basic/test_docs_generate.py @@ -1,6 +1,6 @@ import pytest import os -from datetime import datetime +from datetime import timezone, datetime import dbt from dbt.tests.util import run_dbt, rm_file, get_artifact, check_datetime_between @@ -353,7 +353,7 @@ def run_and_generate(project, args=None): rm_file(project.project_root, "target", "manifest.json") rm_file(project.project_root, "target", "run_results.json") - start_time = datetime.utcnow() + start_time = datetime.now(timezone.utc).replace(tzinfo=None) run_args = ["docs", "generate"] if args: run_args.extend(args) diff --git a/tests/functional/adapter/utils/test_current_timestamp.py b/tests/functional/adapter/utils/test_current_timestamp.py index 2a071c4ba2a..a205b7e0df4 100644 --- a/tests/functional/adapter/utils/test_current_timestamp.py +++ b/tests/functional/adapter/utils/test_current_timestamp.py @@ -50,7 +50,11 @@ def utcnow_matching_type(self, dt: datetime) -> datetime: """ Current UTC datetime with the same timezone-awareness (or naiveness) as the input. """ - return datetime.now(timezone.utc) if is_aware(dt) else datetime.utcnow() + return ( + datetime.now(timezone.utc) + if is_aware(dt) + else datetime.now(timezone.utc).replace(tzinfo=None) + ) class BaseCurrentTimestampAware(BaseCurrentTimestamp): diff --git a/tests/functional/artifacts/test_artifacts.py b/tests/functional/artifacts/test_artifacts.py index 7aadb308513..7a26ea72501 100644 --- a/tests/functional/artifacts/test_artifacts.py +++ b/tests/functional/artifacts/test_artifacts.py @@ -1,6 +1,6 @@ import pytest import os -from datetime import datetime +from datetime import timezone, datetime import dbt import jsonschema @@ -612,7 +612,7 @@ def models(self): # Test generic "docs generate" command def test_run_and_generate(self, project, manifest_schema_path, run_results_schema_path): - start_time = datetime.utcnow() + start_time = datetime.now(timezone.utc).replace(tzinfo=None) results = run_dbt(["compile"]) assert len(results) == 7 verify_manifest( @@ -636,7 +636,7 @@ def models(self): } def test_references(self, project, manifest_schema_path, run_results_schema_path): - start_time = datetime.utcnow() + start_time = datetime.now(timezone.utc).replace(tzinfo=None) results = run_dbt(["compile"]) assert len(results) == 4 verify_manifest( @@ -666,7 +666,7 @@ def snapshots(self): return {} def test_versions(self, project, manifest_schema_path, run_results_schema_path): - start_time = datetime.utcnow() + start_time = datetime.now(timezone.utc).replace(tzinfo=None) results = run_dbt(["compile"]) assert len(results) == 6 verify_manifest( diff --git a/tests/functional/configs/test_configs.py b/tests/functional/configs/test_configs.py index 7ac6259ac13..da613dc257c 100644 --- a/tests/functional/configs/test_configs.py +++ b/tests/functional/configs/test_configs.py @@ -51,7 +51,7 @@ class TestTargetConfigs(BaseConfigProject): @pytest.fixture(scope="class") def project_config_update(self): return { - "target-path": "target_{{ modules.datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%S') }}", + "target-path": "target_{{ modules.datetime.datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%dT%H%M%S') }}", "seeds": { "quote_columns": False, }, diff --git a/tests/functional/source_overrides/test_simple_source_override.py b/tests/functional/source_overrides/test_simple_source_override.py index da1b4856e32..8eade582d04 100644 --- a/tests/functional/source_overrides/test_simple_source_override.py +++ b/tests/functional/source_overrides/test_simple_source_override.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import timezone, datetime, timedelta import pytest from dbt.tests.util import run_dbt, update_config_file, check_relations_equal @@ -61,7 +61,7 @@ def project_config_update(self): } def _set_updated_at_to(self, insert_id, delta, project): - insert_time = datetime.utcnow() + delta + insert_time = datetime.now(timezone.utc).replace(tzinfo=None) + delta timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") # favorite_color,id,first_name,email,ip_address,updated_at diff --git a/tests/functional/sources/test_source_fresher_state.py b/tests/functional/sources/test_source_fresher_state.py index 95c048c769f..cf6cc605493 100644 --- a/tests/functional/sources/test_source_fresher_state.py +++ b/tests/functional/sources/test_source_fresher_state.py @@ -2,7 +2,7 @@ import json import shutil import pytest -from datetime import datetime, timedelta +from datetime import timezone, datetime, timedelta from dbt_common.exceptions import DbtInternalError @@ -30,7 +30,7 @@ class SuccessfulSourceFreshnessTest(BaseSourcesTest): def setUp(self, project): self.run_dbt_with_vars(project, ["seed"]) pytest._id = 101 - pytest.freshness_start_time = datetime.utcnow() + pytest.freshness_start_time = datetime.now(timezone.utc).replace(tzinfo=None) # this is the db initial value pytest.last_inserted_time = "2016-09-19T14:45:51+00:00" @@ -41,7 +41,7 @@ def setUp(self, project): del os.environ["DBT_ENV_CUSTOM_ENV_key"] def _set_updated_at_to(self, project, delta): - insert_time = datetime.utcnow() + delta + insert_time = datetime.now(timezone.utc).replace(tzinfo=None) + delta timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") # favorite_color,id,first_name,email,ip_address,updated_at insert_id = pytest._id @@ -70,7 +70,7 @@ def _set_updated_at_to(self, project, delta): def assertBetween(self, timestr, start, end=None): datefmt = "%Y-%m-%dT%H:%M:%S.%fZ" if end is None: - end = datetime.utcnow() + end = datetime.now(timezone.utc).replace(tzinfo=None) parsed = datetime.strptime(timestr, datefmt) diff --git a/tests/functional/sources/test_source_freshness.py b/tests/functional/sources/test_source_freshness.py index 0e58b33b555..6d3705bf974 100644 --- a/tests/functional/sources/test_source_freshness.py +++ b/tests/functional/sources/test_source_freshness.py @@ -1,7 +1,7 @@ import os import json import pytest -from datetime import datetime, timedelta +from datetime import timezone, datetime, timedelta import yaml import dbt.version @@ -25,7 +25,7 @@ class SuccessfulSourceFreshnessTest(BaseSourcesTest): def setUp(self, project): self.run_dbt_with_vars(project, ["seed"]) pytest._id = 101 - pytest.freshness_start_time = datetime.utcnow() + pytest.freshness_start_time = datetime.now(timezone.utc).replace(tzinfo=None) # this is the db initial value pytest.last_inserted_time = "2016-09-19T14:45:51+00:00" @@ -36,7 +36,7 @@ def setUp(self, project): del os.environ["DBT_ENV_CUSTOM_ENV_key"] def _set_updated_at_to(self, project, delta): - insert_time = datetime.utcnow() + delta + insert_time = datetime.now(timezone.utc).replace(tzinfo=None) + delta timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") # favorite_color,id,first_name,email,ip_address,updated_at insert_id = pytest._id @@ -65,7 +65,7 @@ def _set_updated_at_to(self, project, delta): def assertBetween(self, timestr, start, end=None): datefmt = "%Y-%m-%dT%H:%M:%S.%fZ" if end is None: - end = datetime.utcnow() + end = datetime.now(timezone.utc).replace(tzinfo=None) parsed = datetime.strptime(timestr, datefmt) diff --git a/tests/unit/test_manifest.py b/tests/unit/test_manifest.py index ea443d1147f..a3999f62a85 100644 --- a/tests/unit/test_manifest.py +++ b/tests/unit/test_manifest.py @@ -3,7 +3,7 @@ from argparse import Namespace from collections import namedtuple from copy import deepcopy -from datetime import datetime +from datetime import timezone, datetime from itertools import product from unittest import mock @@ -372,7 +372,9 @@ def test_no_nodes(self, mock_user): exposures={}, metrics={}, selectors={}, - metadata=ManifestMetadata(generated_at=datetime.utcnow()), + metadata=ManifestMetadata( + generated_at=datetime.now(timezone.utc).replace(tzinfo=None) + ), semantic_models={}, saved_queries={}, ) @@ -426,7 +428,9 @@ def test_nested_nodes(self, mock_user): exposures={}, metrics={}, selectors={}, - metadata=ManifestMetadata(generated_at=datetime.utcnow()), + metadata=ManifestMetadata( + generated_at=datetime.now(timezone.utc).replace(tzinfo=None) + ), ) serialized = manifest.writable_manifest().to_dict(omit_none=True) self.assertEqual(serialized["metadata"]["generated_at"], "2018-02-14T09:15:13Z") @@ -530,7 +534,7 @@ def test_no_nodes_with_metadata(self, mock_user): metadata = ManifestMetadata( project_id="098f6bcd4621d373cade4e832627b4f6", adapter_type="postgres", - generated_at=datetime.utcnow(), + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), user_id="cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf", send_anonymous_usage_stats=False, ) @@ -880,7 +884,8 @@ def test_no_nodes(self, mock_user): mock_user.id = "cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf" set_from_args(Namespace(SEND_ANONYMOUS_USAGE_STATS=False), None) metadata = ManifestMetadata( - generated_at=datetime.utcnow(), invocation_id="01234567-0123-0123-0123-0123456789ab" + generated_at=datetime.now(timezone.utc).replace(tzinfo=None), + invocation_id="01234567-0123-0123-0123-0123456789ab", ) manifest = Manifest( nodes={}, @@ -935,7 +940,9 @@ def test_nested_nodes(self): docs={}, disabled={}, selectors={}, - metadata=ManifestMetadata(generated_at=datetime.utcnow()), + metadata=ManifestMetadata( + generated_at=datetime.now(timezone.utc).replace(tzinfo=None) + ), files={}, exposures={}, ) From d8142cddbf0751505d345e2e13d15a2f75e1018d Mon Sep 17 00:00:00 2001 From: slothkong Date: Sat, 30 Mar 2024 10:15:28 +0100 Subject: [PATCH 2/3] Explicit UTC timezone declaration for instances of datetime.now() --- core/dbt/parser/manifest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 1814f32d9f2..a9c7f5922d8 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -588,7 +588,10 @@ def safe_update_project_parser_files_partially(self, project_parser_files: Dict) def check_for_model_deprecations(self): for node in self.manifest.nodes.values(): if isinstance(node, ModelNode): - if node.deprecation_date and node.deprecation_date < datetime.now().astimezone(): + if ( + node.deprecation_date + and node.deprecation_date < datetime.now(timezone.utc).astimezone() + ): warn_or_error( DeprecatedModel( model_name=node.name, @@ -602,7 +605,7 @@ def check_for_model_deprecations(self): node.depends_on for resolved_ref in resolved_model_refs: if resolved_ref.deprecation_date: - if resolved_ref.deprecation_date < datetime.now().astimezone(): + if resolved_ref.deprecation_date < datetime.now(timezone.utc).astimezone(): event_cls = DeprecatedReference else: event_cls = UpcomingReferenceDeprecation From 205a69ea1f7ac07bfa6b22ec71ffd1925cdd4804 Mon Sep 17 00:00:00 2001 From: slothkong Date: Sat, 30 Mar 2024 19:56:29 +0100 Subject: [PATCH 3/3] Keep utcnow() in functional test case to avoid setup errors --- tests/functional/configs/test_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/configs/test_configs.py b/tests/functional/configs/test_configs.py index da613dc257c..7ac6259ac13 100644 --- a/tests/functional/configs/test_configs.py +++ b/tests/functional/configs/test_configs.py @@ -51,7 +51,7 @@ class TestTargetConfigs(BaseConfigProject): @pytest.fixture(scope="class") def project_config_update(self): return { - "target-path": "target_{{ modules.datetime.datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%dT%H%M%S') }}", + "target-path": "target_{{ modules.datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%S') }}", "seeds": { "quote_columns": False, },