From 0c367680b05e620111399f38d1fa76f7c4e91b5d Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Tue, 26 Mar 2024 11:05:33 -0700 Subject: [PATCH] Fix E721 errors in the CDK (#36490) --- .../sources/declarative/create_partial.py | 92 ------------------- .../interpolation/interpolated_mapping.py | 12 +-- .../test_model_to_component_factory.py | 6 +- .../declarative/test_create_partial.py | 83 ----------------- .../utils/test_stream_status_utils.py | 8 +- .../unit_tests/utils/test_traced_exception.py | 6 +- pyproject.toml | 1 - 7 files changed, 16 insertions(+), 192 deletions(-) delete mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/create_partial.py delete mode 100644 airbyte-cdk/python/unit_tests/sources/declarative/test_create_partial.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/create_partial.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/create_partial.py deleted file mode 100644 index bfbdb9078480..000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/create_partial.py +++ /dev/null @@ -1,92 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import inspect -from typing import Any, Mapping - -PARAMETERS_STR = "$parameters" - - -def create(func, /, *args, **keywords): - """ - Create a partial on steroids. - Returns a partial object which when called will behave like func called with the arguments supplied. - Parameters will be interpolated before the creation of the object - The interpolation will take in kwargs, and config as parameters that can be accessed through interpolating. - If any of the parameters are also create functions, they will also be created. - kwargs are propagated to the recursive method calls - - :param func: Function - :param args: - :param keywords: - :return: partially created object - """ - - def newfunc(*fargs, **fkeywords): - - all_keywords = {**keywords} - all_keywords.update(fkeywords) - - # config is a special keyword used for interpolation - config = all_keywords.pop("config", None) - - # $parameters is a special keyword used for interpolation and propagation - if PARAMETERS_STR in all_keywords: - parameters = all_keywords.get(PARAMETERS_STR) - else: - parameters = dict() - - # if config is not none, add it back to the keywords mapping - if config is not None: - all_keywords["config"] = config - - kwargs_to_pass_down = _get_kwargs_to_pass_to_func(func, parameters, all_keywords) - all_keywords_to_pass_down = _get_kwargs_to_pass_to_func(func, all_keywords, all_keywords) - - # parameters is required as part of creation of all declarative components - dynamic_args = {**all_keywords_to_pass_down, **kwargs_to_pass_down} - if "parameters" not in dynamic_args: - dynamic_args["parameters"] = {} - else: - # Handles the case where kwarg parameters and keyword $parameters both exist. We should merge both sets of parameters - # before creating the component - dynamic_args["parameters"] = {**all_keywords_to_pass_down["parameters"], **kwargs_to_pass_down["parameters"]} - try: - ret = func(*args, *fargs, **dynamic_args) - except TypeError as e: - raise Exception(f"failed to create object of type {func} because {e}") - return ret - - newfunc.func = func - newfunc.args = args - newfunc.kwargs = keywords - - return newfunc - - -def _get_kwargs_to_pass_to_func(func, parameters, existing_keyword_parameters): - argspec = inspect.getfullargspec(func) - kwargs_to_pass_down = set(argspec.kwonlyargs) - args_to_pass_down = set(argspec.args) - all_args = args_to_pass_down.union(kwargs_to_pass_down) - kwargs_to_pass_down = { - k: v for k, v in parameters.items() if k in all_args and _key_is_unset_or_identical(k, v, existing_keyword_parameters) - } - if "parameters" in all_args: - kwargs_to_pass_down["parameters"] = parameters - return kwargs_to_pass_down - - -def _key_is_unset_or_identical(key: str, value: Any, mapping: Mapping[str, Any]): - return key not in mapping or mapping[key] == value - - -def _create_inner_objects(keywords, kwargs): - fully_created = dict() - for k, v in keywords.items(): - if type(v) == type(create): - fully_created[k] = v(kwargs=kwargs) - else: - fully_created[k] = v - return fully_created diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py index 72043e017497..bdd95ecde776 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py @@ -4,7 +4,7 @@ from dataclasses import InitVar, dataclass -from typing import Any, Mapping, Optional +from typing import Any, Dict, Mapping, Optional from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation from airbyte_cdk.sources.declarative.types import Config @@ -22,17 +22,17 @@ class InterpolatedMapping: mapping: Mapping[str, str] parameters: InitVar[Mapping[str, Any]] - def __post_init__(self, parameters: Optional[Mapping[str, Any]]): + def __post_init__(self, parameters: Optional[Mapping[str, Any]]) -> None: self._interpolation = JinjaInterpolation() self._parameters = parameters - def eval(self, config: Config, **additional_parameters): + def eval(self, config: Config, **additional_parameters: Any) -> Dict[str, Any]: """ Wrapper around a Mapping[str, str] that allows for both keys and values to be interpolated. :param config: The user-provided configuration as specified by the source's spec :param additional_parameters: Optional parameters used for interpolation - :return: The interpolated string + :return: The interpolated mapping """ valid_key_types = additional_parameters.pop("valid_key_types", (str,)) valid_value_types = additional_parameters.pop("valid_value_types", None) @@ -43,10 +43,10 @@ def eval(self, config: Config, **additional_parameters): for name, value in self.mapping.items() } - def _eval(self, value, config, **kwargs): + def _eval(self, value: str, config: Config, **kwargs: Any) -> Any: # The values in self._mapping can be of Any type # We only want to interpolate them if they are strings - if type(value) == str: + if isinstance(value, str): return self._interpolation.eval(value, config, parameters=self._parameters, **kwargs) else: return value diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 9c2b536832d4..ee0e2e8ae373 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -1725,10 +1725,10 @@ def test_merge_incremental_and_partition_router(incremental, partition_router, e if incremental and partition_router: assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor) - if type(partition_router) == list and len(partition_router) > 1: - assert type(stream.retriever.stream_slicer._partition_router) == CartesianProductStreamSlicer + if isinstance(partition_router, list) and len(partition_router) > 1: + assert isinstance(stream.retriever.stream_slicer._partition_router, CartesianProductStreamSlicer) assert len(stream.retriever.stream_slicer._partition_router.stream_slicers) == len(partition_router) - elif partition_router and type(partition_router) == list and len(partition_router) > 1: + elif partition_router and isinstance(partition_router, list) and len(partition_router) > 1: assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor) assert len(stream.retriever.stream_slicer.stream_slicerS) == len(partition_router) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_create_partial.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_create_partial.py deleted file mode 100644 index 7d7860c5ae57..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_create_partial.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -from airbyte_cdk.sources.declarative.create_partial import _key_is_unset_or_identical, create -from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString - - -class AClass: - def __init__(self, parameter, another_param, parameters): - self.parameter = parameter - self.another_param = another_param - self.parameters = parameters - - -class OuterClass: - def __init__(self, name, some_field, inner_param): - self.name = name - self.some_field = some_field - self.inner_param = inner_param - - -class OuterOuterClass: - def __init__(self, name, param, inner_class): - self.name = name - self.param = param - self.inner_class = inner_class - - -def test_pass_parameter_to_create_function(): - object = create(AClass, parameter="A")(another_param="B") - assert object.parameter == "A" - assert object.another_param == "B" - - -def test_parameter_not_overwritten_by_parameters(): - object = create(AClass, parameter="A", another_param="B", **{"$parameters": {"parameter": "C"}})() - assert object.parameter == "A" - assert object.another_param == "B" - - -def test_overwrite_param(): - object = create(AClass, parameter="A", another_param="B")(parameter="C") - assert object.parameter == "C" - assert object.another_param == "B" - - -def test_string_interpolation(): - s = "{{ next_page_token['next_page_url'] }}" - partial = create(InterpolatedString, string=s) - interpolated_string = partial() - assert interpolated_string.string == s - - -def test_string_interpolation_through_parameters(): - s = "{{ parameters['name'] }}" - parameters = {"name": "airbyte"} - partial = create(InterpolatedString, string=s, **parameters) - interpolated_string = partial() - assert interpolated_string.eval({}) == "airbyte" - - -def test_string_interpolation_through_parameters_keyword(): - s = "{{ parameters['name'] }}" - parameters = {"$parameters": {"name": "airbyte"}} - partial = create(InterpolatedString, string=s, **parameters) - interpolated_string = partial() - assert interpolated_string.eval({}) == "airbyte" - - -@pytest.mark.parametrize( - "test_name, key, value, expected_result", - [ - ("test", "key", "value", True), - ("test", "key", "a_different_value", False), - ("test", "a_different_key", "value", True), - ], -) -def test_key_is_unset_or_identical(test_name, key, value, expected_result): - mapping = {"key": "value"} - result = _key_is_unset_or_identical(key, value, mapping) - assert expected_result == result diff --git a/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py b/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py index 5d41ab7a5700..4862a1e0118e 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py +++ b/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py @@ -13,7 +13,7 @@ def test_started_as_message(): stream_status = AirbyteStreamStatus.STARTED airbyte_message = stream_status_as_airbyte_message(stream, stream_status) - assert type(airbyte_message) == AirbyteMessage + assert isinstance(airbyte_message, AirbyteMessage) assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 @@ -26,7 +26,7 @@ def test_running_as_message(): stream_status = AirbyteStreamStatus.RUNNING airbyte_message = stream_status_as_airbyte_message(stream, stream_status) - assert type(airbyte_message) == AirbyteMessage + assert isinstance(airbyte_message, AirbyteMessage) assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 @@ -39,7 +39,7 @@ def test_complete_as_message(): stream_status = AirbyteStreamStatus.COMPLETE airbyte_message = stream_status_as_airbyte_message(stream, stream_status) - assert type(airbyte_message) == AirbyteMessage + assert isinstance(airbyte_message, AirbyteMessage) assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 @@ -52,7 +52,7 @@ def test_incomplete_failed_as_message(): stream_status = AirbyteStreamStatus.INCOMPLETE airbyte_message = stream_status_as_airbyte_message(stream, stream_status) - assert type(airbyte_message) == AirbyteMessage + assert isinstance(airbyte_message, AirbyteMessage) assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 diff --git a/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py b/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py index bfe55952f993..7612ca80d1d7 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py +++ b/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py @@ -37,7 +37,7 @@ def test_exception_as_airbyte_message(): traced_exc = AirbyteTracedException("an internal message") airbyte_message = traced_exc.as_airbyte_message() - assert type(airbyte_message) == AirbyteMessage + assert isinstance(airbyte_message, AirbyteMessage) assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.ERROR assert airbyte_message.trace.emitted_at > 0 @@ -51,7 +51,7 @@ def test_existing_exception_as_airbyte_message(raised_exception): traced_exc = AirbyteTracedException.from_exception(raised_exception) airbyte_message = traced_exc.as_airbyte_message() - assert type(airbyte_message) == AirbyteMessage + assert isinstance(airbyte_message, AirbyteMessage) assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.ERROR assert airbyte_message.trace.error.message == "Something went wrong in the connector. See the logs for more details." @@ -66,7 +66,7 @@ def test_config_error_as_connection_status_message(): traced_exc = AirbyteTracedException("an internal message", message="Config validation error", failure_type=FailureType.config_error) airbyte_message = traced_exc.as_connection_status_message() - assert type(airbyte_message) == AirbyteMessage + assert isinstance(airbyte_message, AirbyteMessage) assert airbyte_message.type == MessageType.CONNECTION_STATUS assert airbyte_message.connectionStatus.status == Status.FAILED assert airbyte_message.connectionStatus.message == "Config validation error" diff --git a/pyproject.toml b/pyproject.toml index d4f4e4baee27..34f18e75c1f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,6 @@ extend-ignore = [ "E231", # Bad trailing comma (conflicts with Black) "E501", # line too long (conflicts with Black) "W503", # line break before binary operator (conflicts with Black) - "E721", # TODO: ella fix after pflake8 version update "F811", # TODO: ella fix after pflake8 version update ]