From 6ff15232bdf20a33c7565934efb59640c9f2fd30 Mon Sep 17 00:00:00 2001 From: Benji Nguyen <45523555+solidiquis@users.noreply.github.com> Date: Sat, 15 Jun 2024 17:20:30 -0700 Subject: [PATCH] chore(python): replace black with ruff for formatting + linting (#43) --- .github/workflows/python_ci.yaml | 16 ++++- python/lib/sift_internal/convert/protobuf.py | 8 ++- python/lib/sift_internal/types.py | 1 + python/lib/sift_py/ingestion/channel.py | 4 +- .../lib/sift_py/ingestion/config/__init__.py | 43 +++++++++++++ .../ingestion/{config.py => config/yaml.py} | 63 +++++-------------- .../{config_test.py => config/yaml_test.py} | 4 +- .../{ingestion_impl => impl}/__init__.py | 0 .../{ingestion_impl => impl}/ingest.py | 6 +- .../ingestion_config.py | 4 +- python/lib/sift_py/ingestion/service.py | 2 +- python/pyproject.toml | 40 ++++++++++-- python/scripts/dev | 13 +++- 13 files changed, 138 insertions(+), 66 deletions(-) create mode 100644 python/lib/sift_py/ingestion/config/__init__.py rename python/lib/sift_py/ingestion/{config.py => config/yaml.py} (74%) rename python/lib/sift_py/ingestion/{config_test.py => config/yaml_test.py} (98%) rename python/lib/sift_py/ingestion/{ingestion_impl => impl}/__init__.py (100%) rename python/lib/sift_py/ingestion/{ingestion_impl => impl}/ingest.py (98%) rename python/lib/sift_py/ingestion/{ingestion_impl => impl}/ingestion_config.py (98%) diff --git a/.github/workflows/python_ci.yaml b/.github/workflows/python_ci.yaml index a6211632..e2fedaf5 100644 --- a/.github/workflows/python_ci.yaml +++ b/.github/workflows/python_ci.yaml @@ -18,16 +18,26 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: "3.9" + python-version: "3.8" - - name: Install dependencies + - name: Pip install working-directory: python run: | python -m pip install --upgrade pip pip install '.[development]' pip install -e . - - name: Run pytest + - name: Lint + working-directory: python + run: | + ruff check + + - name: Format + working-directory: python + run: | + ruff format --check + + - name: Pytest working-directory: python run: | pytest diff --git a/python/lib/sift_internal/convert/protobuf.py b/python/lib/sift_internal/convert/protobuf.py index d58d0472..0d02ff78 100644 --- a/python/lib/sift_internal/convert/protobuf.py +++ b/python/lib/sift_internal/convert/protobuf.py @@ -4,11 +4,13 @@ ProtobufMessage = Message + class AsProtobuf(ABC): """ Abstract base class used to create create sub-types that can be treated as an object that can be converted into an instance of `ProtobufMessage`. """ + @abstractmethod def as_pb(self, klass: Type[ProtobufMessage]) -> Optional[ProtobufMessage]: """ @@ -19,6 +21,8 @@ def as_pb(self, klass: Type[ProtobufMessage]) -> Optional[ProtobufMessage]: T = TypeVar("T", bound=ProtobufMessage) + + def try_cast_pb(value: AsProtobuf, target_klass: Type[T]) -> T: """ Tries to cast the `value` to `target_klass`, otherwise, returns a `TypeError`. @@ -26,4 +30,6 @@ def try_cast_pb(value: AsProtobuf, target_klass: Type[T]) -> T: value_pb = value.as_pb(target_klass) if isinstance(value_pb, target_klass): return cast(target_klass, value_pb) - raise TypeError(f"Expected a '{target_klass.__module__}{target_klass.__name__}' but got {value.__module__}{value.__class__.__name__}") + raise TypeError( + f"Expected a '{target_klass.__module__}{target_klass.__name__}' but got {value.__module__}{value.__class__.__name__}" + ) diff --git a/python/lib/sift_internal/types.py b/python/lib/sift_internal/types.py index 1ab5eebb..a761e6c6 100644 --- a/python/lib/sift_internal/types.py +++ b/python/lib/sift_internal/types.py @@ -2,6 +2,7 @@ T = TypeVar("T") + def any_as(value: Any, target_klass: Type[T]) -> Optional[T]: """ Attempts to convert `value` of type `Any` to `target_klass`, otherwise return `None`. diff --git a/python/lib/sift_py/ingestion/channel.py b/python/lib/sift_py/ingestion/channel.py index a19d996e..f044d4be 100644 --- a/python/lib/sift_py/ingestion/channel.py +++ b/python/lib/sift_py/ingestion/channel.py @@ -2,7 +2,9 @@ from google.protobuf.empty_pb2 import Empty from sift_internal.convert.protobuf import AsProtobuf, ProtobufMessage, try_cast_pb from enum import Enum -from sift.common.type.v1.channel_enum_type_pb2 import ChannelEnumType as ChannelEnumTypePb +from sift.common.type.v1.channel_enum_type_pb2 import ( + ChannelEnumType as ChannelEnumTypePb, +) from sift.common.type.v1.channel_bit_field_element_pb2 import ( ChannelBitFieldElement as ChannelBitFieldElementPb, ) diff --git a/python/lib/sift_py/ingestion/config/__init__.py b/python/lib/sift_py/ingestion/config/__init__.py new file mode 100644 index 00000000..517431d7 --- /dev/null +++ b/python/lib/sift_py/ingestion/config/__init__.py @@ -0,0 +1,43 @@ +""" +Contains the in memory representation of a telemetry config used to configure ingestion. +""" + +from __future__ import annotations +from ..flow import FlowConfig +from typing import List, Optional + + +class TelemetryConfig: + """ + Configurations necessary to start ingestion. + + Attributes: + asset_name: The name of the asset that you wish to telemeter data for. + ingestion_client_key: An arbitrary string completely chosen by the user to uniquely identify + this ingestion configuration. It should be unique with respect to your + organization. + + flows: The list of `FlowConfig`. A single flow can specify a single channel value + or a set of channel values, with each value belonging to a different channel. Channels + that send data at the same frequency and time should be in the same flow. + + organization_id: ID of your organization in Sift. This field is only required if your user + belongs to multiple organizations + """ + + asset_name: str + ingestion_client_key: str + organization_id: Optional[str] + flows: List[FlowConfig] + + def __init__( + self, + asset_name: str, + ingestion_client_key: str, + organization_id: Optional[str] = None, + flows: List[FlowConfig] = [], + ): + self.asset_name = asset_name + self.ingestion_client_key = ingestion_client_key + self.organization_id = organization_id + self.flows = flows diff --git a/python/lib/sift_py/ingestion/config.py b/python/lib/sift_py/ingestion/config/yaml.py similarity index 74% rename from python/lib/sift_py/ingestion/config.py rename to python/lib/sift_py/ingestion/config/yaml.py index 12555f8c..476985f6 100644 --- a/python/lib/sift_py/ingestion/config.py +++ b/python/lib/sift_py/ingestion/config/yaml.py @@ -1,55 +1,15 @@ -""" -Contains the in memory representation of a telemetry config as well as tools to initialize one -via a YAML file and future file formats in the future. -""" - from __future__ import annotations -from .channel import ChannelDataType, ChannelBitFieldElement, ChannelEnumType -from .error import YamlConfigError -from .flow import ChannelConfig, FlowConfig +from ..channel import ChannelDataType, ChannelBitFieldElement, ChannelEnumType +from ..error import YamlConfigError +from ..flow import ChannelConfig, FlowConfig from pathlib import Path from sift_internal.types import any_as -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List +from . import TelemetryConfig import yaml -class TelemetryConfig: - """ - Configurations necessary to start ingestion. - - Attributes: - asset_name: The name of the asset that you wish to telemeter data for. - ingestion_client_key: An arbitrary string completely chosen by the user to uniquely identify - this ingestion configuration. It should be unique with respect to your - organization. - - flows: The list of `FlowConfig`. A single flow can specify a single channel value - or a set of channel values, with each value belonging to a different channel. Channels - that send data at the same frequency and time should be in the same flow. - - organization_id: ID of your organization in Sift. This field is only required if your user - belongs to multiple organizations - """ - - asset_name: str - ingestion_client_key: str - organization_id: Optional[str] - flows: List[FlowConfig] - - def __init__( - self, - asset_name: str, - ingestion_client_key: str, - organization_id: Optional[str] = None, - flows: List[FlowConfig] = [], - ): - self.asset_name = asset_name - self.ingestion_client_key = ingestion_client_key - self.organization_id = organization_id - self.flows = flows - - def try_load_from_yaml(config_fs_path: Path) -> TelemetryConfig: """ Loads in YAML config file and deserializes it into an instance of `TelemetryConfig`. If @@ -80,6 +40,8 @@ def _try_from_yaml_str(yaml_str: str) -> TelemetryConfig: organization_id = any_as(config.get("organization_id"), str) + # TODO... parse channels top-level first before flows then assign to flows. + raw_flows = any_as(config.get("flows"), list) if raw_flows is None: raise YamlConfigError("Expected 'flows' to be a list property") @@ -105,7 +67,8 @@ def _deserialize_flows_from_yaml(raw_flow_configs: List[Dict]) -> List[FlowConfi raise YamlConfigError("Expected 'channels' to be a list property") flow_config = FlowConfig( - name=flow_name, channels=_deserialize_channels_from_yaml(raw_channel_configs) + name=flow_name, + channels=_deserialize_channels_from_yaml(raw_channel_configs), ) flow_configs.append(flow_config) @@ -113,7 +76,9 @@ def _deserialize_flows_from_yaml(raw_flow_configs: List[Dict]) -> List[FlowConfi return flow_configs -def _deserialize_channels_from_yaml(raw_channel_configs: List[Dict]) -> List[ChannelConfig]: +def _deserialize_channels_from_yaml( + raw_channel_configs: List[Dict], +) -> List[ChannelConfig]: channel_configs = [] for raw_channel_config in raw_channel_configs: @@ -162,7 +127,9 @@ def _deserialize_channels_from_yaml(raw_channel_configs: List[Dict]) -> List[Cha return channel_configs -def _deserialize_bit_field_element_from_yaml(bit_field_element: Dict) -> ChannelBitFieldElement: +def _deserialize_bit_field_element_from_yaml( + bit_field_element: Dict, +) -> ChannelBitFieldElement: name = any_as(bit_field_element.get("name"), str) if name is None or len(name) == 0: raise YamlConfigError( diff --git a/python/lib/sift_py/ingestion/config_test.py b/python/lib/sift_py/ingestion/config/yaml_test.py similarity index 98% rename from python/lib/sift_py/ingestion/config_test.py rename to python/lib/sift_py/ingestion/config/yaml_test.py index 811a37c3..ac43b70b 100644 --- a/python/lib/sift_py/ingestion/config_test.py +++ b/python/lib/sift_py/ingestion/config/yaml_test.py @@ -1,6 +1,6 @@ from __future__ import annotations -from .config import _try_from_yaml_str -from .channel import ChannelDataType +from .yaml import _try_from_yaml_str +from ..channel import ChannelDataType def test_telemetry_config(): diff --git a/python/lib/sift_py/ingestion/ingestion_impl/__init__.py b/python/lib/sift_py/ingestion/impl/__init__.py similarity index 100% rename from python/lib/sift_py/ingestion/ingestion_impl/__init__.py rename to python/lib/sift_py/ingestion/impl/__init__.py diff --git a/python/lib/sift_py/ingestion/ingestion_impl/ingest.py b/python/lib/sift_py/ingestion/impl/ingest.py similarity index 98% rename from python/lib/sift_py/ingestion/ingestion_impl/ingest.py rename to python/lib/sift_py/ingestion/impl/ingest.py index bedff174..bdaff657 100644 --- a/python/lib/sift_py/ingestion/ingestion_impl/ingest.py +++ b/python/lib/sift_py/ingestion/impl/ingest.py @@ -1,7 +1,10 @@ from __future__ import annotations from ..channel import ChannelValue, is_data_type, empty_value from ..flow import FlowConfig -from .ingestion_config import get_ingestion_config_by_client_key, create_ingestion_config +from .ingestion_config import ( + get_ingestion_config_by_client_key, + create_ingestion_config, +) from ..config import TelemetryConfig from ...grpc.transport import SiftChannel from sift.ingestion_configs.v1.ingestion_configs_pb2 import IngestionConfig @@ -36,7 +39,6 @@ def __init__( run_id: Optional[str] = None, end_stream_on_error: bool = False, ): - self.ingestion_config = self.__class__.__get_or_create_ingestion_config(channel, config) self.asset_name = config.asset_name self.transport_channel = channel diff --git a/python/lib/sift_py/ingestion/ingestion_impl/ingestion_config.py b/python/lib/sift_py/ingestion/impl/ingestion_config.py similarity index 98% rename from python/lib/sift_py/ingestion/ingestion_impl/ingestion_config.py rename to python/lib/sift_py/ingestion/impl/ingestion_config.py index 33cf8fb7..8f889911 100644 --- a/python/lib/sift_py/ingestion/ingestion_impl/ingestion_config.py +++ b/python/lib/sift_py/ingestion/impl/ingestion_config.py @@ -14,7 +14,9 @@ ListIngestionConfigsResponse, FlowConfig as FlowConfigPb, ) -from sift.ingestion_configs.v1.ingestion_configs_pb2_grpc import IngestionConfigServiceStub +from sift.ingestion_configs.v1.ingestion_configs_pb2_grpc import ( + IngestionConfigServiceStub, +) from typing import cast, List, Optional diff --git a/python/lib/sift_py/ingestion/service.py b/python/lib/sift_py/ingestion/service.py index 6fd00007..c49c566d 100644 --- a/python/lib/sift_py/ingestion/service.py +++ b/python/lib/sift_py/ingestion/service.py @@ -9,7 +9,7 @@ ) from sift.ingestion_configs.v1.ingestion_configs_pb2 import IngestionConfig from typing import Dict, List, Optional -from .ingestion_impl.ingest import IngestionServiceImpl +from .impl.ingest import IngestionServiceImpl from datetime import datetime diff --git a/python/pyproject.toml b/python/pyproject.toml index 072e6083..30c2e1a3 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ [project.optional-dependencies] development = [ "pytest", # test framework - "black", # formatter + "ruff", # formatter + linter ] [build-system] @@ -27,8 +27,38 @@ build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] where = ["gen", "lib"] -# The formatter -[tool.black] +[tool.ruff] line-length = 100 -target-version = ['py38'] -include = '\.pyi?$' +indent-width = 4 +target-version = "py38" # Python 3.8 +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", + "lib/sift", + "lib/google", + "lib/protoc_gen_openapiv2", +] diff --git a/python/scripts/dev b/python/scripts/dev index fe812ca1..0568a5f8 100755 --- a/python/scripts/dev +++ b/python/scripts/dev @@ -10,7 +10,8 @@ Subcommands: test Execute tests bootstrap Initializes python virtual environment and installs project dependencies pip-install Install project dependencies - fmt Runs 'black' to format the lib directory + fmt Runs 'ruff' to format the lib directory + lint Runs 'ruff' to lint the lib directory Options: -h, --help Print usage text @@ -30,7 +31,12 @@ bootstrap() { fmt() { source venv/bin/activate - black lib/sift_py + ruff format +} + +lint() { + source venv/bin/activate + ruff check } run_tests() { @@ -53,6 +59,9 @@ case "$1" in fmt) fmt ;; + lint) + lint + ;; test) run_tests ;;