chore(python): replace black with ruff for formatting + linting (#43)

.github/workflows/python_ci.yaml

@@ -18,16 +18,26 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: "3.9"
+          python-version: "3.8"
 
-      - name: Install dependencies
+      - name: Pip install
         working-directory: python
         run: |
           python -m pip install --upgrade pip
           pip install '.[development]'
           pip install -e .
 
-      - name: Run pytest
+      - name: Lint
+        working-directory: python
+        run: |
+          ruff check
+          
+      - name: Format
+        working-directory: python
+        run: |
+          ruff format --check
+
+      - name: Pytest
         working-directory: python
         run: |
           pytest

python/lib/sift_internal/convert/protobuf.py

@@ -4,11 +4,13 @@
 
 ProtobufMessage = Message
 
+
 class AsProtobuf(ABC):
     """
     Abstract base class used to create create sub-types that can be treated
     as an object that can be converted into an instance of `ProtobufMessage`.
     """
+
     @abstractmethod
     def as_pb(self, klass: Type[ProtobufMessage]) -> Optional[ProtobufMessage]:
         """
@@ -19,11 +21,15 @@ def as_pb(self, klass: Type[ProtobufMessage]) -> Optional[ProtobufMessage]:
 
 
 T = TypeVar("T", bound=ProtobufMessage)
+
+
 def try_cast_pb(value: AsProtobuf, target_klass: Type[T]) -> T:
     """
     Tries to cast the `value` to `target_klass`, otherwise, returns a `TypeError`.
     """
     value_pb = value.as_pb(target_klass)
     if isinstance(value_pb, target_klass):
         return cast(target_klass, value_pb)
-    raise TypeError(f"Expected a '{target_klass.__module__}{target_klass.__name__}' but got {value.__module__}{value.__class__.__name__}")
+    raise TypeError(
+        f"Expected a '{target_klass.__module__}{target_klass.__name__}' but got {value.__module__}{value.__class__.__name__}"
+    )

python/lib/sift_internal/types.py

@@ -2,6 +2,7 @@
 
 T = TypeVar("T")
 
+
 def any_as(value: Any, target_klass: Type[T]) -> Optional[T]:
     """
     Attempts to convert `value` of type `Any` to `target_klass`, otherwise return `None`.

python/lib/sift_py/ingestion/channel.py

@@ -2,7 +2,9 @@
 from google.protobuf.empty_pb2 import Empty
 from sift_internal.convert.protobuf import AsProtobuf, ProtobufMessage, try_cast_pb
 from enum import Enum
-from sift.common.type.v1.channel_enum_type_pb2 import ChannelEnumType as ChannelEnumTypePb
+from sift.common.type.v1.channel_enum_type_pb2 import (
+    ChannelEnumType as ChannelEnumTypePb,
+)
 from sift.common.type.v1.channel_bit_field_element_pb2 import (
     ChannelBitFieldElement as ChannelBitFieldElementPb,
 )

python/lib/sift_py/ingestion/config/__init__.py

@@ -0,0 +1,43 @@
+"""
+Contains the in memory representation of a telemetry config used to configure ingestion.
+"""
+
+from __future__ import annotations
+from ..flow import FlowConfig
+from typing import List, Optional
+
+
+class TelemetryConfig:
+    """
+    Configurations necessary to start ingestion.
+
+    Attributes:
+      asset_name: The name of the asset that you wish to telemeter data for.
+      ingestion_client_key: An arbitrary string completely chosen by the user to uniquely identify
+                  this ingestion configuration. It should be unique with respect to your
+                  organization.
+
+      flows: The list of `FlowConfig`. A single flow can specify a single channel value
+             or a set of channel values, with each value belonging to a different channel. Channels
+             that send data at the same frequency and time should be in the same flow.
+
+      organization_id: ID of your organization in Sift. This field is only required if your user
+                       belongs to multiple organizations
+    """
+
+    asset_name: str
+    ingestion_client_key: str
+    organization_id: Optional[str]
+    flows: List[FlowConfig]
+
+    def __init__(
+        self,
+        asset_name: str,
+        ingestion_client_key: str,
+        organization_id: Optional[str] = None,
+        flows: List[FlowConfig] = [],
+    ):
+        self.asset_name = asset_name
+        self.ingestion_client_key = ingestion_client_key
+        self.organization_id = organization_id
+        self.flows = flows

python/lib/sift_py/ingestion/config.py → python/lib/sift_py/ingestion/config/yaml.py

@@ -1,55 +1,15 @@
-"""
-Contains the in memory representation of a telemetry config as well as tools to initialize one
-via a YAML file and future file formats in the future.
-"""
-
 from __future__ import annotations
-from .channel import ChannelDataType, ChannelBitFieldElement, ChannelEnumType
-from .error import YamlConfigError
-from .flow import ChannelConfig, FlowConfig
+from ..channel import ChannelDataType, ChannelBitFieldElement, ChannelEnumType
+from ..error import YamlConfigError
+from ..flow import ChannelConfig, FlowConfig
 from pathlib import Path
 from sift_internal.types import any_as
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
+from . import TelemetryConfig
 
 import yaml
 
 
-class TelemetryConfig:
-    """
-    Configurations necessary to start ingestion.
-
-    Attributes:
-      asset_name: The name of the asset that you wish to telemeter data for.
-      ingestion_client_key: An arbitrary string completely chosen by the user to uniquely identify
-                  this ingestion configuration. It should be unique with respect to your
-                  organization.
-
-      flows: The list of `FlowConfig`. A single flow can specify a single channel value
-             or a set of channel values, with each value belonging to a different channel. Channels
-             that send data at the same frequency and time should be in the same flow.
-
-      organization_id: ID of your organization in Sift. This field is only required if your user
-                       belongs to multiple organizations
-    """
-
-    asset_name: str
-    ingestion_client_key: str
-    organization_id: Optional[str]
-    flows: List[FlowConfig]
-
-    def __init__(
-        self,
-        asset_name: str,
-        ingestion_client_key: str,
-        organization_id: Optional[str] = None,
-        flows: List[FlowConfig] = [],
-    ):
-        self.asset_name = asset_name
-        self.ingestion_client_key = ingestion_client_key
-        self.organization_id = organization_id
-        self.flows = flows
-
-
 def try_load_from_yaml(config_fs_path: Path) -> TelemetryConfig:
     """
     Loads in YAML config file and deserializes it into an instance of `TelemetryConfig`. If
@@ -80,6 +40,8 @@ def _try_from_yaml_str(yaml_str: str) -> TelemetryConfig:
 
     organization_id = any_as(config.get("organization_id"), str)
 
+    # TODO... parse channels top-level first before flows then assign to flows.
+
     raw_flows = any_as(config.get("flows"), list)
     if raw_flows is None:
         raise YamlConfigError("Expected 'flows' to be a list property")
@@ -105,15 +67,18 @@ def _deserialize_flows_from_yaml(raw_flow_configs: List[Dict]) -> List[FlowConfi
             raise YamlConfigError("Expected 'channels' to be a list property")
 
         flow_config = FlowConfig(
-            name=flow_name, channels=_deserialize_channels_from_yaml(raw_channel_configs)
+            name=flow_name,
+            channels=_deserialize_channels_from_yaml(raw_channel_configs),
         )
 
         flow_configs.append(flow_config)
 
     return flow_configs
 
 
-def _deserialize_channels_from_yaml(raw_channel_configs: List[Dict]) -> List[ChannelConfig]:
+def _deserialize_channels_from_yaml(
+    raw_channel_configs: List[Dict],
+) -> List[ChannelConfig]:
     channel_configs = []
 
     for raw_channel_config in raw_channel_configs:
@@ -162,7 +127,9 @@ def _deserialize_channels_from_yaml(raw_channel_configs: List[Dict]) -> List[Cha
     return channel_configs
 
 
-def _deserialize_bit_field_element_from_yaml(bit_field_element: Dict) -> ChannelBitFieldElement:
+def _deserialize_bit_field_element_from_yaml(
+    bit_field_element: Dict,
+) -> ChannelBitFieldElement:
     name = any_as(bit_field_element.get("name"), str)
     if name is None or len(name) == 0:
         raise YamlConfigError(

python/lib/sift_py/ingestion/config_test.py → python/lib/sift_py/ingestion/config/yaml_test.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
-from .config import _try_from_yaml_str
-from .channel import ChannelDataType
+from .yaml import _try_from_yaml_str
+from ..channel import ChannelDataType
 
 
 def test_telemetry_config():

python/lib/sift_py/ingestion/ingestion_impl/ingest.py → python/lib/sift_py/ingestion/impl/ingest.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 from ..channel import ChannelValue, is_data_type, empty_value
 from ..flow import FlowConfig
-from .ingestion_config import get_ingestion_config_by_client_key, create_ingestion_config
+from .ingestion_config import (
+    get_ingestion_config_by_client_key,
+    create_ingestion_config,
+)
 from ..config import TelemetryConfig
 from ...grpc.transport import SiftChannel
 from sift.ingestion_configs.v1.ingestion_configs_pb2 import IngestionConfig
@@ -36,7 +39,6 @@ def __init__(
         run_id: Optional[str] = None,
         end_stream_on_error: bool = False,
     ):
-
         self.ingestion_config = self.__class__.__get_or_create_ingestion_config(channel, config)
         self.asset_name = config.asset_name
         self.transport_channel = channel

python/lib/sift_py/ingestion/ingestion_impl/ingestion_config.py → python/lib/sift_py/ingestion/impl/ingestion_config.py

@@ -14,7 +14,9 @@
     ListIngestionConfigsResponse,
     FlowConfig as FlowConfigPb,
 )
-from sift.ingestion_configs.v1.ingestion_configs_pb2_grpc import IngestionConfigServiceStub
+from sift.ingestion_configs.v1.ingestion_configs_pb2_grpc import (
+    IngestionConfigServiceStub,
+)
 from typing import cast, List, Optional
 
 

python/lib/sift_py/ingestion/service.py

@@ -9,7 +9,7 @@
 )
 from sift.ingestion_configs.v1.ingestion_configs_pb2 import IngestionConfig
 from typing import Dict, List, Optional
-from .ingestion_impl.ingest import IngestionServiceImpl
+from .impl.ingest import IngestionServiceImpl
 from datetime import datetime
 
 

python/pyproject.toml

@@ -17,7 +17,7 @@ dependencies = [
 [project.optional-dependencies]
 development = [
   "pytest", # test framework
-  "black", # formatter
+  "ruff", # formatter + linter
 ]
 
 [build-system]
@@ -27,8 +27,38 @@ build-backend = "setuptools.build_meta"
 [tool.setuptools.packages.find]
 where = ["gen", "lib"]
 
-# The formatter
-[tool.black]
+[tool.ruff]
 line-length = 100
-target-version = ['py38']
-include = '\.pyi?$'
+indent-width = 4
+target-version = "py38" # Python 3.8
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+    "lib/sift",
+    "lib/google",
+    "lib/protoc_gen_openapiv2",
+]

python/scripts/dev

@@ -10,7 +10,8 @@ Subcommands:
   test           Execute tests
   bootstrap      Initializes python virtual environment and installs project dependencies
   pip-install    Install project dependencies
-  fmt            Runs 'black' to format the lib directory
+  fmt            Runs 'ruff' to format the lib directory
+  lint           Runs 'ruff' to lint the lib directory
 
 Options:
   -h, --help     Print usage text
@@ -30,7 +31,12 @@ bootstrap() {
 
 fmt() {
   source venv/bin/activate
-  black lib/sift_py
+  ruff format
+}
+
+lint() {
+  source venv/bin/activate
+  ruff check
 }
 
 run_tests() {
@@ -53,6 +59,9 @@ case "$1" in
     fmt)
         fmt
         ;;
+    lint)
+        lint
+        ;;
     test)
         run_tests
         ;;