python(chore): changelog for rc.2

python/CHANGELOG.md

@@ -0,0 +1,158 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+
+This project adheres to [Semantic Versioning](http://semver.org/).
+
+## [v0.1.0-rc.2] - July 1, 2024
+
+Summary of changes:
+- [Introduced automated ingestion request buffering to improve performance](https://github.com/sift-stack/sift/pull/65)
+- [Added support for multi-config ingestion and creating new flows at run-time](https://github.com/sift-stack/sift/pull/66)
+- Added methods that combine request creation and ingestion into a single-step.
+- Updates to documentation.
+
+For in-depth documentation please see the [documentation section of the README](https://github.com/sift-stack/sift/tree/main/python#documentation) for instructions
+on how to build the documentation locally.
+
+### Table of Contents
+- [Combining Request Generation and Ingestion into a Single Step](#combining-request-generation-and-ingestion-into-a-single-step)
+- [Request Buffering](#request-buffering)
+- [Creating New Flows on the Fly](#creating-new-flows-on-the-fly)
+- [Multi-config Ingestion](#multi-config-ingestion)
+
+### Combining Request Generation and Ingestion into a Single Step
+
+Previously to ingest a single request you needed to do the following:
+
+```python
+request = ingestion_service.try_create_ingestion_request(
+    flow_name="logs",
+    timestamp=timestamp,
+    channel_values=[
+        {
+            "channel_name": "log",
+            "value": string_value("some log"),
+        },
+    ],
+)
+ingestion_service.ingest(request)
+```
+
+Now you can combine both steps using either of the following APIs:
+- [try_ingest_flows](https://github.com/sift-stack/sift/blob/e7e59e63344059fb232ce883d269c479e1857f09/python/lib/sift_py/ingestion/service.py#L145)
+- [ingest_flows](https://github.com/sift-stack/sift/blob/e7e59e63344059fb232ce883d269c479e1857f09/python/lib/sift_py/ingestion/service.py#L138)
+
+```python
+ingestion_service.try_ingest_flows({
+    "flow_name": "log",
+    "timestamp": timestamp,
+    "channel_values": [
+        {
+            "channel_name": "log",
+            "value": string_value("some string")
+        },
+    ],
+})
+```
+
+You can also send multiple flows:
+
+```python
+# Send data for both logs and readings flows
+ingestion_service.try_ingest_flows(
+    {
+        "flow_name": "readings",
+        "timestamp": datetime.now(timezone.utc),
+        "channel_values": [
+            {
+                "channel_name": "velocity",
+                "component": "mainmotor",
+                "value": double_value(10),
+            },
+            {
+                "channel_name": "voltage",
+                "value": int32_value(5),
+            },
+            {
+                "channel_name": "vehicle_state",
+                "value": enum_value(2),
+            },
+        ],
+    },
+    {
+        "flow_name": "logs",
+        "timestamp": datetime.now(timezone.utc),
+        "channel_values": [
+            {
+                "channel_name": "logs",
+                "value": string_value("INFO: some message")
+            },
+        ],
+    },
+)
+```
+
+### Request Buffering
+
+Requests are now automatically buffered using the [buffered ingestion API](https://github.com/sift-stack/sift/blob/1fbd2eb02d484b277a8b799940587b0f11e291da/python/lib/sift_py/ingestion/service.py#L152).
+Using this may significantly improve performance as it allows serialization and ingestion to occur in batches:
+
+```python
+# Defaults to a buffer size of `sift_py.ingestion.buffer.DEFAULT_BUFFER_SIZE` requests.
+with ingestion_service.buffered_ingestion() as buffered_ingestion:
+    buffered_ingestion.try_ingest_flows(*lots_of_flows)
+    buffered_ingestion.try_ingest_flows(*lots_more_flows)
+
+# Custom buffer size of 750 requests
+with ingestion_service.buffered_ingestion(750) as buffered_ingestion:
+    buffered_ingestion.try_ingest_flows(*lots_of_flows)
+    buffered_ingestion.try_ingest_flows(*lots_more_flows)
+```
+
+Once the with-block ends the remaining requests will be automatically flushed and ingested, but flushing may also
+be done manually:
+
+```python
+with ingestion_service.buffered_ingestion() as buffered_ingestion:
+    buffered_ingestion.try_ingest_flows(*lots_of_flows)
+    buffered_ingestion.flush()
+```
+
+Contrast this with regular ingestion:
+
+```python
+ingestion_service.try_ingest_flows(*lots_of_flows)
+ingestion_service.try_ingest_flows(*lots_more_flows)
+```
+
+### Creating New Flows on the Fly
+
+If there is a flow you need to create on the fly which wasn't declared in your initial telemetry config,
+you may use either of the following APIs:
+- `try_create_flow`
+- `create_flow`
+
+```python
+new_flow_config = FlowConfig(
+    name="my_new_flow", channels=[ChannelConfig("new_channel", ChannelDataType.DOUBLE)]
+)
+ingestion_service.try_create_flow(new_flow_config)
+```
+
+### Multi-config Ingestion
+
+There is now an ergonomic utility class, [IngestionServicesManager](https://github.com/sift-stack/sift/blob/e7e59e63344059fb232ce883d269c479e1857f09/python/lib/sift_py/ingestion/manager.py#L15),
+that allows users to manage telemetry for multiple configs:
+
+```python
+manager = IngestionServicesManager.from_telementry_configs(grpc_channel, {
+    "config_a": config_a,
+    "config_b": config_b,
+})
+
+with manager.ingestion_service("config_a") as config_a:
+    config_a.try_ingest_flow(...)
+
+with manager.ingestion_service("config_b") as config_b:
+    config_b.try_ingest_flow(...)
+```

python/lib/sift_py/__init__.py

@@ -108,7 +108,7 @@
     )
 
     # Send data to Sift for the 'temperature_reading' flow
-    temperature_reading = self.ingestion_service.try_ingest_flows({
+    ingestion_service.try_ingest_flows({
         "flow_name": "temperature_reading",
         "timestamp": datetime.now(timezone.utc),
         "channel_values": [
@@ -674,7 +674,7 @@ def nostromos_lv_426() -> TelemetryConfig:
             ],
         },
         {
-            "flow_name": "readings",
+            "flow_name": "logs",
             "timestamp": datetime.now(timezone.utc),
             "channel_values": [
                 {

python/lib/sift_py/ingestion/_internal/ingest.py

@@ -231,11 +231,11 @@ def create_ingestion_request(
             end_stream_on_validation_error=self.end_stream_on_error,
         )
 
-    def try_create_new_flow(self, flow_config: FlowConfig):
+    def try_create_flow(self, flow_config: FlowConfig):
         """
         Tries to create a new flow at runtime. Will raise an `IngestionValidationError` if there already exists
         a flow with the name of the `flow_config` argument. If you'd like to overwrite any flow configs with that
-        have the same name as the provided `flow_config`, then see `create_new_flow`.
+        have the same name as the provided `flow_config`, then see `create_flow`.
         """
 
         if flow_config.name in self.flow_configs_by_name:
@@ -251,10 +251,10 @@ def try_create_new_flow(self, flow_config: FlowConfig):
 
         self.flow_configs_by_name[flow_config.name] = flow_config
 
-    def create_new_flow(self, flow_config: FlowConfig):
+    def create_flow(self, flow_config: FlowConfig):
         """
-        Like `try_create_new_flow` but will automatically overwrite any existing flow config with `flow_config` if they
-        share the same name. If you'd an exception to be raise in the case of a name collision then see `try_create_new_flow`.
+        Like `try_create_flow` but will automatically overwrite any existing flow config with `flow_config` if they
+        share the same name. If you'd an exception to be raise in the case of a name collision then see `try_create_flow`.
         """
         create_flow_configs(
             self.transport_channel,

python/lib/sift_py/ingestion/_service_test.py

@@ -314,7 +314,7 @@ def test_ingestion_service_register_new_flow(mocker: MockFixture):
 
     assert ingestion_service.flow_configs_by_name.get("my_new_flow") is None
 
-    ingestion_service.try_create_new_flow(new_flow_config)
+    ingestion_service.try_create_flow(new_flow_config)
 
     mock_create_flow_configs.assert_called_once_with(
         mock_channel, mock_ingestion_config.ingestion_config_id, [new_flow_config]
@@ -327,9 +327,9 @@ def test_ingestion_service_register_new_flow(mocker: MockFixture):
     )
 
     with pytest.raises(IngestionValidationError):
-        ingestion_service.try_create_new_flow(new_flow_config_name_collision)
+        ingestion_service.try_create_flow(new_flow_config_name_collision)
 
     # Bypass the validation
-    ingestion_service.create_new_flow(new_flow_config_name_collision)
+    ingestion_service.create_flow(new_flow_config_name_collision)
     assert ingestion_service.flow_configs_by_name["my_new_flow"] == new_flow_config_name_collision
     assert ingestion_service.flow_configs_by_name["my_new_flow"] != new_flow_config

python/lib/sift_py/ingestion/service.py

@@ -188,17 +188,17 @@ def buffered_ingestion(self, buffer_size: Optional[int] = None) -> BufferedInges
         """
         return BufferedIngestionService(self, buffer_size)
 
-    def create_new_flow(self, flow_config: FlowConfig):
+    def create_flow(self, flow_config: FlowConfig):
         """
         Like `try_create_new_flow` but will automatically overwrite any existing flow config with `flow_config` if they
         share the same name. If you'd an exception to be raise in the case of a name collision then see `try_create_new_flow`.
         """
-        super().create_new_flow(flow_config)
+        super().create_flow(flow_config)
 
-    def try_create_new_flow(self, flow_config: FlowConfig):
+    def try_create_flow(self, flow_config: FlowConfig):
         """
         Tries to create a new flow at runtime. Will raise an `IngestionValidationError` if there already exists
         a flow with the name of the `flow_config` argument. If you'd like to overwrite any flow configs with that
         have the same name as the provided `flow_config`, then see `create_new_flow`.
         """
-        super().try_create_new_flow(flow_config)
+        super().try_create_flow(flow_config)

python/pyproject.toml

@@ -37,6 +37,7 @@ dependencies = [
 Homepage = "https://github.com/sift-stack/sift/tree/main/python"
 Documentation = "https://docs.siftstack.com/"
 Repository = "https://github.com/sift-stack/sift/tree/main/python"
+Changelog = "https://github.com/sift-stack/sift/tree/main/python/CHANGELOG.md"
 
 [project.optional-dependencies]
 development = [