Port 5369 ocean handle request http errors change merge to true by de…

…fault (#249)

* Changed the port request options defaults to be constructed in the port app config model instead of setting the defaults in many places

* Changed port request option `merge` to be true by default

* changelogs

CHANGELOG.md

@@ -7,6 +7,19 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+## 0.4.5 (2023-11-30)
+
+
+### Features
+
+- Added handling for transport errors like connection timeout error for outbound requests from ocean integrations and core (PORT-5369)
+- Changed port request option `merge` to be true by default (PORT-5396)
+
+### Improvements
+
+- Changed the port request options defaults to be constructed in the port app config model instead of setting the defaults in many places (PORT-5369)
+
+
 ## 0.4.4 (2023-11-29)
 
 

port_ocean/clients/port/mixins/entities.py

@@ -21,7 +21,7 @@ async def upsert_entity(
         user_agent_type: UserAgentType | None = None,
         should_raise: bool = True,
     ) -> None:
-        validation_only = request_options.get("validation_only", False)
+        validation_only = request_options["validation_only"]
         logger.info(
             f"{'Validating' if validation_only else 'Upserting'} entity: {entity.identifier} of blueprint: {entity.blueprint}"
         )
@@ -33,9 +33,9 @@ async def upsert_entity(
             headers=headers,
             params={
                 "upsert": "true",
-                "merge": str(request_options.get("merge", False)).lower(),
+                "merge": str(request_options["merge"]).lower(),
                 "create_missing_related_entities": str(
-                    request_options.get("create_missing_related_entities", True)
+                    request_options["create_missing_related_entities"]
                 ).lower(),
                 "validation_only": str(validation_only).lower(),
             },
@@ -64,7 +64,7 @@ async def delete_entity(
             headers=await self.auth.headers(user_agent_type),
             params={
                 "delete_dependents": str(
-                    request_options.get("delete_dependent_entities", True)
+                    request_options["delete_dependent_entities"]
                 ).lower()
             },
         )
@@ -142,16 +142,15 @@ async def search_dependent_entities(self, entity: Entity) -> list[Entity]:
         return [Entity.parse_obj(result) for result in response.json()["entities"]]
 
     async def validate_entity_payload(
-        self, entity: Entity, options: RequestOptions
+        self, entity: Entity, merge: bool, create_missing_related_entities: bool
     ) -> None:
         logger.info(f"Validating entity {entity.identifier}")
         await self.upsert_entity(
             entity,
             {
-                "merge": options.get("merge", False),
-                "create_missing_related_entities": options.get(
-                    "create_missing_related_entities", True
-                ),
+                "merge": merge,
+                "create_missing_related_entities": create_missing_related_entities,
+                "delete_dependent_entities": False,
                 "validation_only": True,
             },
         )

port_ocean/clients/port/types.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import TypedDict, NotRequired
+from typing import TypedDict
 
 
 class UserAgentType(Enum):
@@ -18,11 +18,9 @@ class UserAgentType(Enum):
 RequestOptions = TypedDict(
     "RequestOptions",
     {
-        "merge": NotRequired[bool],
-        "create_missing_related_entities": NotRequired[bool],
-        "delete_dependent_entities": NotRequired[bool],
-        "validation_only": NotRequired[bool],
-        "upsert": NotRequired[bool],
-        "user_agent": NotRequired[str],
+        "merge": bool,
+        "create_missing_related_entities": bool,
+        "delete_dependent_entities": bool,
+        "validation_only": bool,
     },
 )

port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -67,10 +67,8 @@ async def _validate_entity_diff(self, diff: EntityPortDiff) -> None:
                 *(
                     self.context.port_client.validate_entity_payload(
                         entity,
-                        {
-                            "merge": config.enable_merge_entity,
-                            "create_missing_related_entities": config.create_missing_related_entities,
-                        },
+                        config.enable_merge_entity,
+                        create_missing_related_entities=config.create_missing_related_entities,
                     )
                     for entity in modified_or_created_entities
                 )

port_ocean/core/handlers/port_app_config/models.py

@@ -31,7 +31,7 @@ class ResourceConfig(BaseModel):
 
 
 class PortAppConfig(BaseModel):
-    enable_merge_entity: bool = Field(alias="enableMergeEntity", default=False)
+    enable_merge_entity: bool = Field(alias="enableMergeEntity", default=True)
     delete_dependent_entities: bool = Field(
         alias="deleteDependentEntities", default=True
     )
@@ -45,6 +45,7 @@ def get_port_request_options(self) -> RequestOptions:
             "delete_dependent_entities": self.delete_dependent_entities,
             "create_missing_related_entities": self.create_missing_related_entities,
             "merge": self.enable_merge_entity,
+            "validation_only": False,
         }
 
     def to_request(self) -> dict[str, Any]:

port_ocean/helpers/retry.py

@@ -181,6 +181,24 @@ def close(self) -> None:
     def _should_retry(self, response: httpx.Response) -> bool:
         return response.status_code in self._retry_status_codes
 
+    def _log_failure(
+        self,
+        request: httpx.Request,
+        sleep_time: float,
+        response: httpx.Response | None,
+        error: Exception | None,
+    ) -> None:
+        if self._logger and response:
+            self._logger.warning(
+                f"Request {request.method} {request.url} failed with status code:"
+                f" {response.status_code}, retrying in {sleep_time} seconds."  # noqa: F821
+            )
+        elif self._logger and error:
+            self._logger.warning(
+                f"Request {request.method} {request.url} failed with exception:"
+                f" {type(error).__name__} - {str(error) or 'No error message'}, retrying in {sleep_time} seconds."
+            )
+
     async def _should_retry_async(self, response: httpx.Response) -> bool:
         return response.status_code in self._retry_status_codes
 
@@ -223,21 +241,28 @@ async def _retry_operation_async(
     ) -> httpx.Response:
         remaining_attempts = self._max_attempts
         attempts_made = 0
+        response: httpx.Response | None = None
+        error: Exception | None = None
         while True:
-            response: httpx.Response
             if attempts_made > 0:
                 sleep_time = self._calculate_sleep(attempts_made, {})
-                if self._logger:
-                    self._logger.warning(
-                        f"Request {request.method} {request.url} failed with status code:"
-                        f" {response.status_code}, retrying in {sleep_time} seconds."  # noqa: F821
-                    )
+                self._log_failure(request, sleep_time, response, error)
                 await asyncio.sleep(sleep_time)
-            response = await send_method(request)
-            response.request = request
-            if remaining_attempts < 1 or not (await self._should_retry_async(response)):
-                return response
-            await response.aclose()
+
+            error = None
+            response = None
+            try:
+                response = await send_method(request)
+                response.request = request
+                if remaining_attempts < 1 or not (
+                    await self._should_retry_async(response)
+                ):
+                    return response
+                await response.aclose()
+            except httpx.HTTPError as e:
+                error = e
+                if remaining_attempts < 1:
+                    raise
             attempts_made += 1
             remaining_attempts -= 1
 
@@ -248,20 +273,25 @@ def _retry_operation(
     ) -> httpx.Response:
         remaining_attempts = self._max_attempts
         attempts_made = 0
+        response: httpx.Response | None = None
+        error: Exception | None = None
         while True:
-            response: httpx.Response
             if attempts_made > 0:
                 sleep_time = self._calculate_sleep(attempts_made, {})
-                if self._logger:
-                    self._logger.warning(
-                        f"Request {request.method} {request.url} failed with status code:"
-                        f" {response.status_code}, retrying in {sleep_time} seconds."  # noqa: F821
-                    )
+                self._log_failure(request, sleep_time, response, error)
                 time.sleep(sleep_time)
-            response = send_method(request)
-            response.request = request
-            if remaining_attempts < 1 or not self._should_retry(response):
-                return response
-            response.close()
+
+            error = None
+            response = None
+            try:
+                response = send_method(request)
+                response.request = request
+                if remaining_attempts < 1 or not self._should_retry(response):
+                    return response
+                response.close()
+            except httpx.HTTPError as e:
+                error = e
+                if remaining_attempts < 1:
+                    raise
             attempts_made += 1
             remaining_attempts -= 1

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.4.4"
+version = "0.4.5"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"