From 504e3a666aa78691fbfe3d065c412e49e3e4b74c Mon Sep 17 00:00:00 2001 From: shariff-6 Date: Mon, 30 Dec 2024 22:12:34 +0300 Subject: [PATCH] [Integration][Datadog] Datadog Teams and Users (#1256) --- .../datadog/.port/resources/blueprints.json | 120 +++++++++-- .../.port/resources/port-app-config.yaml | 42 ++++ integrations/datadog/.port/spec.yaml | 4 + integrations/datadog/CHANGELOG.md | 8 + integrations/datadog/client.py | 77 +++++++ integrations/datadog/main.py | 37 +++- integrations/datadog/overrides.py | 18 +- integrations/datadog/pyproject.toml | 2 +- integrations/datadog/tests/test_client.py | 188 ++++++++++++++++++ 9 files changed, 477 insertions(+), 19 deletions(-) create mode 100644 integrations/datadog/tests/test_client.py diff --git a/integrations/datadog/.port/resources/blueprints.json b/integrations/datadog/.port/resources/blueprints.json index 4f3c5e6254..11ce34c0e2 100644 --- a/integrations/datadog/.port/resources/blueprints.json +++ b/integrations/datadog/.port/resources/blueprints.json @@ -1,4 +1,100 @@ [ + { + "identifier": "datadogUser", + "description": "This blueprint represents a Datadog user account. Users can be assigned to teams, granted specific permissions, and can interact with various Datadog features based on their access levels.", + "title": "Datadog User", + "icon": "Datadog", + "schema": { + "properties": { + "email": { + "type": "string", + "format": "email", + "title": "Email", + "description": "The email address associated with the user account" + }, + "handle": { + "type": "string", + "title": "Handle", + "description": "The unique handle identifier for the user within Datadog" + }, + "status": { + "type": "string", + "title": "Status", + "description": "The current status of the user account (e.g., active, pending, disabled)" + }, + "disabled": { + "type": "boolean", + "title": "Disabled", + "description": "Indicates whether the user account is currently disabled" + }, + "verified": { + "type": "boolean", + "title": "Verified", + "description": "Indicates whether the user's email address has been verified" + }, + "createdAt": { + "type": "string", + "format": "date-time", + "title": "Created At", + "description": "The timestamp when the user account was created" + } + }, + "required": [] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "aggregationProperties": {}, + "relations": {} + }, + { + "identifier": "datadogTeam", + "description": "This blueprint represents a Datadog team", + "title": "Datadog Team", + "icon": "Datadog", + "schema": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "A description of the team's purpose and responsibilities" + }, + "handle": { + "type": "string", + "title": "Handle", + "description": "The unique handle identifier for the team within Datadog" + }, + "userCount": { + "type": "number", + "title": "User Count", + "description": "The total number of users that are members of this team" + }, + "summary": { + "type": "string", + "title": "Summary", + "description": "A brief summary of the team's purpose or main responsibilities" + }, + "createdAt": { + "type": "string", + "format": "date-time", + "title": "Created At", + "description": "The timestamp when the team was created" + } + }, + "required": [] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "aggregationProperties": {}, + "relations": { + "members": { + "target": "datadogUser", + "title": "Members", + "description": "Users who are members of this team", + "many": true, + "required": false + } + } + }, { "identifier": "datadogHost", "description": "This blueprint represents a datadog host", @@ -202,7 +298,14 @@ "mirrorProperties": {}, "calculationProperties": {}, "aggregationProperties": {}, - "relations": {} + "relations": { + "team": { + "target": "datadogTeam", + "title": "Team", + "many": false, + "required": false + } + } }, { "identifier": "datadogSlo", @@ -252,20 +355,7 @@ }, "mirrorProperties": {}, "calculationProperties": {}, - "aggregationProperties": { - "sli_average": { - "title": "SLI Average", - "type": "number", - "target": "datadogSloHistory", - "calculationSpec": { - "func": "average", - "averageOf": "total", - "property": "sliValue", - "measureTimeBy": "$createdAt", - "calculationBy": "property" - } - } - }, + "aggregationProperties": {}, "relations": { "monitors": { "title": "SLO Monitors", diff --git a/integrations/datadog/.port/resources/port-app-config.yaml b/integrations/datadog/.port/resources/port-app-config.yaml index 01d9dbb790..3a726099bc 100644 --- a/integrations/datadog/.port/resources/port-app-config.yaml +++ b/integrations/datadog/.port/resources/port-app-config.yaml @@ -1,6 +1,41 @@ deleteDependentEntities: true createMissingRelatedEntities: true resources: + - kind: user + selector: + query: 'true' + port: + entity: + mappings: + identifier: .id | tostring + title: .attributes.name + blueprint: '"datadogUser"' + properties: + email: .attributes.email + handle: .attributes.handle + status: .attributes.status + disabled: .attributes.disabled + verified: .attributes.verified + createdAt: .attributes.created_at | todate + - kind: team + selector: + query: 'true' + includeMembers: 'true' + port: + entity: + mappings: + identifier: .id | tostring + title: .attributes.name + blueprint: '"datadogTeam"' + properties: + description: .attributes.description + handle: .attributes.handle + userCount: .attributes.user_count + summary: .attributes.summary + createdAt: .attributes.created_at | todate + relations: + members: if .__members then .__members[].id else [] end + - kind: host selector: query: "true" @@ -58,6 +93,13 @@ resources: owners: >- [.attributes.schema.contacts[] | select(.type == "email") | .contact] + relations: + team: + combinator: '"and"' + rules: + - property: '"handle"' + operator: '"="' + value: .attributes.schema.team - kind: slo selector: query: "true" diff --git a/integrations/datadog/.port/spec.yaml b/integrations/datadog/.port/spec.yaml index cb74c539cb..5d7f4ae7a1 100644 --- a/integrations/datadog/.port/spec.yaml +++ b/integrations/datadog/.port/spec.yaml @@ -6,12 +6,16 @@ features: - type: exporter section: APM & Alerting resources: + - kind: user + - kind: team - kind: host - kind: monitor - kind: service - kind: slo - kind: sloHistory - kind: serviceMetric + + configurations: - name: datadogBaseUrl description: Datadog Base URL (e.g., https://api.datadoghq.com or https://api.datadoghq.eu. To identify your base URL, see the Datadog documentation. diff --git a/integrations/datadog/CHANGELOG.md b/integrations/datadog/CHANGELOG.md index f2f417f444..2a3fabc156 100644 --- a/integrations/datadog/CHANGELOG.md +++ b/integrations/datadog/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +## 0.1.71 (2024-12-30) + + +### Improvements + +- Added Datadog Users and Teams + + ## 0.1.70 (2024-12-30) diff --git a/integrations/datadog/client.py b/integrations/datadog/client.py index 52d3b8d717..a4059fcdb4 100644 --- a/integrations/datadog/client.py +++ b/integrations/datadog/client.py @@ -158,6 +158,83 @@ async def _fetch_with_rate_limit_handling( raise return response.json() + async def get_team_members( + self, team_id: str, page_size: int = MAX_PAGE_SIZE + ) -> AsyncGenerator[list[dict[str, Any]], None]: + """Get teams members from DataDog + Docs: https://docs.datadoghq.com/api/latest/teams/#get-team-memberships + """ + + logger.info(f"Enriching team {team_id} with members information") + + page = 0 + + while True: + url = f"{self.api_url}/api/v2/team/{team_id}/memberships" + result = await self._send_api_request( + url, + params={ + "page[size]": page_size, + "page[number]": page, + }, + ) + + users = result.get("included", []) + + if not users: + break + + yield users + page += 1 + + async def get_teams(self) -> AsyncGenerator[list[dict[str, Any]], None]: + """Get teams from DataDog + Docs: https://docs.datadoghq.com/api/latest/teams/#get-all-teams + """ + page = 0 + page_size = MAX_PAGE_SIZE + + while True: + url = f"{self.api_url}/api/v2/team" + result = await self._send_api_request( + url, + params={ + "page[size]": page_size, + "page[number]": page, + }, + ) + + teams = result.get("data", []) + if not teams: + break + + yield teams + page += 1 + + async def get_users(self) -> AsyncGenerator[list[dict[str, Any]], None]: + """Get users from DataDog + Docs: https://docs.datadoghq.com/api/latest/users/#list-all-users + """ + page = 0 + page_size = MAX_PAGE_SIZE + + while True: + url = f"{self.api_url}/api/v2/users" + result = await self._send_api_request( + url, + params={ + "page[number]": page, + "page[size]": page_size, + }, + ) + + users = result.get("data", []) + if not users: + break + + yield users + page += 1 + async def get_hosts(self) -> AsyncGenerator[list[dict[str, Any]], None]: start = 0 count = MAX_PAGE_SIZE diff --git a/integrations/datadog/main.py b/integrations/datadog/main.py index 603c21de12..ced2cc4d45 100644 --- a/integrations/datadog/main.py +++ b/integrations/datadog/main.py @@ -1,11 +1,16 @@ import typing from enum import StrEnum -from typing import Any +from typing import Any, cast from loguru import logger from client import DatadogClient -from overrides import SLOHistoryResourceConfig, DatadogResourceConfig, DatadogSelector +from overrides import ( + SLOHistoryResourceConfig, + DatadogResourceConfig, + DatadogSelector, + TeamResourceConfig, +) from port_ocean.context.event import event from port_ocean.context.ocean import ocean from port_ocean.core.ocean_types import ASYNC_GENERATOR_RESYNC_TYPE @@ -18,6 +23,8 @@ class ObjectKind(StrEnum): SERVICE = "service" SLO_HISTORY = "sloHistory" SERVICE_METRIC = "serviceMetric" + TEAM = "team" + USER = "user" def init_client() -> DatadogClient: @@ -28,6 +35,32 @@ def init_client() -> DatadogClient: ) +@ocean.on_resync(ObjectKind.TEAM) +async def on_resync_teams(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + dd_client = init_client() + + selector = cast(TeamResourceConfig, event.resource_config).selector + + async for teams in dd_client.get_teams(): + logger.info(f"Received teams batch with {len(teams)} teams") + if selector.include_members: + for team in teams: + members = [] + async for member_batch in dd_client.get_team_members(team["id"]): + members.extend(member_batch) + team["__members"] = members + yield teams + + +@ocean.on_resync(ObjectKind.USER) +async def on_resync_users(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + dd_client = init_client() + + async for users in dd_client.get_users(): + logger.info(f"Received batch with {len(users)} users") + yield users + + @ocean.on_resync(ObjectKind.HOST) async def on_resync_hosts(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: dd_client = init_client() diff --git a/integrations/datadog/overrides.py b/integrations/datadog/overrides.py index afe90ec6b3..5b596cc1fc 100644 --- a/integrations/datadog/overrides.py +++ b/integrations/datadog/overrides.py @@ -65,9 +65,25 @@ class DatadogResourceConfig(ResourceConfig): selector: DatadogResourceSelector +class TeamSelector(Selector): + include_members: bool = Field( + alias="includeMembers", + default=False, + description="Whether to include the members of the team, defaults to false", + ) + + +class TeamResourceConfig(ResourceConfig): + kind: typing.Literal["team"] + selector: TeamSelector + + class DataDogPortAppConfig(PortAppConfig): resources: list[ - SLOHistoryResourceConfig | DatadogResourceConfig | ResourceConfig + TeamResourceConfig + | SLOHistoryResourceConfig + | DatadogResourceConfig + | ResourceConfig ] = Field(default_factory=list) diff --git a/integrations/datadog/pyproject.toml b/integrations/datadog/pyproject.toml index 20e684c9a5..3e81efd9c1 100644 --- a/integrations/datadog/pyproject.toml +++ b/integrations/datadog/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datadog" -version = "0.1.70" +version = "0.1.71" description = "Datadog Ocean Integration" authors = ["Albert Luganga "] diff --git a/integrations/datadog/tests/test_client.py b/integrations/datadog/tests/test_client.py new file mode 100644 index 0000000000..d1b9f171f7 --- /dev/null +++ b/integrations/datadog/tests/test_client.py @@ -0,0 +1,188 @@ +import pytest +from typing import Any +from unittest.mock import AsyncMock, patch, MagicMock + +from port_ocean.context.ocean import initialize_port_ocean_context +from port_ocean.exceptions.context import PortOceanContextAlreadyInitializedError +from client import DatadogClient, MAX_PAGE_SIZE + + +@pytest.fixture(autouse=True) +def mock_ocean_context() -> None: + try: + mock_ocean_app = MagicMock() + mock_ocean_app.config.integration.config = { + "api_key": "test_api_key", + "app_key": "test_app_key", + "api_url": "api.datadoghq.com", + } + mock_ocean_app.integration_router = MagicMock() + mock_ocean_app.port_client = MagicMock() + initialize_port_ocean_context(mock_ocean_app) + except PortOceanContextAlreadyInitializedError: + pass + + +@pytest.fixture +def mock_datadog_client() -> DatadogClient: + return DatadogClient( + api_key="test_api_key", app_key="test_app_key", api_url="api.datadoghq.com" + ) + + +@pytest.mark.asyncio +async def test_get_teams(mock_datadog_client: DatadogClient) -> None: + teams_response: dict[str, list[dict[str, Any]]] = { + "data": [{"id": "1", "type": "team"}, {"id": "2", "type": "team"}] + } + empty_response: dict[str, list[dict[str, Any]]] = {"data": []} + + with patch.object( + mock_datadog_client, "_send_api_request", new_callable=AsyncMock + ) as mock_request: + mock_request.side_effect = [teams_response, empty_response] + + teams = [] + async for team_batch in mock_datadog_client.get_teams(): + teams.extend(team_batch) + + assert len(teams) == 2 + assert teams == teams_response["data"] + mock_request.assert_called_with( + f"{mock_datadog_client.api_url}/api/v2/team", + params={"page[size]": MAX_PAGE_SIZE, "page[number]": 1}, + ) + + +@pytest.mark.asyncio +async def test_get_teams_multiple_pages(mock_datadog_client: DatadogClient) -> None: + first_page: dict[str, list[dict[str, Any]]] = { + "data": [{"id": "1", "type": "team"}] + } + second_page: dict[str, list[dict[str, Any]]] = { + "data": [{"id": "2", "type": "team"}] + } + third_page: dict[str, list[dict[str, Any]]] = { + "data": [{"id": "3", "type": "team"}] + } + empty_page: dict[str, list[dict[str, Any]]] = {"data": []} + + with patch.object( + mock_datadog_client, "_send_api_request", new_callable=AsyncMock + ) as mock_request: + mock_request.side_effect = [first_page, second_page, third_page, empty_page] + + teams = [] + async for team_batch in mock_datadog_client.get_teams(): + teams.extend(team_batch) + + assert len(teams) == 3 + assert teams == first_page["data"] + second_page["data"] + third_page["data"] + assert mock_request.call_count == 4 + + +@pytest.mark.asyncio +async def test_get_users(mock_datadog_client: DatadogClient) -> None: + users_response: dict[str, list[dict[str, Any]]] = { + "data": [{"id": "1", "type": "users"}, {"id": "2", "type": "users"}] + } + empty_response: dict[str, list[dict[str, Any]]] = {"data": []} + + with patch.object( + mock_datadog_client, "_send_api_request", new_callable=AsyncMock + ) as mock_request: + mock_request.side_effect = [users_response, empty_response] + + users = [] + async for user_batch in mock_datadog_client.get_users(): + users.extend(user_batch) + + assert len(users) == 2 + assert users == users_response["data"] + mock_request.assert_called_with( + f"{mock_datadog_client.api_url}/api/v2/users", + params={"page[size]": MAX_PAGE_SIZE, "page[number]": 1}, + ) + + +@pytest.mark.asyncio +async def test_get_users_multiple_pages(mock_datadog_client: DatadogClient) -> None: + first_page: dict[str, list[dict[str, Any]]] = { + "data": [{"id": "1", "type": "users"}] + } + second_page: dict[str, list[dict[str, Any]]] = { + "data": [{"id": "2", "type": "users"}] + } + third_page: dict[str, list[dict[str, Any]]] = { + "data": [{"id": "3", "type": "users"}] + } + empty_page: dict[str, list[dict[str, Any]]] = {"data": []} + + with patch.object( + mock_datadog_client, "_send_api_request", new_callable=AsyncMock + ) as mock_request: + mock_request.side_effect = [first_page, second_page, third_page, empty_page] + + users = [] + async for user_batch in mock_datadog_client.get_users(): + users.extend(user_batch) + + assert len(users) == 3 + assert users == first_page["data"] + second_page["data"] + third_page["data"] + assert mock_request.call_count == 4 + + +@pytest.mark.asyncio +async def test_get_team_members(mock_datadog_client: DatadogClient) -> None: + members_response: dict[str, list[dict[str, Any]]] = { + "included": [{"id": "1", "type": "users"}, {"id": "2", "type": "users"}] + } + empty_response: dict[str, list[dict[str, Any]]] = {"included": []} + + with patch.object( + mock_datadog_client, "_send_api_request", new_callable=AsyncMock + ) as mock_request: + mock_request.side_effect = [members_response, empty_response] + + members = [] + async for member_batch in mock_datadog_client.get_team_members("team1"): + members.extend(member_batch) + + assert len(members) == 2 + assert members == members_response["included"] + mock_request.assert_called_with( + f"{mock_datadog_client.api_url}/api/v2/team/team1/memberships", + params={"page[size]": MAX_PAGE_SIZE, "page[number]": 1}, + ) + + +@pytest.mark.asyncio +async def test_get_team_members_multiple_pages( + mock_datadog_client: DatadogClient, +) -> None: + first_page: dict[str, list[dict[str, Any]]] = { + "included": [{"id": "1", "type": "users"}] + } + second_page: dict[str, list[dict[str, Any]]] = { + "included": [{"id": "2", "type": "users"}] + } + third_page: dict[str, list[dict[str, Any]]] = { + "included": [{"id": "3", "type": "users"}] + } + empty_page: dict[str, list[dict[str, Any]]] = {"included": []} + + with patch.object( + mock_datadog_client, "_send_api_request", new_callable=AsyncMock + ) as mock_request: + mock_request.side_effect = [first_page, second_page, third_page, empty_page] + + members = [] + async for member_batch in mock_datadog_client.get_team_members("team1"): + members.extend(member_batch) + + assert len(members) == 3 + assert ( + members + == first_page["included"] + second_page["included"] + third_page["included"] + ) + assert mock_request.call_count == 4