diff --git a/app/integrations/aws/identity_store.py b/app/integrations/aws/identity_store.py index f6d288f6..6279a109 100644 --- a/app/integrations/aws/identity_store.py +++ b/app/integrations/aws/identity_store.py @@ -1,5 +1,7 @@ import os import logging + +import pandas as pd from integrations.aws.client import execute_aws_api_call, handle_aws_api_errors from utils import filters @@ -288,10 +290,8 @@ def list_group_memberships(group_id, **kwargs): @handle_aws_api_errors def list_groups_with_memberships( - group_members: bool = True, - members_details: bool = True, - include_empty_groups: bool = True, groups_filters: list | None = None, + tolerate_errors: bool = False, ): """Retrieves groups with their members from the AWS Identity Center (identitystore) @@ -314,18 +314,84 @@ def list_groups_with_memberships( for groups_filter in groups_filters: groups = filters.filter_by_condition(groups, groups_filter) logger.info(f"Founds {len(groups)} groups in AWS Identity Store.") - if not group_members: - return groups + + filtered_groups = [ + { + k: v + for k, v in group.items() + if k in ["GroupId", "DisplayName", "Description", "IdentityStoreId"] + } + for group in groups + ] groups_with_memberships = [] - for group in groups: - group["GroupMemberships"] = list_group_memberships(group["GroupId"]) - if group["GroupMemberships"]: - if members_details: - for membership in group["GroupMemberships"]: - membership["MemberId"] = describe_user( - membership["MemberId"]["UserId"] - ) - if group["GroupMemberships"] or include_empty_groups: + for group in filtered_groups: + error_occurred = False + logger.info(f"Getting members for group: {group['DisplayName']}") + try: + memberships = list_group_memberships(group["GroupId"]) + except Exception as error: + logger.warning( + f"Error getting members for group {group['GroupId']}: {error}" + ) + continue + for membership in memberships: + member_details = {} + try: + logger.info( + f"Getting details for member: {membership['MemberId']['UserId']}" + ) + member_details = describe_user(membership["MemberId"]["UserId"]) + except Exception as error: + logger.warning( + f"Error getting details for member {membership['MemberId']['UserId']}: {error}" + ) + error_occurred = True + if not tolerate_errors: + break + if member_details: + membership["MemberId"].update(member_details) + if memberships and (not error_occurred or tolerate_errors): + group["GroupMemberships"] = memberships groups_with_memberships.append(group) return groups_with_memberships + + +def convert_aws_groups_members_to_dataframe(groups): + """Converts a list of AWS groups with members to a DataFrame. + + Args: + groups (list): A list of group objects with members. + + Returns: + DataFrame: A DataFrame with group members. + """ + flattened_data = [] + for group in groups: + group_id = group.get("GroupId") + group_name = group.get("DisplayName") + group_description = group.get("Description") + group_identity_store_id = group.get("IdentityStoreId") + + for membership in group.get("GroupMemberships", []): + member = membership.get("MemberId", {}) + member_user_id = member.get("UserId") + member_email = member.get("UserName") + member_given_name = member.get("Name", {}).get("GivenName") + member_family_name = member.get("Name", {}).get("FamilyName") + member_display_name = member.get("DisplayName") + + flattened_record = { + "group_id": group_id, + "group_name": group_name, + "group_description": group_description, + "group_identity_store_id": group_identity_store_id, + "member_user_id": member_user_id, + "member_email": member_email, + "member_given_name": member_given_name, + "member_family_name": member_family_name, + "member_display_name": member_display_name, + } + flattened_data.append(flattened_record) + + return pd.DataFrame(flattened_data) diff --git a/app/integrations/google_workspace/google_directory.py b/app/integrations/google_workspace/google_directory.py index 314e2349..9fe9943b 100644 --- a/app/integrations/google_workspace/google_directory.py +++ b/app/integrations/google_workspace/google_directory.py @@ -1,6 +1,8 @@ """Google Directory module to interact with the Google Workspace Directory API.""" from logging import getLogger + +import pandas as pd from integrations.google_workspace.google_service import ( handle_google_api_errors, execute_google_api_call, @@ -171,8 +173,6 @@ def add_users_to_group(group, group_key): def list_groups_with_members( - group_members: bool = True, - members_details: bool = True, groups_filters: list = [], query: str | None = None, tolerate_errors: bool = False, @@ -180,8 +180,6 @@ def list_groups_with_members( """List all groups in the Google Workspace domain with their members. Args: - group_members (bool): Include the group members in the response. - members_details (bool): Include the members details in the response. groups_filters (list): List of filters to apply to the groups. query (str): The query to search for groups. tolerate_errors (bool): Whether to include groups that encountered errors during member detail retrieval. @@ -199,11 +197,19 @@ def list_groups_with_members( for groups_filter in groups_filters: groups = filters.filter_by_condition(groups, groups_filter) logger.info(f"Found {len(groups)} groups.") - if not group_members: - return groups + + filtered_groups = [ + { + k: v + for k, v in group.items() + if k in ["id", "email", "name", "directMembersCount", "description"] + } + for group in groups + ] groups_with_members = [] - for group in groups: + for group in filtered_groups: + error_occured = False logger.info(f"Getting members for group: {group['email']}") try: members = list_group_members( @@ -213,24 +219,65 @@ def list_groups_with_members( logger.warning(f"Error getting members for group {group['email']}: {e}") continue - if members and members_details: - detailed_members = [] - error_occurred = False - for member in members: - try: - logger.info(f"Getting user details for member: {member['email']}") - detailed_members.append( - get_user(member["email"], fields="name, primaryEmail") - ) - except Exception as e: - logger.warning( - f"Error getting user details for member {member['email']}: {e}" - ) - error_occurred = True - if not tolerate_errors: - break - if error_occurred and not tolerate_errors: - continue - group["members"] = detailed_members + for member in members: + user_details = {} + try: + logger.info(f"Getting user details for member: {member['email']}") + user_details = get_user(member["email"], fields="name, primaryEmail") + except Exception as e: + logger.warning( + f"Error getting user details for member {member['email']}: {e}" + ) + error_occured = True + if not tolerate_errors: + break + if user_details: + member.update(user_details) + if members and (not error_occured or tolerate_errors): + group.update({"members": members}) groups_with_members.append(group) + return groups_with_members + + +def convert_google_groups_members_to_dataframe(groups): + """Converts a list of Google groups with members to a DataFrame. + + Args: + groups (list): A list of group objects with members. + + Returns: + DataFrame: A DataFrame with group members. + """ + flattened_data = [] + for group in groups: + group_email = group.get("email") + group_name = group.get("name") + group_direct_members_count = group.get("directMembersCount") + group_description = group.get("description") + + for member in group.get("members", []): + member_email = member.get("email") + member_role = member.get("role") + member_type = member.get("type") + member_status = member.get("status") + member_primary_email = member.get("primaryEmail") + member_given_name = member.get("name", {}).get("givenName") + member_family_name = member.get("name", {}).get("familyName") + + flattened_record = { + "group_email": group_email, + "group_name": group_name, + "group_direct_members_count": group_direct_members_count, + "group_description": group_description, + "member_email": member_email, + "member_role": member_role, + "member_type": member_type, + "member_status": member_status, + "member_primary_email": member_primary_email, + "member_given_name": member_given_name, + "member_family_name": member_family_name, + } + flattened_data.append(flattened_record) + + return pd.DataFrame(flattened_data) diff --git a/app/modules/provisioning/groups.py b/app/modules/provisioning/groups.py index 0f87b22d..b256de0d 100644 --- a/app/modules/provisioning/groups.py +++ b/app/modules/provisioning/groups.py @@ -12,8 +12,7 @@ def get_groups_from_integration( pre_processing_filters: list = [], post_processing_filters: list = [], query: str | None = None, - group_members: bool = True, - members_details: bool = True, + return_dataframe: bool = False, ) -> list: """Retrieve the users from an integration group source. Supported sources are: @@ -39,11 +38,13 @@ def get_groups_from_integration( case "google_groups": logger.info("Getting Google Groups with members.") groups = google_directory.list_groups_with_members( - group_members=group_members, - members_details=members_details, groups_filters=pre_processing_filters, query=query, ) + if return_dataframe: + groups_dataframe = ( + google_directory.convert_google_groups_members_to_dataframe(groups) + ) integration_name = "Google" group_display_key = "name" members = "members" @@ -51,10 +52,12 @@ def get_groups_from_integration( case "aws_identity_center": logger.info("Getting AWS Identity Center Groups with members.") groups = identity_store.list_groups_with_memberships( - group_members=group_members, - members_details=members_details, groups_filters=pre_processing_filters, ) + if return_dataframe: + groups_dataframe = ( + identity_store.convert_aws_groups_members_to_dataframe(groups) + ) integration_name = "AWS" group_display_key = "DisplayName" members = "GroupMemberships" @@ -69,18 +72,16 @@ def get_groups_from_integration( groups, group_display_key=group_display_key, members=members, - members_details=members_details, members_display_key=members_display_key, integration_name=integration_name, ) - return groups + return groups_dataframe if return_dataframe else groups def log_groups( groups, group_display_key=None, members=None, - members_details=True, members_display_key=None, integration_name="No Integration Name Provided", ): @@ -110,12 +111,9 @@ def log_groups( members_display_name = filters.get_nested_value( member, members_display_key ) - if not members_display_name and members_details: + if not members_display_name: members_display_name = "" - if members_details: - logger.info( - f"{integration_name}Group:Member: {members_display_name}" - ) + logger.info(f"{integration_name}Group:Member: {members_display_name}") else: logger.info( f"{integration_name}Group: {group_display_name} has no members." diff --git a/app/tests/conftest.py b/app/tests/conftest.py index fa1cc017..33164219 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -13,6 +13,8 @@ def _google_groups(n=3, prefix="", domain="test.com"): "id": f"{prefix}google_group_id{i+1}", "name": f"{prefix}group-name{i+1}", "email": f"{prefix}group-name{i+1}@{domain}", + "description": f"{prefix}description{i+1}", + "directMembersCount": i + 1, } for i in range(n) ] @@ -70,14 +72,21 @@ def _google_group_members(n=3, prefix="", domain="test.com"): # Fixture with users @pytest.fixture -def google_groups_w_users(google_groups, google_users): +def google_groups_w_users(google_groups, google_group_members, google_users): def _google_groups_w_users( n_groups=1, n_users=3, group_prefix="", user_prefix="", domain="test.com" ): groups = google_groups(n_groups, prefix=group_prefix, domain=domain) + members = google_group_members(n_users, prefix=user_prefix, domain=domain) users = google_users(n_users, prefix=user_prefix, domain=domain) + + combined_members = [] + for member, user in zip(members, users): + combined_member = {**member, **user} + combined_members.append(combined_member) + for group in groups: - group["members"] = users + group["members"] = combined_members return groups return _google_groups_w_users diff --git a/app/tests/integrations/aws/test_identity_store.py b/app/tests/integrations/aws/test_identity_store.py index 6dd49fbb..4efff2b5 100644 --- a/app/tests/integrations/aws/test_identity_store.py +++ b/app/tests/integrations/aws/test_identity_store.py @@ -787,13 +787,6 @@ def test_list_groups_with_memberships( ] users = aws_users(2, prefix="test-", domain="test.com") expected_output = [ - { - "Description": "A group to test resolving AWS-group1 memberships", - "DisplayName": "test-group-name1", - "GroupId": "test-aws-group_id1", - "GroupMemberships": [], - "IdentityStoreId": "d-123412341234", - }, { "GroupId": "test-aws-group_id2", "DisplayName": "test-group-name2", @@ -877,23 +870,6 @@ def test_list_groups_with_memberships_empty_groups( assert mock_describe_user.call_count == 0 -@patch("integrations.aws.identity_store.list_groups") -@patch("integrations.aws.identity_store.list_group_memberships") -@patch("integrations.aws.identity_store.describe_user") -def test_list_groups_with_memberships_empty_groups_memberships_with_flag( - mock_describe_user, mock_list_group_memberships, mock_list_groups, aws_groups -): - groups = aws_groups(2, prefix="test-") - expected_output = [] - groups_memberships = [[], []] - mock_list_groups.return_value = groups - mock_list_group_memberships.side_effect = groups_memberships - result = identity_store.list_groups_with_memberships(include_empty_groups=False) - assert result == expected_output - assert mock_list_group_memberships.call_count == 2 - assert mock_describe_user.call_count == 0 - - @patch("integrations.aws.identity_store.filters.filter_by_condition") @patch("integrations.aws.identity_store.list_groups") @patch("integrations.aws.identity_store.list_group_memberships") @@ -917,13 +893,6 @@ def test_list_groups_with_memberships_filtered( users = aws_users(2, prefix="test-", domain="test.com") expected_output = [ - { - "Description": "A group to test resolving AWS-group1 memberships", - "DisplayName": "test-group-name1", - "GroupId": "test-aws-group_id1", - "GroupMemberships": [], - "IdentityStoreId": "d-123412341234", - }, { "GroupId": "test-aws-group_id2", "DisplayName": "test-group-name2", @@ -981,55 +950,12 @@ def test_list_groups_with_memberships_filtered( mock_list_group_memberships.side_effect = memberships - user_side_effect = [] - for user in users: - user_side_effect.append(user) - - mock_describe_user.side_effect = user_side_effect + mock_describe_user.side_effect = [users[0], users[1], users[0], users[1]] mock_filter_by_condition.return_value = groups[:2] groups_filters = [lambda group: "test-" in group["DisplayName"]] result = identity_store.list_groups_with_memberships(groups_filters=groups_filters) + assert result == expected_output assert mock_filter_by_condition.call_count == 1 assert mock_list_group_memberships.call_count == 2 assert mock_describe_user.call_count == 2 - assert result == expected_output - - -@patch("integrations.aws.identity_store.list_groups") -@patch("integrations.aws.identity_store.list_group_memberships") -@patch("integrations.aws.identity_store.describe_user") -def test_list_groups_with_memberhips_without_members_enabled( - mock_describe_user, - mock_list_group_memberships, - mock_list_groups, - aws_groups, - aws_groups_memberships, -): - groups = aws_groups(2, prefix="test-") - memberships = [ - [], - aws_groups_memberships(2, prefix="test-", group_id=2)["GroupMemberships"], - ] - expected_output = [ - { - "Description": "A group to test resolving AWS-group1 memberships", - "DisplayName": "test-group-name1", - "GroupId": "test-aws-group_id1", - "IdentityStoreId": "d-123412341234", - }, - { - "GroupId": "test-aws-group_id2", - "DisplayName": "test-group-name2", - "Description": "A group to test resolving AWS-group2 memberships", - "IdentityStoreId": "d-123412341234", - }, - ] - mock_list_groups.return_value = groups - mock_list_group_memberships.side_effect = memberships - - result = identity_store.list_groups_with_memberships(group_members=False) - - assert result == expected_output - assert mock_list_group_memberships.call_count == 0 - assert mock_describe_user.call_count == 0 diff --git a/app/tests/integrations/google_workspace/test_google_directory.py b/app/tests/integrations/google_workspace/test_google_directory.py index e483a28c..445c7a91 100644 --- a/app/tests/integrations/google_workspace/test_google_directory.py +++ b/app/tests/integrations/google_workspace/test_google_directory.py @@ -1,6 +1,8 @@ """Unit tests for google_directory module.""" from unittest.mock import patch + +import pandas as pd from integrations.google_workspace import google_directory @@ -415,7 +417,9 @@ def test_list_groups_with_members_error_in_list_group_members( # Only the second group should be processed expected_groups_with_users = [groups[1]] - expected_groups_with_users[0]["members"] = users + expected_groups_with_users[0]["members"] = group_members[1] + expected_groups_with_users[0]["members"][0].update(users[0]) + expected_groups_with_users[0]["members"][1].update(users[1]) assert google_directory.list_groups_with_members() == expected_groups_with_users @@ -423,105 +427,193 @@ def test_list_groups_with_members_error_in_list_group_members( @patch("integrations.google_workspace.google_directory.list_groups") @patch("integrations.google_workspace.google_directory.list_group_members") @patch("integrations.google_workspace.google_directory.get_user") -def test_list_groups_with_members_without_details( +def test_list_groups_with_members_error_in_get_user( mock_get_user, mock_list_group_members, mock_list_groups, google_groups, google_group_members, google_users, - google_groups_w_users, ): groups = google_groups(2) - group_members = [[], google_group_members(2)] - users = google_users(2) - groups_with_users = google_groups_w_users(2, 2) - - groups_with_users[0].pop("members", None) - groups_with_users[1].pop("members", None) + group_members = [google_group_members(2), google_group_members(2)] + users = [ + Exception("Error fetching user details"), + google_users(2)[1], + google_users(1)[0], + google_users(2)[1], + ] mock_list_groups.return_value = groups mock_list_group_members.side_effect = group_members mock_get_user.side_effect = users - assert google_directory.list_groups_with_members(members_details=False) == [] + # Only the second group should be processed + expected_groups_with_users = [groups[1]] + expected_groups_with_users[0]["members"] = group_members[1] + expected_groups_with_users[0]["members"][0].update(users[2]) + expected_groups_with_users[0]["members"][1].update(users[3]) + + assert google_directory.list_groups_with_members() == expected_groups_with_users @patch("integrations.google_workspace.google_directory.list_groups") @patch("integrations.google_workspace.google_directory.list_group_members") @patch("integrations.google_workspace.google_directory.get_user") -def test_list_groups_with_members_without_members_enabled( +def test_list_groups_with_members_tolerate_errors( mock_get_user, mock_list_group_members, mock_list_groups, - google_groups, + google_groups_w_users, ): - groups = google_groups(2) - mock_list_groups.return_value = groups - assert google_directory.list_groups_with_members(group_members=False) == groups - mock_list_group_members.assert_not_called() - mock_get_user.assert_not_called() + groups = [ + { + "id": "group1", + "email": "groupEmail1", + "name": "name1", + "directMembersCount": 2, + }, + { + "id": "group2", + "email": "groupEmail2", + "name": "name2", + "directMembersCount": 2, + }, + ] + group_members = [ + [ + {"email": "email1", "role": "MEMBER", "type": "USER", "status": "ACTIVE"}, + {"email": "email2", "role": "MEMBER", "type": "USER", "status": "ACTIVE"}, + ], + [ + {"email": "email3", "role": "MEMBER", "type": "USER", "status": "ACTIVE"}, + {"email": "email4", "role": "MEMBER", "type": "USER", "status": "ACTIVE"}, + ], + ] -@patch("integrations.google_workspace.google_directory.list_groups") -@patch("integrations.google_workspace.google_directory.list_group_members") -@patch("integrations.google_workspace.google_directory.get_user") -def test_list_groups_with_members_error_in_get_user( - mock_get_user, - mock_list_group_members, - mock_list_groups, - google_groups, - google_group_members, - google_users, -): - groups = google_groups(2) - group_members = [google_group_members(2), google_group_members(2)] - users = [Exception("Error fetching user details"), google_users(1)[0]] + users = [ + Exception("Error fetching user details"), + {"id": "user2", "name": "user2", "primaryEmail": "email2"}, + {"id": "user3", "name": "user3", "primaryEmail": "email3"}, + {"id": "user4", "name": "user4", "primaryEmail": "email4"}, + ] mock_list_groups.return_value = groups mock_list_group_members.side_effect = group_members mock_get_user.side_effect = users - # No groups should be processed due to error in get_user + # Expected result should include both groups, with the second group having one member + expected_groups_with_users = [ + { + "id": "group1", + "email": "groupEmail1", + "name": "name1", + "directMembersCount": 2, + "members": [ + { + "email": "email1", + "role": "MEMBER", + "type": "USER", + "status": "ACTIVE", + }, + { + "email": "email2", + "primaryEmail": "email2", + "role": "MEMBER", + "type": "USER", + "status": "ACTIVE", + "id": "user2", + "name": "user2", + }, + ], + }, + { + "id": "group2", + "email": "groupEmail2", + "name": "name2", + "directMembersCount": 2, + "members": [ + { + "email": "email3", + "primaryEmail": "email3", + "role": "MEMBER", + "type": "USER", + "status": "ACTIVE", + "id": "user3", + "name": "user3", + }, + { + "email": "email4", + "primaryEmail": "email4", + "role": "MEMBER", + "type": "USER", + "status": "ACTIVE", + "id": "user4", + "name": "user4", + }, + ], + }, + ] + + result = google_directory.list_groups_with_members(tolerate_errors=True) + + assert result == expected_groups_with_users + + +@patch("integrations.google_workspace.google_directory.list_groups") +def test_list_groups_with_members_skips_when_no_groups(mock_list_groups): + mock_list_groups.return_value = [] assert google_directory.list_groups_with_members() == [] +@patch("integrations.google_workspace.google_directory.filters.filter_by_condition") @patch("integrations.google_workspace.google_directory.list_groups") @patch("integrations.google_workspace.google_directory.list_group_members") @patch("integrations.google_workspace.google_directory.get_user") -def test_list_groups_with_members_tolerate_errors( +def test_list_groups_with_members_filtered_dataframe( mock_get_user, mock_list_group_members, mock_list_groups, + mock_filter_by_condition, google_groups, google_group_members, google_users, + google_groups_w_users, ): - groups = google_groups(2) - group_members = [google_group_members(2), google_group_members(2)] - users = [ - Exception("Error fetching user details"), - google_users(1)[0], - Exception("Error fetching user details"), - google_users(1)[0], - ] + groups = google_groups(2, prefix="test-") + groups_to_filter_out = google_groups(4)[2:] + groups.extend(groups_to_filter_out) + group_members = [[], google_group_members(2)] + users = google_users(2) + + groups_with_users = google_groups_w_users(4, 2, group_prefix="test-")[:2] + groups_with_users.remove(groups_with_users[0]) mock_list_groups.return_value = groups mock_list_group_members.side_effect = group_members mock_get_user.side_effect = users + mock_filter_by_condition.return_value = groups[:2] + groups_filters = [lambda group: "test-" in group["name"]] - # Expected result should include both groups, with the second group having one member - expected_groups_with_users = [groups[0], groups[1]] - expected_groups_with_users[0]["members"] = [] - expected_groups_with_users[1]["members"] = [google_users(1)[0]] - - result = google_directory.list_groups_with_members(tolerate_errors=True) - - assert result == expected_groups_with_users - - -@patch("integrations.google_workspace.google_directory.list_groups") -def test_list_groups_with_members_skips_when_no_groups(mock_list_groups): - mock_list_groups.return_value = [] - assert google_directory.list_groups_with_members() == [] + groups_result = google_directory.list_groups_with_members( + groups_filters=groups_filters + ) + result = google_directory.convert_google_groups_members_to_dataframe(groups_result) + + assert isinstance(result, pd.DataFrame) + assert not result.empty + assert set(result.columns) == { + "group_email", + "group_name", + "group_direct_members_count", + "group_description", + "member_email", + "member_role", + "member_type", + "member_status", + "member_primary_email", + "member_given_name", + "member_family_name", + }