Fix/list groups w users

app/integrations/aws/identity_store.py

@@ -1,5 +1,7 @@
 import os
 import logging
+
+import pandas as pd
 from integrations.aws.client import execute_aws_api_call, handle_aws_api_errors
 from utils import filters
 
@@ -288,10 +290,8 @@ def list_group_memberships(group_id, **kwargs):
 
 @handle_aws_api_errors
 def list_groups_with_memberships(
-    group_members: bool = True,
-    members_details: bool = True,
-    include_empty_groups: bool = True,
     groups_filters: list | None = None,
+    tolerate_errors: bool = False,
 ):
     """Retrieves groups with their members from the AWS Identity Center (identitystore)
 
@@ -314,18 +314,84 @@ def list_groups_with_memberships(
         for groups_filter in groups_filters:
             groups = filters.filter_by_condition(groups, groups_filter)
     logger.info(f"Founds {len(groups)} groups in AWS Identity Store.")
-    if not group_members:
-        return groups
+
+    filtered_groups = [
+        {
+            k: v
+            for k, v in group.items()
+            if k in ["GroupId", "DisplayName", "Description", "IdentityStoreId"]
+        }
+        for group in groups
+    ]
 
     groups_with_memberships = []
-    for group in groups:
-        group["GroupMemberships"] = list_group_memberships(group["GroupId"])
-        if group["GroupMemberships"]:
-            if members_details:
-                for membership in group["GroupMemberships"]:
-                    membership["MemberId"] = describe_user(
-                        membership["MemberId"]["UserId"]
-                    )
-        if group["GroupMemberships"] or include_empty_groups:
+    for group in filtered_groups:
+        error_occurred = False
+        logger.info(f"Getting members for group: {group['DisplayName']}")
+        try:
+            memberships = list_group_memberships(group["GroupId"])
+        except Exception as error:
+            logger.warning(
+                f"Error getting members for group {group['GroupId']}: {error}"
+            )
+            continue
+        for membership in memberships:
+            member_details = {}
+            try:
+                logger.info(
+                    f"Getting details for member: {membership['MemberId']['UserId']}"
+                )
+                member_details = describe_user(membership["MemberId"]["UserId"])
+            except Exception as error:
+                logger.warning(
+                    f"Error getting details for member {membership['MemberId']['UserId']}: {error}"
+                )
+                error_occurred = True
+                if not tolerate_errors:
+                    break
+            if member_details:
+                membership["MemberId"].update(member_details)
+        if memberships and (not error_occurred or tolerate_errors):
+            group["GroupMemberships"] = memberships
             groups_with_memberships.append(group)
     return groups_with_memberships
+
+
+def convert_aws_groups_members_to_dataframe(groups):
+    """Converts a list of AWS groups with members to a DataFrame.
+
+    Args:
+        groups (list): A list of group objects with members.
+
+    Returns:
+        DataFrame: A DataFrame with group members.
+    """
+    flattened_data = []
+    for group in groups:
+        group_id = group.get("GroupId")
+        group_name = group.get("DisplayName")
+        group_description = group.get("Description")
+        group_identity_store_id = group.get("IdentityStoreId")
+
+        for membership in group.get("GroupMemberships", []):
+            member = membership.get("MemberId", {})
+            member_user_id = member.get("UserId")
+            member_email = member.get("UserName")
+            member_given_name = member.get("Name", {}).get("GivenName")
+            member_family_name = member.get("Name", {}).get("FamilyName")
+            member_display_name = member.get("DisplayName")
+
+            flattened_record = {
+                "group_id": group_id,
+                "group_name": group_name,
+                "group_description": group_description,
+                "group_identity_store_id": group_identity_store_id,
+                "member_user_id": member_user_id,
+                "member_email": member_email,
+                "member_given_name": member_given_name,
+                "member_family_name": member_family_name,
+                "member_display_name": member_display_name,
+            }
+            flattened_data.append(flattened_record)
+
+    return pd.DataFrame(flattened_data)

app/integrations/google_workspace/google_directory.py

@@ -1,6 +1,8 @@
 """Google Directory module to interact with the Google Workspace Directory API."""
 
 from logging import getLogger
+
+import pandas as pd
 from integrations.google_workspace.google_service import (
     handle_google_api_errors,
     execute_google_api_call,
@@ -171,17 +173,13 @@ def add_users_to_group(group, group_key):
 
 
 def list_groups_with_members(
-    group_members: bool = True,
-    members_details: bool = True,
     groups_filters: list = [],
     query: str | None = None,
     tolerate_errors: bool = False,
 ):
     """List all groups in the Google Workspace domain with their members.
 
     Args:
-        group_members (bool): Include the group members in the response.
-        members_details (bool): Include the members details in the response.
         groups_filters (list): List of filters to apply to the groups.
         query (str): The query to search for groups.
         tolerate_errors (bool): Whether to include groups that encountered errors during member detail retrieval.
@@ -199,11 +197,19 @@ def list_groups_with_members(
         for groups_filter in groups_filters:
             groups = filters.filter_by_condition(groups, groups_filter)
     logger.info(f"Found {len(groups)} groups.")
-    if not group_members:
-        return groups
+
+    filtered_groups = [
+        {
+            k: v
+            for k, v in group.items()
+            if k in ["id", "email", "name", "directMembersCount", "description"]
+        }
+        for group in groups
+    ]
 
     groups_with_members = []
-    for group in groups:
+    for group in filtered_groups:
+        error_occured = False
         logger.info(f"Getting members for group: {group['email']}")
         try:
             members = list_group_members(
@@ -213,24 +219,65 @@ def list_groups_with_members(
             logger.warning(f"Error getting members for group {group['email']}: {e}")
             continue
 
-        if members and members_details:
-            detailed_members = []
-            error_occurred = False
-            for member in members:
-                try:
-                    logger.info(f"Getting user details for member: {member['email']}")
-                    detailed_members.append(
-                        get_user(member["email"], fields="name, primaryEmail")
-                    )
-                except Exception as e:
-                    logger.warning(
-                        f"Error getting user details for member {member['email']}: {e}"
-                    )
-                    error_occurred = True
-                    if not tolerate_errors:
-                        break
-            if error_occurred and not tolerate_errors:
-                continue
-            group["members"] = detailed_members
+        for member in members:
+            user_details = {}
+            try:
+                logger.info(f"Getting user details for member: {member['email']}")
+                user_details = get_user(member["email"], fields="name, primaryEmail")
+            except Exception as e:
+                logger.warning(
+                    f"Error getting user details for member {member['email']}: {e}"
+                )
+                error_occured = True
+                if not tolerate_errors:
+                    break
+            if user_details:
+                member.update(user_details)
+        if members and (not error_occured or tolerate_errors):
+            group.update({"members": members})
             groups_with_members.append(group)
+
     return groups_with_members
+
+
+def convert_google_groups_members_to_dataframe(groups):
+    """Converts a list of Google groups with members to a DataFrame.
+
+    Args:
+        groups (list): A list of group objects with members.
+
+    Returns:
+        DataFrame: A DataFrame with group members.
+    """
+    flattened_data = []
+    for group in groups:
+        group_email = group.get("email")
+        group_name = group.get("name")
+        group_direct_members_count = group.get("directMembersCount")
+        group_description = group.get("description")
+
+        for member in group.get("members", []):
+            member_email = member.get("email")
+            member_role = member.get("role")
+            member_type = member.get("type")
+            member_status = member.get("status")
+            member_primary_email = member.get("primaryEmail")
+            member_given_name = member.get("name", {}).get("givenName")
+            member_family_name = member.get("name", {}).get("familyName")
+
+            flattened_record = {
+                "group_email": group_email,
+                "group_name": group_name,
+                "group_direct_members_count": group_direct_members_count,
+                "group_description": group_description,
+                "member_email": member_email,
+                "member_role": member_role,
+                "member_type": member_type,
+                "member_status": member_status,
+                "member_primary_email": member_primary_email,
+                "member_given_name": member_given_name,
+                "member_family_name": member_family_name,
+            }
+            flattened_data.append(flattened_record)
+
+    return pd.DataFrame(flattened_data)

app/modules/provisioning/groups.py

@@ -12,8 +12,7 @@ def get_groups_from_integration(
     pre_processing_filters: list = [],
     post_processing_filters: list = [],
     query: str | None = None,
-    group_members: bool = True,
-    members_details: bool = True,
+    return_dataframe: bool = False,
 ) -> list:
     """Retrieve the users from an integration group source.
     Supported sources are:
@@ -39,22 +38,26 @@ def get_groups_from_integration(
         case "google_groups":
             logger.info("Getting Google Groups with members.")
             groups = google_directory.list_groups_with_members(
-                group_members=group_members,
-                members_details=members_details,
                 groups_filters=pre_processing_filters,
                 query=query,
             )
+            if return_dataframe:
+                groups_dataframe = (
+                    google_directory.convert_google_groups_members_to_dataframe(groups)
+                )
             integration_name = "Google"
             group_display_key = "name"
             members = "members"
             members_display_key = "primaryEmail"
         case "aws_identity_center":
             logger.info("Getting AWS Identity Center Groups with members.")
             groups = identity_store.list_groups_with_memberships(
-                group_members=group_members,
-                members_details=members_details,
                 groups_filters=pre_processing_filters,
             )
+            if return_dataframe:
+                groups_dataframe = (
+                    identity_store.convert_aws_groups_members_to_dataframe(groups)
+                )
             integration_name = "AWS"
             group_display_key = "DisplayName"
             members = "GroupMemberships"
@@ -69,18 +72,16 @@ def get_groups_from_integration(
         groups,
         group_display_key=group_display_key,
         members=members,
-        members_details=members_details,
         members_display_key=members_display_key,
         integration_name=integration_name,
     )
-    return groups
+    return groups_dataframe if return_dataframe else groups
 
 
 def log_groups(
     groups,
     group_display_key=None,
     members=None,
-    members_details=True,
     members_display_key=None,
     integration_name="No Integration Name Provided",
 ):
@@ -110,12 +111,9 @@ def log_groups(
                 members_display_name = filters.get_nested_value(
                     member, members_display_key
                 )
-                if not members_display_name and members_details:
+                if not members_display_name:
                     members_display_name = "<User Name not found>"
-                if members_details:
-                    logger.info(
-                        f"{integration_name}Group:Member: {members_display_name}"
-                    )
+                logger.info(f"{integration_name}Group:Member: {members_display_name}")
         else:
             logger.info(
                 f"{integration_name}Group: {group_display_name} has no members."

app/tests/conftest.py

@@ -13,6 +13,8 @@ def _google_groups(n=3, prefix="", domain="test.com"):
                 "id": f"{prefix}google_group_id{i+1}",
                 "name": f"{prefix}group-name{i+1}",
                 "email": f"{prefix}group-name{i+1}@{domain}",
+                "description": f"{prefix}description{i+1}",
+                "directMembersCount": i + 1,
             }
             for i in range(n)
         ]
@@ -70,14 +72,21 @@ def _google_group_members(n=3, prefix="", domain="test.com"):
 
 # Fixture with users
 @pytest.fixture
-def google_groups_w_users(google_groups, google_users):
+def google_groups_w_users(google_groups, google_group_members, google_users):
     def _google_groups_w_users(
         n_groups=1, n_users=3, group_prefix="", user_prefix="", domain="test.com"
     ):
         groups = google_groups(n_groups, prefix=group_prefix, domain=domain)
+        members = google_group_members(n_users, prefix=user_prefix, domain=domain)
         users = google_users(n_users, prefix=user_prefix, domain=domain)
+
+        combined_members = []
+        for member, user in zip(members, users):
+            combined_member = {**member, **user}
+            combined_members.append(combined_member)
+
         for group in groups:
-            group["members"] = users
+            group["members"] = combined_members
         return groups
 
     return _google_groups_w_users