From e2b7ecbc7138cbf4aa7fd8cd04bf0bc98bfb68eb Mon Sep 17 00:00:00 2001
From: Guillaume Charest <1690085+gcharest@users.noreply.github.com>
Date: Tue, 1 Oct 2024 17:45:45 +0000
Subject: [PATCH 01/10] feat: ensure only required keys are kept

---
 .../google_workspace/google_directory.py        | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/app/integrations/google_workspace/google_directory.py b/app/integrations/google_workspace/google_directory.py
index 314e2349..491eaf78 100644
--- a/app/integrations/google_workspace/google_directory.py
+++ b/app/integrations/google_workspace/google_directory.py
@@ -195,9 +195,18 @@ def list_groups_with_members(
     if not groups:
         return []
 
+    filtered_keys_groups = [
+        {
+            k: v
+            for k, v in group.items()
+            if k in ["id", "email", "name", "directMembersCount", "description"]
+        }
+        for group in groups
+    ]
+
     if groups_filters is not None:
         for groups_filter in groups_filters:
-            groups = filters.filter_by_condition(groups, groups_filter)
+            groups = filters.filter_by_condition(filtered_keys_groups, groups_filter)
     logger.info(f"Found {len(groups)} groups.")
     if not group_members:
         return groups
@@ -219,9 +228,11 @@ def list_groups_with_members(
             for member in members:
                 try:
                     logger.info(f"Getting user details for member: {member['email']}")
-                    detailed_members.append(
-                        get_user(member["email"], fields="name, primaryEmail")
+                    user_details = get_user(
+                        member["email"], fields="name, primaryEmail"
                     )
+                    combined_user_details = {**member, **user_details}
+                    detailed_members.append(combined_user_details)
                 except Exception as e:
                     logger.warning(
                         f"Error getting user details for member {member['email']}: {e}"

From 0a599c0a6e77e9f9c61b0e2ba97ef097de6bcebc Mon Sep 17 00:00:00 2001
From: Guillaume Charest <1690085+gcharest@users.noreply.github.com>
Date: Tue, 1 Oct 2024 17:46:04 +0000
Subject: [PATCH 02/10] fix: adjust the fixture for expected google groups with
 members

---
 app/tests/conftest.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/app/tests/conftest.py b/app/tests/conftest.py
index fa1cc017..12892625 100644
--- a/app/tests/conftest.py
+++ b/app/tests/conftest.py
@@ -70,14 +70,21 @@ def _google_group_members(n=3, prefix="", domain="test.com"):
 
 # Fixture with users
 @pytest.fixture
-def google_groups_w_users(google_groups, google_users):
+def google_groups_w_users(google_groups, google_group_members, google_users):
     def _google_groups_w_users(
         n_groups=1, n_users=3, group_prefix="", user_prefix="", domain="test.com"
     ):
         groups = google_groups(n_groups, prefix=group_prefix, domain=domain)
+        members = google_group_members(n_users, prefix=user_prefix, domain=domain)
         users = google_users(n_users, prefix=user_prefix, domain=domain)
+
+        combined_members = []
+        for member, user in zip(members, users):
+            combined_member = {**member, **user}
+            combined_members.append(combined_member)
+
         for group in groups:
-            group["members"] = users
+            group["members"] = combined_members
         return groups
 
     return _google_groups_w_users

From 7d88223c3d2e23a073397bf767d5f6f0f6b55dae Mon Sep 17 00:00:00 2001
From: Guillaume Charest <1690085+gcharest@users.noreply.github.com>
Date: Tue, 1 Oct 2024 18:06:07 +0000
Subject: [PATCH 03/10] feat: add function to export groups to data frame

---
 .../google_workspace/google_directory.py      | 21 ++++++++
 app/tests/conftest.py                         |  2 +
 .../google_workspace/test_google_directory.py | 54 +++++++++++++++++++
 3 files changed, 77 insertions(+)

diff --git a/app/integrations/google_workspace/google_directory.py b/app/integrations/google_workspace/google_directory.py
index 491eaf78..7867b19b 100644
--- a/app/integrations/google_workspace/google_directory.py
+++ b/app/integrations/google_workspace/google_directory.py
@@ -1,6 +1,8 @@
 """Google Directory module to interact with the Google Workspace Directory API."""
 
 from logging import getLogger
+
+import pandas as pd
 from integrations.google_workspace.google_service import (
     handle_google_api_errors,
     execute_google_api_call,
@@ -244,4 +246,23 @@ def list_groups_with_members(
                 continue
             group["members"] = detailed_members
             groups_with_members.append(group)
+
     return groups_with_members
+
+
+def convert_group_members_to_dataframe(groups):
+    """Converts a list of groups with members to a DataFrame.
+
+    Args:
+        groups (list): A list of group objects with members.
+
+    Returns:
+        DataFrame: A DataFrame with group members.
+    """
+    flattened_data = []
+    for group in groups:
+        for member in group.get("members", []):
+            flattened_record = {**group, **member}
+            flattened_record.pop("members", None)
+            flattened_data.append(flattened_record)
+    return pd.DataFrame(flattened_data)
diff --git a/app/tests/conftest.py b/app/tests/conftest.py
index 12892625..33164219 100644
--- a/app/tests/conftest.py
+++ b/app/tests/conftest.py
@@ -13,6 +13,8 @@ def _google_groups(n=3, prefix="", domain="test.com"):
                 "id": f"{prefix}google_group_id{i+1}",
                 "name": f"{prefix}group-name{i+1}",
                 "email": f"{prefix}group-name{i+1}@{domain}",
+                "description": f"{prefix}description{i+1}",
+                "directMembersCount": i + 1,
             }
             for i in range(n)
         ]
diff --git a/app/tests/integrations/google_workspace/test_google_directory.py b/app/tests/integrations/google_workspace/test_google_directory.py
index e483a28c..4f9323cb 100644
--- a/app/tests/integrations/google_workspace/test_google_directory.py
+++ b/app/tests/integrations/google_workspace/test_google_directory.py
@@ -1,6 +1,8 @@
 """Unit tests for google_directory module."""
 
 from unittest.mock import patch
+
+import pandas as pd
 from integrations.google_workspace import google_directory
 
 
@@ -525,3 +527,55 @@ def test_list_groups_with_members_tolerate_errors(
 def test_list_groups_with_members_skips_when_no_groups(mock_list_groups):
     mock_list_groups.return_value = []
     assert google_directory.list_groups_with_members() == []
+
+
+@patch("integrations.google_workspace.google_directory.filters.filter_by_condition")
+@patch("integrations.google_workspace.google_directory.list_groups")
+@patch("integrations.google_workspace.google_directory.list_group_members")
+@patch("integrations.google_workspace.google_directory.get_user")
+def test_list_groups_with_members_filtered_dataframe(
+    mock_get_user,
+    mock_list_group_members,
+    mock_list_groups,
+    mock_filter_by_condition,
+    google_groups,
+    google_group_members,
+    google_users,
+    google_groups_w_users,
+):
+    groups = google_groups(2, prefix="test-")
+    groups_to_filter_out = google_groups(4)[2:]
+    groups.extend(groups_to_filter_out)
+    group_members = [[], google_group_members(2)]
+    users = google_users(2)
+
+    groups_with_users = google_groups_w_users(4, 2, group_prefix="test-")[:2]
+    groups_with_users.remove(groups_with_users[0])
+
+    mock_list_groups.return_value = groups
+    mock_list_group_members.side_effect = group_members
+    mock_get_user.side_effect = users
+    mock_filter_by_condition.return_value = groups[:2]
+    groups_filters = [lambda group: "test-" in group["name"]]
+
+    groups_result = google_directory.list_groups_with_members(
+        groups_filters=groups_filters
+    )
+    result = google_directory.convert_group_members_to_dataframe(groups_result)
+
+    assert isinstance(result, pd.DataFrame)
+    assert not result.empty
+    assert set(result.columns) == {
+        "id",
+        "email",
+        "name",
+        "directMembersCount",
+        "description",
+        "role",
+        "type",
+        "status",
+        "primaryEmail",
+        "emails",
+        "suspended",
+        "kind",
+    }

From 82391e47f1c8c7132bd8ac808f4ede1d080aef65 Mon Sep 17 00:00:00 2001
From: Guillaume Charest <1690085+gcharest@users.noreply.github.com>
Date: Tue, 1 Oct 2024 18:37:35 +0000
Subject: [PATCH 04/10] fix: return properly formatted groups data in dataframe

---
 .../google_workspace/google_directory.py      | 33 ++++++++++++++++---
 .../google_workspace/test_google_directory.py | 25 +++++++-------
 2 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/app/integrations/google_workspace/google_directory.py b/app/integrations/google_workspace/google_directory.py
index 7867b19b..ef68d108 100644
--- a/app/integrations/google_workspace/google_directory.py
+++ b/app/integrations/google_workspace/google_directory.py
@@ -250,8 +250,8 @@ def list_groups_with_members(
     return groups_with_members
 
 
-def convert_group_members_to_dataframe(groups):
-    """Converts a list of groups with members to a DataFrame.
+def convert_google_groups_members_to_dataframe(groups):
+    """Converts a list of Google groups with members to a DataFrame.
 
     Args:
         groups (list): A list of group objects with members.
@@ -261,8 +261,33 @@ def convert_group_members_to_dataframe(groups):
     """
     flattened_data = []
     for group in groups:
+        group_email = group.get("email")
+        group_name = group.get("name")
+        group_direct_members_count = group.get("directMembersCount")
+        group_description = group.get("description")
+
         for member in group.get("members", []):
-            flattened_record = {**group, **member}
-            flattened_record.pop("members", None)
+            member_email = member.get("email")
+            member_role = member.get("role")
+            member_type = member.get("type")
+            member_status = member.get("status")
+            member_primary_email = member.get("primaryEmail")
+            member_given_name = member.get("name", {}).get("givenName")
+            member_family_name = member.get("name", {}).get("familyName")
+
+            flattened_record = {
+                "group_email": group_email,
+                "group_name": group_name,
+                "group_direct_members_count": group_direct_members_count,
+                "group_description": group_description,
+                "member_email": member_email,
+                "member_role": member_role,
+                "member_type": member_type,
+                "member_status": member_status,
+                "member_primary_email": member_primary_email,
+                "member_given_name": member_given_name,
+                "member_family_name": member_family_name,
+            }
             flattened_data.append(flattened_record)
+
     return pd.DataFrame(flattened_data)
diff --git a/app/tests/integrations/google_workspace/test_google_directory.py b/app/tests/integrations/google_workspace/test_google_directory.py
index 4f9323cb..6a0cc783 100644
--- a/app/tests/integrations/google_workspace/test_google_directory.py
+++ b/app/tests/integrations/google_workspace/test_google_directory.py
@@ -561,21 +561,20 @@ def test_list_groups_with_members_filtered_dataframe(
     groups_result = google_directory.list_groups_with_members(
         groups_filters=groups_filters
     )
-    result = google_directory.convert_group_members_to_dataframe(groups_result)
+    result = google_directory.convert_google_groups_members_to_dataframe(groups_result)
 
     assert isinstance(result, pd.DataFrame)
     assert not result.empty
     assert set(result.columns) == {
-        "id",
-        "email",
-        "name",
-        "directMembersCount",
-        "description",
-        "role",
-        "type",
-        "status",
-        "primaryEmail",
-        "emails",
-        "suspended",
-        "kind",
+        "group_email",
+        "group_name",
+        "group_direct_members_count",
+        "group_description",
+        "member_email",
+        "member_role",
+        "member_type",
+        "member_status",
+        "member_primary_email",
+        "member_given_name",
+        "member_family_name",
     }

From 98cccf7b4548d5b2372ea1910b01e95fe38e7932 Mon Sep 17 00:00:00 2001
From: Guillaume Charest <1690085+gcharest@users.noreply.github.com>
Date: Wed, 2 Oct 2024 18:24:41 +0000
Subject: [PATCH 05/10] fix: update list groups w members to handle errors

---
 .../google_workspace/google_directory.py      |  57 +++----
 .../google_workspace/test_google_directory.py | 155 +++++++++++-------
 2 files changed, 124 insertions(+), 88 deletions(-)

diff --git a/app/integrations/google_workspace/google_directory.py b/app/integrations/google_workspace/google_directory.py
index ef68d108..f3496a2f 100644
--- a/app/integrations/google_workspace/google_directory.py
+++ b/app/integrations/google_workspace/google_directory.py
@@ -197,7 +197,12 @@ def list_groups_with_members(
     if not groups:
         return []
 
-    filtered_keys_groups = [
+    if groups_filters is not None:
+        for groups_filter in groups_filters:
+            groups = filters.filter_by_condition(groups, groups_filter)
+    logger.info(f"Found {len(groups)} groups.")
+
+    filtered_groups = [
         {
             k: v
             for k, v in group.items()
@@ -206,15 +211,9 @@ def list_groups_with_members(
         for group in groups
     ]
 
-    if groups_filters is not None:
-        for groups_filter in groups_filters:
-            groups = filters.filter_by_condition(filtered_keys_groups, groups_filter)
-    logger.info(f"Found {len(groups)} groups.")
-    if not group_members:
-        return groups
-
     groups_with_members = []
-    for group in groups:
+    for group in filtered_groups:
+        error_occured = False
         logger.info(f"Getting members for group: {group['email']}")
         try:
             members = list_group_members(
@@ -224,27 +223,25 @@ def list_groups_with_members(
             logger.warning(f"Error getting members for group {group['email']}: {e}")
             continue
 
-        if members and members_details:
-            detailed_members = []
-            error_occurred = False
-            for member in members:
-                try:
-                    logger.info(f"Getting user details for member: {member['email']}")
-                    user_details = get_user(
-                        member["email"], fields="name, primaryEmail"
-                    )
-                    combined_user_details = {**member, **user_details}
-                    detailed_members.append(combined_user_details)
-                except Exception as e:
-                    logger.warning(
-                        f"Error getting user details for member {member['email']}: {e}"
-                    )
-                    error_occurred = True
-                    if not tolerate_errors:
-                        break
-            if error_occurred and not tolerate_errors:
-                continue
-            group["members"] = detailed_members
+        for member in members:
+            user_details = {}
+            try:
+                logger.info(f"Getting user details for member: {member['email']}")
+                user_details = get_user(
+                    member["email"], fields="name, primaryEmail"
+                )
+            except Exception as e:
+                logger.warning(
+                    f"Error getting user details for member {member['email']}: {e}"
+                )
+                error_occured = True
+                if not tolerate_errors:
+                    break
+            if user_details:
+                member.update(user_details)
+        if members and (not error_occured or tolerate_errors):
+            group.update({"members": members})
+            logger.info(f"Group {group['email']} has {len(members)} members.")
             groups_with_members.append(group)
 
     return groups_with_members
diff --git a/app/tests/integrations/google_workspace/test_google_directory.py b/app/tests/integrations/google_workspace/test_google_directory.py
index 6a0cc783..445c7a91 100644
--- a/app/tests/integrations/google_workspace/test_google_directory.py
+++ b/app/tests/integrations/google_workspace/test_google_directory.py
@@ -417,7 +417,9 @@ def test_list_groups_with_members_error_in_list_group_members(
 
     # Only the second group should be processed
     expected_groups_with_users = [groups[1]]
-    expected_groups_with_users[0]["members"] = users
+    expected_groups_with_users[0]["members"] = group_members[1]
+    expected_groups_with_users[0]["members"][0].update(users[0])
+    expected_groups_with_users[0]["members"][1].update(users[1])
 
     assert google_directory.list_groups_with_members() == expected_groups_with_users
 
@@ -425,88 +427,77 @@ def test_list_groups_with_members_error_in_list_group_members(
 @patch("integrations.google_workspace.google_directory.list_groups")
 @patch("integrations.google_workspace.google_directory.list_group_members")
 @patch("integrations.google_workspace.google_directory.get_user")
-def test_list_groups_with_members_without_details(
+def test_list_groups_with_members_error_in_get_user(
     mock_get_user,
     mock_list_group_members,
     mock_list_groups,
     google_groups,
     google_group_members,
     google_users,
-    google_groups_w_users,
 ):
     groups = google_groups(2)
-    group_members = [[], google_group_members(2)]
-    users = google_users(2)
-    groups_with_users = google_groups_w_users(2, 2)
-
-    groups_with_users[0].pop("members", None)
-    groups_with_users[1].pop("members", None)
+    group_members = [google_group_members(2), google_group_members(2)]
+    users = [
+        Exception("Error fetching user details"),
+        google_users(2)[1],
+        google_users(1)[0],
+        google_users(2)[1],
+    ]
 
     mock_list_groups.return_value = groups
     mock_list_group_members.side_effect = group_members
     mock_get_user.side_effect = users
 
-    assert google_directory.list_groups_with_members(members_details=False) == []
-
-
-@patch("integrations.google_workspace.google_directory.list_groups")
-@patch("integrations.google_workspace.google_directory.list_group_members")
-@patch("integrations.google_workspace.google_directory.get_user")
-def test_list_groups_with_members_without_members_enabled(
-    mock_get_user,
-    mock_list_group_members,
-    mock_list_groups,
-    google_groups,
-):
-    groups = google_groups(2)
-    mock_list_groups.return_value = groups
-    assert google_directory.list_groups_with_members(group_members=False) == groups
+    # Only the second group should be processed
+    expected_groups_with_users = [groups[1]]
+    expected_groups_with_users[0]["members"] = group_members[1]
+    expected_groups_with_users[0]["members"][0].update(users[2])
+    expected_groups_with_users[0]["members"][1].update(users[3])
 
-    mock_list_group_members.assert_not_called()
-    mock_get_user.assert_not_called()
+    assert google_directory.list_groups_with_members() == expected_groups_with_users
 
 
 @patch("integrations.google_workspace.google_directory.list_groups")
 @patch("integrations.google_workspace.google_directory.list_group_members")
 @patch("integrations.google_workspace.google_directory.get_user")
-def test_list_groups_with_members_error_in_get_user(
+def test_list_groups_with_members_tolerate_errors(
     mock_get_user,
     mock_list_group_members,
     mock_list_groups,
-    google_groups,
-    google_group_members,
-    google_users,
+    google_groups_w_users,
 ):
-    groups = google_groups(2)
-    group_members = [google_group_members(2), google_group_members(2)]
-    users = [Exception("Error fetching user details"), google_users(1)[0]]
 
-    mock_list_groups.return_value = groups
-    mock_list_group_members.side_effect = group_members
-    mock_get_user.side_effect = users
-
-    # No groups should be processed due to error in get_user
-    assert google_directory.list_groups_with_members() == []
+    groups = [
+        {
+            "id": "group1",
+            "email": "groupEmail1",
+            "name": "name1",
+            "directMembersCount": 2,
+        },
+        {
+            "id": "group2",
+            "email": "groupEmail2",
+            "name": "name2",
+            "directMembersCount": 2,
+        },
+    ]
 
+    group_members = [
+        [
+            {"email": "email1", "role": "MEMBER", "type": "USER", "status": "ACTIVE"},
+            {"email": "email2", "role": "MEMBER", "type": "USER", "status": "ACTIVE"},
+        ],
+        [
+            {"email": "email3", "role": "MEMBER", "type": "USER", "status": "ACTIVE"},
+            {"email": "email4", "role": "MEMBER", "type": "USER", "status": "ACTIVE"},
+        ],
+    ]
 
-@patch("integrations.google_workspace.google_directory.list_groups")
-@patch("integrations.google_workspace.google_directory.list_group_members")
-@patch("integrations.google_workspace.google_directory.get_user")
-def test_list_groups_with_members_tolerate_errors(
-    mock_get_user,
-    mock_list_group_members,
-    mock_list_groups,
-    google_groups,
-    google_group_members,
-    google_users,
-):
-    groups = google_groups(2)
-    group_members = [google_group_members(2), google_group_members(2)]
     users = [
         Exception("Error fetching user details"),
-        google_users(1)[0],
-        Exception("Error fetching user details"),
-        google_users(1)[0],
+        {"id": "user2", "name": "user2", "primaryEmail": "email2"},
+        {"id": "user3", "name": "user3", "primaryEmail": "email3"},
+        {"id": "user4", "name": "user4", "primaryEmail": "email4"},
     ]
 
     mock_list_groups.return_value = groups
@@ -514,9 +505,57 @@ def test_list_groups_with_members_tolerate_errors(
     mock_get_user.side_effect = users
 
     # Expected result should include both groups, with the second group having one member
-    expected_groups_with_users = [groups[0], groups[1]]
-    expected_groups_with_users[0]["members"] = []
-    expected_groups_with_users[1]["members"] = [google_users(1)[0]]
+    expected_groups_with_users = [
+        {
+            "id": "group1",
+            "email": "groupEmail1",
+            "name": "name1",
+            "directMembersCount": 2,
+            "members": [
+                {
+                    "email": "email1",
+                    "role": "MEMBER",
+                    "type": "USER",
+                    "status": "ACTIVE",
+                },
+                {
+                    "email": "email2",
+                    "primaryEmail": "email2",
+                    "role": "MEMBER",
+                    "type": "USER",
+                    "status": "ACTIVE",
+                    "id": "user2",
+                    "name": "user2",
+                },
+            ],
+        },
+        {
+            "id": "group2",
+            "email": "groupEmail2",
+            "name": "name2",
+            "directMembersCount": 2,
+            "members": [
+                {
+                    "email": "email3",
+                    "primaryEmail": "email3",
+                    "role": "MEMBER",
+                    "type": "USER",
+                    "status": "ACTIVE",
+                    "id": "user3",
+                    "name": "user3",
+                },
+                {
+                    "email": "email4",
+                    "primaryEmail": "email4",
+                    "role": "MEMBER",
+                    "type": "USER",
+                    "status": "ACTIVE",
+                    "id": "user4",
+                    "name": "user4",
+                },
+            ],
+        },
+    ]
 
     result = google_directory.list_groups_with_members(tolerate_errors=True)
 

From 1f5ef1ec389ce494da16656864f30478e66a019f Mon Sep 17 00:00:00 2001
From: Guillaume Charest <1690085+gcharest@users.noreply.github.com>
Date: Wed, 2 Oct 2024 19:07:50 +0000
Subject: [PATCH 06/10] fix: simplify method's logic

---
 app/integrations/google_workspace/google_directory.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/app/integrations/google_workspace/google_directory.py b/app/integrations/google_workspace/google_directory.py
index f3496a2f..c5ed4569 100644
--- a/app/integrations/google_workspace/google_directory.py
+++ b/app/integrations/google_workspace/google_directory.py
@@ -173,8 +173,6 @@ def add_users_to_group(group, group_key):
 
 
 def list_groups_with_members(
-    group_members: bool = True,
-    members_details: bool = True,
     groups_filters: list = [],
     query: str | None = None,
     tolerate_errors: bool = False,
@@ -182,8 +180,6 @@ def list_groups_with_members(
     """List all groups in the Google Workspace domain with their members.
 
     Args:
-        group_members (bool): Include the group members in the response.
-        members_details (bool): Include the members details in the response.
         groups_filters (list): List of filters to apply to the groups.
         query (str): The query to search for groups.
         tolerate_errors (bool): Whether to include groups that encountered errors during member detail retrieval.
@@ -227,9 +223,7 @@ def list_groups_with_members(
             user_details = {}
             try:
                 logger.info(f"Getting user details for member: {member['email']}")
-                user_details = get_user(
-                    member["email"], fields="name, primaryEmail"
-                )
+                user_details = get_user(member["email"], fields="name, primaryEmail")
             except Exception as e:
                 logger.warning(
                     f"Error getting user details for member {member['email']}: {e}"

From 50a7bcdf8932a7ea513fea16675ae69b75dc742c Mon Sep 17 00:00:00 2001
From: Guillaume Charest <1690085+gcharest@users.noreply.github.com>
Date: Wed, 2 Oct 2024 19:08:44 +0000
Subject: [PATCH 07/10] fix: simplify method and handle errors

---
 app/integrations/aws/identity_store.py        | 94 ++++++++++++++++---
 .../integrations/aws/test_identity_store.py   | 64 +------------
 2 files changed, 83 insertions(+), 75 deletions(-)

diff --git a/app/integrations/aws/identity_store.py b/app/integrations/aws/identity_store.py
index f6d288f6..24a9a313 100644
--- a/app/integrations/aws/identity_store.py
+++ b/app/integrations/aws/identity_store.py
@@ -1,5 +1,8 @@
+import json
 import os
 import logging
+
+import pandas as pd
 from integrations.aws.client import execute_aws_api_call, handle_aws_api_errors
 from utils import filters
 
@@ -288,10 +291,8 @@ def list_group_memberships(group_id, **kwargs):
 
 @handle_aws_api_errors
 def list_groups_with_memberships(
-    group_members: bool = True,
-    members_details: bool = True,
-    include_empty_groups: bool = True,
     groups_filters: list | None = None,
+    tolerate_errors: bool = False,
 ):
     """Retrieves groups with their members from the AWS Identity Center (identitystore)
 
@@ -314,18 +315,85 @@ def list_groups_with_memberships(
         for groups_filter in groups_filters:
             groups = filters.filter_by_condition(groups, groups_filter)
     logger.info(f"Founds {len(groups)} groups in AWS Identity Store.")
-    if not group_members:
-        return groups
+
+    filtered_groups = [
+        {
+            k: v
+            for k, v in group.items()
+            if k in ["GroupId", "DisplayName", "Description"]
+        }
+        for group in groups
+    ]
 
     groups_with_memberships = []
-    for group in groups:
-        group["GroupMemberships"] = list_group_memberships(group["GroupId"])
+    for group in filtered_groups:
+        error_occurred = False
+        logger.info(f"Getting members for group: {group['DisplayName']}")
+        try:
+            # memberships = list_group_memberships(group["GroupId"])
+            group["GroupMemberships"] = list_group_memberships(group["GroupId"])
+        except Exception as error:
+            logger.warning(
+                f"Error getting members for group {group['GroupId']}: {error}"
+            )
+            continue
+
+        for membership in group["GroupMemberships"]:
+            member_details = {}
+            try:
+                logger.info(
+                    f"Getting details for member: {membership['MemberId']['UserId']}"
+                )
+                member_details = describe_user(membership["MemberId"]["UserId"])
+            except Exception as error:
+                logger.warning(
+                    f"Error getting details for member {membership['MemberId']['UserId']}: {error}"
+                )
+                error_occurred = True
+                if not tolerate_errors:
+                    break
+            if member_details and (not error_occurred or tolerate_errors):
+                membership.update(member_details)
         if group["GroupMemberships"]:
-            if members_details:
-                for membership in group["GroupMemberships"]:
-                    membership["MemberId"] = describe_user(
-                        membership["MemberId"]["UserId"]
-                    )
-        if group["GroupMemberships"] or include_empty_groups:
             groups_with_memberships.append(group)
     return groups_with_memberships
+
+
+def convert_aws_groups_members_to_dataframe(groups):
+    """Converts a list of AWS groups with members to a DataFrame.
+
+    Args:
+        groups (list): A list of group objects with members.
+
+    Returns:
+        DataFrame: A DataFrame with group members.
+    """
+    flattened_data = []
+    for group in groups:
+        group_id = group.get("GroupId")
+        group_name = group.get("DisplayName")
+        group_description = group.get("Description")
+        group_identity_store_id = group.get("IdentityStoreId")
+
+        for membership in group.get("GroupMemberships", []):
+            member = membership.get("MemberId", {})
+            member_user_id = member.get("UserId")
+            member_email = member.get("UserName")
+            member_given_name = member.get("Name", {}).get("GivenName")
+            member_family_name = member.get("Name", {}).get("FamilyName")
+            member_display_name = member.get("DisplayName")
+
+            flattened_record = {
+                "group_id": group_id,
+                "group_name": group_name,
+                "group_description": group_description,
+                "group_identity_store_id": group_identity_store_id,
+                "member_user_id": member_user_id,
+                "member_email": member_email,
+                "member_given_name": member_given_name,
+                "member_family_name": member_family_name,
+                "member_display_name": member_display_name,
+            }
+            flattened_data.append(flattened_record)
+
+    return pd.DataFrame(flattened_data)
diff --git a/app/tests/integrations/aws/test_identity_store.py b/app/tests/integrations/aws/test_identity_store.py
index 6dd49fbb..01b99b44 100644
--- a/app/tests/integrations/aws/test_identity_store.py
+++ b/app/tests/integrations/aws/test_identity_store.py
@@ -877,23 +877,6 @@ def test_list_groups_with_memberships_empty_groups(
     assert mock_describe_user.call_count == 0
 
 
-@patch("integrations.aws.identity_store.list_groups")
-@patch("integrations.aws.identity_store.list_group_memberships")
-@patch("integrations.aws.identity_store.describe_user")
-def test_list_groups_with_memberships_empty_groups_memberships_with_flag(
-    mock_describe_user, mock_list_group_memberships, mock_list_groups, aws_groups
-):
-    groups = aws_groups(2, prefix="test-")
-    expected_output = []
-    groups_memberships = [[], []]
-    mock_list_groups.return_value = groups
-    mock_list_group_memberships.side_effect = groups_memberships
-    result = identity_store.list_groups_with_memberships(include_empty_groups=False)
-    assert result == expected_output
-    assert mock_list_group_memberships.call_count == 2
-    assert mock_describe_user.call_count == 0
-
-
 @patch("integrations.aws.identity_store.filters.filter_by_condition")
 @patch("integrations.aws.identity_store.list_groups")
 @patch("integrations.aws.identity_store.list_group_memberships")
@@ -981,55 +964,12 @@ def test_list_groups_with_memberships_filtered(
 
     mock_list_group_memberships.side_effect = memberships
 
-    user_side_effect = []
-    for user in users:
-        user_side_effect.append(user)
-
-    mock_describe_user.side_effect = user_side_effect
+    mock_describe_user.side_effect = [users[0], users[1], users[0], users[1]]
     mock_filter_by_condition.return_value = groups[:2]
     groups_filters = [lambda group: "test-" in group["DisplayName"]]
     result = identity_store.list_groups_with_memberships(groups_filters=groups_filters)
 
+    assert result == expected_output
     assert mock_filter_by_condition.call_count == 1
     assert mock_list_group_memberships.call_count == 2
     assert mock_describe_user.call_count == 2
-    assert result == expected_output
-
-
-@patch("integrations.aws.identity_store.list_groups")
-@patch("integrations.aws.identity_store.list_group_memberships")
-@patch("integrations.aws.identity_store.describe_user")
-def test_list_groups_with_memberhips_without_members_enabled(
-    mock_describe_user,
-    mock_list_group_memberships,
-    mock_list_groups,
-    aws_groups,
-    aws_groups_memberships,
-):
-    groups = aws_groups(2, prefix="test-")
-    memberships = [
-        [],
-        aws_groups_memberships(2, prefix="test-", group_id=2)["GroupMemberships"],
-    ]
-    expected_output = [
-        {
-            "Description": "A group to test resolving AWS-group1 memberships",
-            "DisplayName": "test-group-name1",
-            "GroupId": "test-aws-group_id1",
-            "IdentityStoreId": "d-123412341234",
-        },
-        {
-            "GroupId": "test-aws-group_id2",
-            "DisplayName": "test-group-name2",
-            "Description": "A group to test resolving AWS-group2 memberships",
-            "IdentityStoreId": "d-123412341234",
-        },
-    ]
-    mock_list_groups.return_value = groups
-    mock_list_group_memberships.side_effect = memberships
-
-    result = identity_store.list_groups_with_memberships(group_members=False)
-
-    assert result == expected_output
-    assert mock_list_group_memberships.call_count == 0
-    assert mock_describe_user.call_count == 0

From 501321ca56873a240e4c61173fa1cfded996a72b Mon Sep 17 00:00:00 2001
From: Guillaume Charest <1690085+gcharest@users.noreply.github.com>
Date: Wed, 2 Oct 2024 19:08:59 +0000
Subject: [PATCH 08/10] fix: simplify method

---
 app/modules/provisioning/groups.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/app/modules/provisioning/groups.py b/app/modules/provisioning/groups.py
index 0f87b22d..b256de0d 100644
--- a/app/modules/provisioning/groups.py
+++ b/app/modules/provisioning/groups.py
@@ -12,8 +12,7 @@ def get_groups_from_integration(
     pre_processing_filters: list = [],
     post_processing_filters: list = [],
     query: str | None = None,
-    group_members: bool = True,
-    members_details: bool = True,
+    return_dataframe: bool = False,
 ) -> list:
     """Retrieve the users from an integration group source.
     Supported sources are:
@@ -39,11 +38,13 @@ def get_groups_from_integration(
         case "google_groups":
             logger.info("Getting Google Groups with members.")
             groups = google_directory.list_groups_with_members(
-                group_members=group_members,
-                members_details=members_details,
                 groups_filters=pre_processing_filters,
                 query=query,
             )
+            if return_dataframe:
+                groups_dataframe = (
+                    google_directory.convert_google_groups_members_to_dataframe(groups)
+                )
             integration_name = "Google"
             group_display_key = "name"
             members = "members"
@@ -51,10 +52,12 @@ def get_groups_from_integration(
         case "aws_identity_center":
             logger.info("Getting AWS Identity Center Groups with members.")
             groups = identity_store.list_groups_with_memberships(
-                group_members=group_members,
-                members_details=members_details,
                 groups_filters=pre_processing_filters,
             )
+            if return_dataframe:
+                groups_dataframe = (
+                    identity_store.convert_aws_groups_members_to_dataframe(groups)
+                )
             integration_name = "AWS"
             group_display_key = "DisplayName"
             members = "GroupMemberships"
@@ -69,18 +72,16 @@ def get_groups_from_integration(
         groups,
         group_display_key=group_display_key,
         members=members,
-        members_details=members_details,
         members_display_key=members_display_key,
         integration_name=integration_name,
     )
-    return groups
+    return groups_dataframe if return_dataframe else groups
 
 
 def log_groups(
     groups,
     group_display_key=None,
     members=None,
-    members_details=True,
     members_display_key=None,
     integration_name="No Integration Name Provided",
 ):
@@ -110,12 +111,9 @@ def log_groups(
                 members_display_name = filters.get_nested_value(
                     member, members_display_key
                 )
-                if not members_display_name and members_details:
+                if not members_display_name:
                     members_display_name = "<User Name not found>"
-                if members_details:
-                    logger.info(
-                        f"{integration_name}Group:Member: {members_display_name}"
-                    )
+                logger.info(f"{integration_name}Group:Member: {members_display_name}")
         else:
             logger.info(
                 f"{integration_name}Group: {group_display_name} has no members."

From 14807cb101e70e5fb6d590cea61060e559b04ce6 Mon Sep 17 00:00:00 2001
From: Guillaume Charest <1690085+gcharest@users.noreply.github.com>
Date: Wed, 2 Oct 2024 19:36:54 +0000
Subject: [PATCH 09/10] fix: cleanup logs

---
 app/integrations/aws/identity_store.py           | 16 +++++++---------
 .../google_workspace/google_directory.py         |  1 -
 .../integrations/aws/test_identity_store.py      | 14 --------------
 3 files changed, 7 insertions(+), 24 deletions(-)

diff --git a/app/integrations/aws/identity_store.py b/app/integrations/aws/identity_store.py
index 24a9a313..1bef4205 100644
--- a/app/integrations/aws/identity_store.py
+++ b/app/integrations/aws/identity_store.py
@@ -1,4 +1,3 @@
-import json
 import os
 import logging
 
@@ -320,7 +319,7 @@ def list_groups_with_memberships(
         {
             k: v
             for k, v in group.items()
-            if k in ["GroupId", "DisplayName", "Description"]
+            if k in ["GroupId", "DisplayName", "Description", "IdentityStoreId"]
         }
         for group in groups
     ]
@@ -330,15 +329,13 @@ def list_groups_with_memberships(
         error_occurred = False
         logger.info(f"Getting members for group: {group['DisplayName']}")
         try:
-            # memberships = list_group_memberships(group["GroupId"])
-            group["GroupMemberships"] = list_group_memberships(group["GroupId"])
+            memberships = list_group_memberships(group["GroupId"])
         except Exception as error:
             logger.warning(
                 f"Error getting members for group {group['GroupId']}: {error}"
             )
             continue
-
-        for membership in group["GroupMemberships"]:
+        for membership in memberships:
             member_details = {}
             try:
                 logger.info(
@@ -352,9 +349,10 @@ def list_groups_with_memberships(
                 error_occurred = True
                 if not tolerate_errors:
                     break
-            if member_details and (not error_occurred or tolerate_errors):
-                membership.update(member_details)
-        if group["GroupMemberships"]:
+            if member_details:
+                membership['MemberId'].update(member_details)
+        if memberships and (not error_occurred or tolerate_errors):
+            group["GroupMemberships"] = memberships
             groups_with_memberships.append(group)
     return groups_with_memberships
 
diff --git a/app/integrations/google_workspace/google_directory.py b/app/integrations/google_workspace/google_directory.py
index c5ed4569..9fe9943b 100644
--- a/app/integrations/google_workspace/google_directory.py
+++ b/app/integrations/google_workspace/google_directory.py
@@ -235,7 +235,6 @@ def list_groups_with_members(
                 member.update(user_details)
         if members and (not error_occured or tolerate_errors):
             group.update({"members": members})
-            logger.info(f"Group {group['email']} has {len(members)} members.")
             groups_with_members.append(group)
 
     return groups_with_members
diff --git a/app/tests/integrations/aws/test_identity_store.py b/app/tests/integrations/aws/test_identity_store.py
index 01b99b44..4efff2b5 100644
--- a/app/tests/integrations/aws/test_identity_store.py
+++ b/app/tests/integrations/aws/test_identity_store.py
@@ -787,13 +787,6 @@ def test_list_groups_with_memberships(
     ]
     users = aws_users(2, prefix="test-", domain="test.com")
     expected_output = [
-        {
-            "Description": "A group to test resolving AWS-group1 memberships",
-            "DisplayName": "test-group-name1",
-            "GroupId": "test-aws-group_id1",
-            "GroupMemberships": [],
-            "IdentityStoreId": "d-123412341234",
-        },
         {
             "GroupId": "test-aws-group_id2",
             "DisplayName": "test-group-name2",
@@ -900,13 +893,6 @@ def test_list_groups_with_memberships_filtered(
     users = aws_users(2, prefix="test-", domain="test.com")
 
     expected_output = [
-        {
-            "Description": "A group to test resolving AWS-group1 memberships",
-            "DisplayName": "test-group-name1",
-            "GroupId": "test-aws-group_id1",
-            "GroupMemberships": [],
-            "IdentityStoreId": "d-123412341234",
-        },
         {
             "GroupId": "test-aws-group_id2",
             "DisplayName": "test-group-name2",

From 2b67b488b751a507efb08b5ab681a9ff41c71db4 Mon Sep 17 00:00:00 2001
From: Guillaume Charest <1690085+gcharest@users.noreply.github.com>
Date: Wed, 2 Oct 2024 19:46:18 +0000
Subject: [PATCH 10/10] fix: fmt

---
 app/integrations/aws/identity_store.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/integrations/aws/identity_store.py b/app/integrations/aws/identity_store.py
index 1bef4205..6279a109 100644
--- a/app/integrations/aws/identity_store.py
+++ b/app/integrations/aws/identity_store.py
@@ -350,7 +350,7 @@ def list_groups_with_memberships(
                 if not tolerate_errors:
                     break
             if member_details:
-                membership['MemberId'].update(member_details)
+                membership["MemberId"].update(member_details)
         if memberships and (not error_occurred or tolerate_errors):
             group["GroupMemberships"] = memberships
             groups_with_memberships.append(group)