Feat/refactor get unique users to generic function

app/modules/aws/identity_center.py

@@ -1,7 +1,7 @@
 """Module to sync the AWS Identity Center with the Google Workspace."""
 from logging import getLogger
 from integrations.aws import identity_store
-from modules.provisioning import users, groups
+from modules.provisioning import groups
 from utils import filters
 
 
@@ -31,8 +31,7 @@ def synchronize(**kwargs):
     source_groups = groups.get_groups_with_members_from_integration(
         "google_groups", query=query, filters=source_groups_filters
     )
-    source_users = users.get_unique_users_from_groups(source_groups, "members")
-
+    source_users = filters.get_unique_nested_dicts(source_groups, "members")
     logger.info(
         f"synchronize:Found {len(source_groups)} Source Groups and {len(source_users)} Users"
     )

app/modules/provisioning/users.py

@@ -1,41 +1,12 @@
 # from logging import getLogger
 import logging
 
-from utils import filters as filter_tools
-
 
 logger = logging.getLogger(__name__)
 
 DISPLAY_KEYS = {"aws": "UserName", "google": "primaryEmail"}
 
 
-def get_unique_users_from_groups(groups, key):
-    """Get the unique users from a list of groups with the same data schema or a single group dict.
-    Considers the whole object for uniqueness, not specific keys.
-
-    Args:
-        groups (list or dict): A list of groups or a single group.
-        key (str): The key to get the users from the groups.
-
-    Returns:
-        list: A list of unique users from the groups
-    """
-    users_dict = {}
-    if isinstance(groups, list):
-        logger.info(f"Getting unique users from {len(groups)} groups.")
-        for group in groups:
-            for user in filter_tools.get_nested_value(group, key):
-                if user:
-                    users_dict[str(user)] = user
-    elif isinstance(groups, dict):
-        logger.info("Getting unique users from a single group.")
-        for user in filter_tools.get_nested_value(groups, key):
-            if user:
-                users_dict[str(user)] = user
-    logger.info(f"Found {len(users_dict)} unique users.")
-    return list(users_dict.values())
-
-
 def provision_users(
     integration, operation, function, users, display_key=None, **kwargs
 ):

app/tests/modules/aws/test_sync_identity_center.py

@@ -623,11 +623,11 @@ def test_synchronize_enable_delete_dry_run_true(
 @patch("modules.aws.identity_center.sync_groups")
 @patch("modules.aws.identity_center.sync_users")
 @patch("modules.aws.identity_center.identity_store.list_users")
-@patch("modules.aws.identity_center.users.get_unique_users_from_groups")
+@patch("modules.aws.identity_center.filters.get_unique_nested_dicts")
 @patch("modules.aws.identity_center.groups.get_groups_with_members_from_integration")
 def test_synchronize_sync_skip_users_if_false(
     mock_get_groups_with_members_from_integration,
-    mock_get_unique_users_from_groups,
+    mock_get_unique_nested_dicts,
     mock_list_users,
     mock_sync_identity_center_users,
     mock_sync_identity_center_groups,
@@ -647,7 +647,7 @@ def test_synchronize_sync_skip_users_if_false(
         target_groups,
     ]
     mock_preformat_groups.return_value = source_groups
-    mock_get_unique_users_from_groups.return_value = source_users
+    mock_get_unique_nested_dicts.return_value = source_users
     mock_list_users.return_value = target_users
     mock_sync_identity_center_users.return_value = ("users_created", "users_deleted")
     mock_sync_identity_center_groups.return_value = (
@@ -671,7 +671,7 @@ def test_synchronize_sync_skip_users_if_false(
         aws_identity_center_call,
     ]
 
-    assert mock_get_unique_users_from_groups.call_count == 1
+    assert mock_get_unique_nested_dicts.call_count == 1
     assert mock_list_users.call_count == 1
     assert mock_sync_identity_center_users.call_count == 0
     assert mock_sync_identity_center_groups.call_count == 1
@@ -712,11 +712,11 @@ def test_synchronize_sync_skip_users_if_false(
 @patch("modules.aws.identity_center.sync_groups")
 @patch("modules.aws.identity_center.sync_users")
 @patch("modules.aws.identity_center.identity_store.list_users")
-@patch("modules.aws.identity_center.users.get_unique_users_from_groups")
+@patch("modules.aws.identity_center.filters.get_unique_nested_dicts")
 @patch("modules.aws.identity_center.groups.get_groups_with_members_from_integration")
 def test_synchronize_sync_skip_groups_false_if_false(
     mock_get_groups_with_members_from_integration,
-    mock_get_unique_users_from_groups,
+    mock_get_unique_nested_dicts,
     mock_list_users,
     mock_sync_identity_center_users,
     mock_sync_identity_center_groups,
@@ -736,7 +736,7 @@ def test_synchronize_sync_skip_groups_false_if_false(
         target_groups,
     ]
     mock_preformat_groups.return_value = source_groups
-    mock_get_unique_users_from_groups.return_value = source_users
+    mock_get_unique_nested_dicts.return_value = source_users
     mock_list_users.return_value = target_users
     mock_sync_identity_center_users.return_value = ("users_created", "users_deleted")
     mock_sync_identity_center_groups.return_value = (
@@ -757,7 +757,7 @@ def test_synchronize_sync_skip_groups_false_if_false(
         aws_identity_center_call,
     ]
 
-    assert mock_get_unique_users_from_groups.call_count == 1
+    assert mock_get_unique_nested_dicts.call_count == 1
     assert mock_list_users.call_count == 2
     assert mock_sync_identity_center_users.call_count == 1
     assert mock_sync_identity_center_groups.call_count == 0
@@ -796,11 +796,11 @@ def test_synchronize_sync_skip_groups_false_if_false(
 @patch("modules.aws.identity_center.sync_groups")
 @patch("modules.aws.identity_center.sync_users")
 @patch("modules.aws.identity_center.identity_store.list_users")
-@patch("modules.aws.identity_center.users.get_unique_users_from_groups")
+@patch("modules.aws.identity_center.filters.get_unique_nested_dicts")
 @patch("modules.aws.identity_center.groups.get_groups_with_members_from_integration")
 def test_synchronize_sync_skip_users_and_groups_if_false(
     mock_get_groups_with_members_from_integration,
-    mock_get_unique_users_from_groups,
+    mock_get_unique_nested_dicts,
     mock_list_users,
     mock_sync_identity_center_users,
     mock_sync_identity_center_groups,
@@ -818,7 +818,7 @@ def test_synchronize_sync_skip_users_and_groups_if_false(
         source_groups,
         target_groups,
     ]
-    mock_get_unique_users_from_groups.return_value = source_users
+    mock_get_unique_nested_dicts.return_value = source_users
     mock_list_users.return_value = target_users
     mock_sync_identity_center_users.return_value = ("users_created", "users_deleted")
     mock_sync_identity_center_groups.return_value = (
@@ -841,7 +841,7 @@ def test_synchronize_sync_skip_users_and_groups_if_false(
         aws_identity_center_call,
     ]
 
-    assert mock_get_unique_users_from_groups.call_count == 1
+    assert mock_get_unique_nested_dicts.call_count == 1
     assert mock_list_users.call_count == 1
 
     assert mock_sync_identity_center_users.call_count == 0

app/tests/modules/provisioning/test_provisioning_users.py

@@ -2,57 +2,6 @@
 from modules.provisioning import users
 
 
-def test_get_unique_users_from_groups(google_groups_w_users):
-    groups = google_groups_w_users()
-    unique_users = []
-    for group in groups:
-        for user in group["members"]:
-            if user not in unique_users:
-                unique_users.append(user)
-    users_from_groups = users.get_unique_users_from_groups(groups, "members")
-
-    assert sorted(users_from_groups, key=lambda user: user["id"]) == sorted(
-        unique_users, key=lambda user: user["id"]
-    )
-
-
-def test_get_unique_users_from_groups_with_empty_groups():
-    groups = []
-    users_from_groups = users.get_unique_users_from_groups(groups, "members")
-    assert users_from_groups == []
-
-
-def test_get_unique_users_from_dict_group(google_groups_w_users):
-    source_group = google_groups_w_users()[0]
-    users_from_groups = users.get_unique_users_from_groups(source_group, "members")
-    expected_users = source_group["members"]
-    assert sorted(users_from_groups, key=lambda user: user["id"]) == sorted(
-        expected_users, key=lambda user: user["id"]
-    )
-
-
-def test_get_unique_users_from_dict_group_with_duplicate_key():
-    group = {
-        "id": "source_group_id1",
-        "name": "SOURCE-group1",
-        "email": "[email protected]",
-        "members": [
-            {"email": "[email protected]", "id": "user1_id", "username": "user1"},
-            {"email": "[email protected]", "id": "user2_id", "username": "user1"},
-            {"email": "[email protected]", "id": "user3_id", "username": "user2"},
-        ],
-    }
-    users_from_groups = users.get_unique_users_from_groups(group, "members")
-    expected_users = [
-        {"email": "[email protected]", "id": "user1_id", "username": "user1"},
-        {"email": "[email protected]", "id": "user2_id", "username": "user1"},
-        {"email": "[email protected]", "id": "user3_id", "username": "user2"},
-    ]
-    assert sorted(users_from_groups, key=lambda user: user["id"]) == sorted(
-        expected_users, key=lambda user: user["id"]
-    )
-
-
 @patch("modules.provisioning.users.logger")
 def test_provision_users_success(mock_logger):
     users_list = [{"name": "user1"}, {"name": "user2"}, {"name": "user3"}]
@@ -65,7 +14,6 @@ def test_provision_users_success(mock_logger):
     assert mock_function.call_count == len(users_list)
     mock_logger.info.assert_any_call("aws:Starting creation of 3 user(s)")
     for user in users_list:
-        mock_logger.info.assert_any_call(f"user's data:\n{user}")
         mock_logger.info.assert_any_call(
             f"aws:Successful creation of user {user['name']}"
         )
@@ -84,7 +32,6 @@ def test_provision_users_failure(mock_logger):
     assert mock_function.call_count == len(users_list)
     mock_logger.info.assert_any_call("aws:Starting creation of 3 user(s)")
     for i in range(len(users_list)):
-        mock_logger.info.assert_any_call(f"user's data:\n{users_list[i]}")
         if i == 1:
             mock_logger.error.assert_any_call(
                 f"aws:Failed creation user {users_list[i]['name']}"

app/tests/utils/test_filters.py

@@ -181,3 +181,54 @@ def test_compare_list_with_complex_values_sync_mode(google_users, aws_users):
     response = filters.compare_lists(source, target, mode="sync")
 
     assert response == ([], target["values"][3:])
+
+
+def test_get_unique_nested_dicts(google_groups_w_users):
+    groups = google_groups_w_users()
+    unique_users = []
+    for group in groups:
+        for user in group["members"]:
+            if user not in unique_users:
+                unique_users.append(user)
+    users_from_groups = filters.get_unique_nested_dicts(groups, "members")
+
+    assert sorted(users_from_groups, key=lambda user: user["id"]) == sorted(
+        unique_users, key=lambda user: user["id"]
+    )
+
+
+def test_get_unique_nested_dicts_with_empty_source():
+    groups = []
+    users_from_groups = filters.get_unique_nested_dicts(groups, "members")
+    assert users_from_groups == []
+
+
+def test_get_unique_nested_dicts_from_single_dict(google_groups_w_users):
+    source_group = google_groups_w_users()[0]
+    users_from_groups = filters.get_unique_nested_dicts(source_group, "members")
+    expected_users = source_group["members"]
+    assert sorted(users_from_groups, key=lambda user: user["id"]) == sorted(
+        expected_users, key=lambda user: user["id"]
+    )
+
+
+def test_get_unique_nested_dicts_with_duplicate_key():
+    group = {
+        "id": "source_group_id1",
+        "name": "SOURCE-group1",
+        "email": "[email protected]",
+        "members": [
+            {"email": "[email protected]", "id": "user1_id", "username": "user1"},
+            {"email": "[email protected]", "id": "user2_id", "username": "user1"},
+            {"email": "[email protected]", "id": "user3_id", "username": "user2"},
+        ],
+    }
+    users_from_groups = filters.get_unique_nested_dicts(group, "members")
+    expected_users = [
+        {"email": "[email protected]", "id": "user1_id", "username": "user1"},
+        {"email": "[email protected]", "id": "user2_id", "username": "user1"},
+        {"email": "[email protected]", "id": "user3_id", "username": "user2"},
+    ]
+    assert sorted(users_from_groups, key=lambda user: user["id"]) == sorted(
+        expected_users, key=lambda user: user["id"]
+    )

app/utils/filters.py

@@ -106,3 +106,30 @@ def compare_lists(source, target, mode="sync"):
         filtered_target_groups.sort(key=lambda x: x[target_key])
 
         return filtered_source_groups, filtered_target_groups
+
+
+def get_unique_nested_dicts(source_items, nested_key):
+    """Get the unique items from a list located in a dict or from a list of dicts with the same data schema.
+    Considers the whole object for uniqueness, not specific keys.
+
+    Args:
+        source_items (list or dict): A list of dicts or a single dict.
+        nested_key (str): The key to search for nested items.
+
+    Returns:
+        list: A list containing the unique dictionaries found in the nested key.
+    """
+    unique_dicts = {}
+    if isinstance(source_items, list):
+        logger.info(f"Getting unique dictionaries from {len(source_items)} items.")
+        for item in source_items:
+            for nested_dict in get_nested_value(item, nested_key):
+                if nested_dict:
+                    unique_dicts[str(nested_dict)] = nested_dict
+    elif isinstance(source_items, dict):
+        logger.info("Getting unique dictionaries from a single item.")
+        for nested_dict in get_nested_value(source_items, nested_key):
+            if nested_dict:
+                unique_dicts[str(nested_dict)] = nested_dict
+    logger.info(f"Found {len(unique_dicts)} unique dictionaries.")
+    return list(unique_dicts.values())