Feat/provisioning module (#486)

* fix: Refactor AWS group handling in conftest.py

* feat: Add functions to retrieve groups with members from integration sources

* feat: Add user synchronization functionality

* chore: run fmt

* feat: Add filter functions

* feat: add group function to map users

* feat: hoist users and groups matching functions in single function

* fix: update var names to be generic

* Refactor variable names and update group and user functions for readability

* fix: expected output to match updated fixtures

* fix: set expected value for group emails in google groups

* fix: Add tests for compare_lists function in test_filters.py

* chore: remove unused functions

* chore: run lint

* fix: Refactor function to use a dictionary for storing unique users

* fix: remove unused code

* fix: remove unused code

* fix: lint

app/integrations/google_workspace/google_directory.py

@@ -73,7 +73,7 @@ def list_groups(
     customer=None,
     **kwargs,
 ):
-    """List all groups in the Google Workspace domain.
+    """List all groups in the Google Workspace domain. A query can be provided to filter the results (e.g. query="email:prefix-*" will filter for all groups where the email starts with 'prefix-').
 
     Returns:
         list: A list of group objects.
@@ -126,3 +126,44 @@ def list_group_members(group_key, delegated_user_email=None):
         groupKey=group_key,
         maxResults=200,
     )
+
+
+def add_users_to_group(group, group_key):
+    """Add users to a group in the Google Workspace domain.
+
+    Args:
+        group_key (str): The group's email address or unique group ID.
+
+    Returns:
+        list: A list of user objects.
+
+    Ref: https://developers.google.com/admin-sdk/directory/reference/rest/v1/members/insert
+    """
+    result = list_group_members(group_key)
+    if result:
+        group["members"] = result
+    return group
+
+
+def list_groups_with_members(**kwargs):
+    """List all groups in the Google Workspace domain with their members.
+
+    Returns:
+        list: A list of group objects with members.
+    """
+    members_details = kwargs.get("members_details", True)
+    kwargs.pop("members_details", None)
+    groups = list_groups(**kwargs)
+    if not groups:
+        return []
+    for group in range(len(groups)):
+        members = list_group_members(groups[group]["email"])
+        if members and members_details:
+            groups[group]["members"] = members
+
+            for member in range(len(groups[group]["members"])):
+                groups[group]["members"][member] = get_user(
+                    groups[group]["members"][member]["email"]
+                )
+
+    return groups

app/modules/provisioning/groups.py

@@ -0,0 +1,42 @@
+from integrations.google_workspace import google_directory
+from integrations.aws import identity_store
+from utils import filters as filter_tools
+
+
+def get_groups_with_members_from_integration(integration_source, **kwargs):
+    """Retrieve the users from an integration group source.
+    Supported sources are:
+    - Google Groups
+    - AWS Identity Center (Identity Store)
+
+    Args:
+        integration_source (str): The source of the groups.
+        **kwargs: Additional keyword arguments. Supported arguments are:
+
+            - `filters` (list): List of filters to apply to the groups.
+            - `query` (str): The query to search for groups.
+            - `members_details` (bool): Include the members details in the groups.
+
+    Returns:
+        list: A list of groups with members, empty list if no groups are found.
+    """
+    filters = kwargs.get("filters", [])
+    query = kwargs.get("query", None)
+    members_details = kwargs.get("members_details", True)
+
+    groups = []
+    match integration_source:
+        case "google_groups":
+            groups = google_directory.list_groups_with_members(
+                query=query, members_details=members_details
+            )
+        case "aws_identity_center":
+            groups = identity_store.list_groups_with_memberships(
+                members_details=members_details
+            )
+        case _:
+            return groups
+
+    for filter in filters:
+        groups = filter_tools.filter_by_condition(groups, filter)
+    return groups

app/modules/provisioning/users.py

@@ -0,0 +1,31 @@
+from logging import getLogger
+
+from utils import filters as filter_tools
+
+
+logger = getLogger(__name__)
+
+
+def get_unique_users_from_groups(groups, key):
+    """Get the unique users from a list of groups with the same data schema or a single group dict.
+    Considers the whole object for uniqueness, not specific keys.
+
+    Args:
+        groups (list or dict): A list of groups or a single group.
+        key (str): The key to get the users from the groups.
+
+    Returns:
+        list: A list of unique users from the groups
+    """
+    users_dict = {}
+    if isinstance(groups, list):
+        for group in groups:
+            for user in filter_tools.get_nested_value(group, key):
+                if user:
+                    users_dict[str(user)] = user
+    elif isinstance(groups, dict):
+        for user in filter_tools.get_nested_value(groups, key):
+            if user:
+                users_dict[str(user)] = user
+
+    return list(users_dict.values())

app/tests/conftest.py

@@ -10,9 +10,9 @@ def google_groups():
     def _google_groups(n=3, prefix="", domain="test.com"):
         return [
             {
-                "id": f"{prefix}_google_group_id{i+1}",
-                "name": f"AWS-group{i+1}",
-                "email": f"{prefix}_aws-group{i+1}@{domain}",
+                "id": f"{prefix}google_group_id{i+1}",
+                "name": f"{prefix}group-name{i+1}",
+                "email": f"{prefix}group-name{i+1}@{domain}",
             }
             for i in range(n)
         ]
@@ -26,21 +26,21 @@ def _google_users(n=3, prefix="", domain="test.com"):
         users = []
         for i in range(n):
             user = {
-                "id": f"{prefix}_id_{i}",
-                "primaryEmail": f"{prefix}_email_{i}@{domain}",
+                "id": f"{prefix}user_id{i+1}",
+                "primaryEmail": f"{prefix}user-email{i+1}@{domain}",
                 "emails": [
                     {
-                        "address": f"{prefix}_email_{i}@{domain}",
+                        "address": f"{prefix}user-email{i+1}@{domain}",
                         "primary": True,
                         "type": "work",
                     }
                 ],
                 "suspended": False,
                 "name": {
-                    "fullName": f"Given_name_{i} Family_name_{i}",
-                    "familyName": f"Family_name_{i}",
-                    "givenName": f"Given_name_{i}",
-                    "displayName": f"Given_name_{i} Family_name_{i}",
+                    "fullName": f"Given_name_{i+1} Family_name_{i+1}",
+                    "familyName": f"Family_name_{i+1}",
+                    "givenName": f"Given_name_{i+1}",
+                    "displayName": f"Given_name_{i+1} Family_name_{i+1}",
                 },
             }
             users.append(user)
@@ -90,16 +90,16 @@ def _aws_users(n=3, prefix="", domain="test.com", store_id="d-123412341234"):
         users = []
         for i in range(n):
             user = {
-                "UserName": f"{prefix}_email_{i}@{domain}",
-                "UserId": f"{prefix}_id_{i}",
+                "UserName": f"{prefix}user-email{i+1}@{domain}",
+                "UserId": f"{prefix}user_id{i+1}",
                 "Name": {
-                    "FamilyName": f"Family_name_{i}",
-                    "GivenName": f"Given_name_{i}",
+                    "FamilyName": f"Family_name_{i+1}",
+                    "GivenName": f"Given_name_{i+1}",
                 },
-                "DisplayName": f"Given_name_{i} Family_name_{i}",
+                "DisplayName": f"Given_name_{i+1} Family_name_{i+1}",
                 "Emails": [
                     {
-                        "Value": f"{prefix}_email_{i}@{domain}",
+                        "Value": f"{prefix}user-email{i+1}@{domain}",
                         "Type": "work",
                         "Primary": True,
                     }
@@ -118,8 +118,8 @@ def _aws_groups(n=3, prefix="", store_id="d-123412341234"):
         return {
             "Groups": [
                 {
-                    "GroupId": f"{prefix}_aws-group_id{i+1}",
-                    "DisplayName": f"AWS-group{i+1}",
+                    "GroupId": f"{prefix}aws-group_id{i+1}",
+                    "DisplayName": f"{prefix}group-name{i+1}",
                     "Description": f"A group to test resolving AWS-group{i+1} memberships",
                     "IdentityStoreId": f"{store_id}",
                 }
@@ -137,10 +137,10 @@ def _aws_groups_memberships(n=3, prefix="", store_id="d-123412341234"):
             "GroupMemberships": [
                 {
                     "IdentityStoreId": f"{store_id}",
-                    "MembershipId": f"{prefix}_membership_id_{i+1}",
-                    "GroupId": f"{prefix}_aws-group_id{i+1}",
+                    "MembershipId": f"{prefix}membership_id_{i+1}",
+                    "GroupId": f"{prefix}aws-group_id{i+1}",
                     "MemberId": {
-                        "UserId": f"{prefix}_id_{i}",
+                        "UserId": f"{prefix}user_id{i+1}",
                     },
                 }
                 for i in range(n)
@@ -155,9 +155,9 @@ def aws_groups_w_users(aws_groups, aws_users, aws_groups_memberships):
     def _aws_groups_w_users(
         n_groups=1, n_users=3, prefix="", domain="test.com", store_id="d-123412341234"
     ):
-        groups = aws_groups(n_groups, prefix, domain, store_id)["Groups"]
+        groups = aws_groups(n_groups, prefix, store_id)["Groups"]
         users = aws_users(n_users, prefix, domain, store_id)
-        memberships = aws_groups_memberships(n_groups, prefix, domain, store_id)[
+        memberships = aws_groups_memberships(n_groups, prefix, store_id)[
             "GroupMemberships"
         ]
         for group, membership in zip(groups, memberships):

app/tests/integrations/aws/test_identity_store.py

@@ -156,16 +156,16 @@ def test_describe_user(
     user_id = "test_user_id1"
 
     expected = {
-        "UserName": "_email_0@test.com",
-        "UserId": "_id_0",
+        "UserName": "user-email1@test.com",
+        "UserId": "user_id1",
         "Name": {
-            "FamilyName": "Family_name_0",
-            "GivenName": "Given_name_0",
+            "FamilyName": "Family_name_1",
+            "GivenName": "Given_name_1",
         },
-        "DisplayName": "Given_name_0 Family_name_0",
+        "DisplayName": "Given_name_1 Family_name_1",
         "Emails": [
             {
-                "Value": "_email_0@test.com",
+                "Value": "user-email1@test.com",
                 "Type": "work",
                 "Primary": True,
             }
@@ -683,37 +683,37 @@ def test_list_groups_with_memberships(
 ):
     # groups = aws_groups_w_users(2, 3, prefix="test", domain="test.com")
     groups = aws_groups(2, prefix="test")["Groups"]
-    memberships = [[], aws_groups_memberships(2, prefix="test")["GroupMemberships"]]
-    users = aws_users(2, prefix="test", domain="test.com")
+    memberships = [[], aws_groups_memberships(2, prefix="test-")["GroupMemberships"]]
+    users = aws_users(2, prefix="test-", domain="test.com")
     expected_output = [
         {
-            "IdentityStoreId": "d-123412341234",
-            "GroupId": "test_aws-group_id1",
-            "DisplayName": "AWS-group1",
+            "GroupId": "testaws-group_id1",
+            "DisplayName": "testgroup-name1",
             "Description": "A group to test resolving AWS-group1 memberships",
+            "IdentityStoreId": "d-123412341234",
             "GroupMemberships": [],
         },
         {
-            "IdentityStoreId": "d-123412341234",
-            "GroupId": "test_aws-group_id2",
-            "DisplayName": "AWS-group2",
+            "GroupId": "testaws-group_id2",
+            "DisplayName": "testgroup-name2",
             "Description": "A group to test resolving AWS-group2 memberships",
+            "IdentityStoreId": "d-123412341234",
             "GroupMemberships": [
                 {
                     "IdentityStoreId": "d-123412341234",
-                    "MembershipId": "test_membership_id_1",
-                    "GroupId": "test_aws-group_id1",
+                    "MembershipId": "test-membership_id_1",
+                    "GroupId": "test-aws-group_id1",
                     "MemberId": {
-                        "UserName": "test_email_0@test.com",
-                        "UserId": "test_id_0",
+                        "UserName": "test-user-email1@test.com",
+                        "UserId": "test-user_id1",
                         "Name": {
-                            "FamilyName": "Family_name_0",
-                            "GivenName": "Given_name_0",
+                            "FamilyName": "Family_name_1",
+                            "GivenName": "Given_name_1",
                         },
-                        "DisplayName": "Given_name_0 Family_name_0",
+                        "DisplayName": "Given_name_1 Family_name_1",
                         "Emails": [
                             {
-                                "Value": "test_email_0@test.com",
+                                "Value": "test-user-email1@test.com",
                                 "Type": "work",
                                 "Primary": True,
                             }
@@ -723,19 +723,19 @@ def test_list_groups_with_memberships(
                 },
                 {
                     "IdentityStoreId": "d-123412341234",
-                    "MembershipId": "test_membership_id_2",
-                    "GroupId": "test_aws-group_id2",
+                    "MembershipId": "test-membership_id_2",
+                    "GroupId": "test-aws-group_id2",
                     "MemberId": {
-                        "UserName": "test_email_1@test.com",
-                        "UserId": "test_id_1",
+                        "UserName": "test-user-email2@test.com",
+                        "UserId": "test-user_id2",
                         "Name": {
-                            "FamilyName": "Family_name_1",
-                            "GivenName": "Given_name_1",
+                            "FamilyName": "Family_name_2",
+                            "GivenName": "Given_name_2",
                         },
-                        "DisplayName": "Given_name_1 Family_name_1",
+                        "DisplayName": "Given_name_2 Family_name_2",
                         "Emails": [
                             {
-                                "Value": "test_email_1@test.com",
+                                "Value": "test-user-email2@test.com",
                                 "Type": "work",
                                 "Primary": True,
                             }
@@ -757,5 +757,5 @@ def test_list_groups_with_memberships(
     mock_describe_user.side_effect = user_side_effect
 
     result = identity_store.list_groups_with_memberships()
-
+    # print(json.dumps(result, indent=4))
     assert result == expected_output