Add pagination limit/offset in Query

- Track/Check graphql schema changes
- Check migration changes
- Refactor pagination components
- Upgrade packages
    - Major upgrade for strawberry
    - Minor upgrade for other packages

.github/workflows/actions.yml

@@ -12,29 +12,34 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install dependencies
       working-directory: ./mapswipe_workers
       run: |
         python -m pip install --upgrade pip
         pip install flake8 black==22.3.0 isort
+
     - name: Code style
       working-directory: ./mapswipe_workers
       run: |
         black --check mapswipe_workers ../django
         flake8 --count --config setup.cfg mapswipe_workers/ ../django/
         isort --check --settings-file setup.cfg mapswipe_workers/ ../django/
+
     - name: Assert check
       run: |
           cmp --silent ./postgres/initdb.sql ./mapswipe_workers/tests/integration/set_up_db.sql || {
             echo 'The set_up_db.sql is not same as initdb.sql. Please sync this files and push';
             diff ./postgres/initdb.sql ./mapswipe_workers/tests/integration/set_up_db.sql;
             exit 1;
           }
-    - name:  Setup Postgres Database Container
+
+    - name: Setup Postgres Database Container
       env:
         POSTGRES_PASSWORD: postgres
         POSTGRES_USER: postgres
@@ -44,12 +49,14 @@ jobs:
         touch postgres/serviceAccountKey.json
         docker-compose up --build --detach postgres
         for i in {1..5}; do docker-compose exec -T postgres pg_isready && s=0 && break || s=$? && sleep 5; done; (docker-compose logs postgres && exit $s)
+
     - name: Deploy Firebase Rules and Functions
       env:
         FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
         FIREBASE_DB: ${{ secrets.FIREBASE_DB }}
       run: |
         docker-compose run --rm firebase_deploy sh -c "firebase use $FIREBASE_DB && firebase deploy --token $FIREBASE_TOKEN --only database"
+
     - name: Decrypt Service Account Key File
       working-directory: ./
       run: |
@@ -58,6 +65,7 @@ jobs:
         OPENSSL_PASSPHRASE: ${{ secrets.OPENSSL_PASSPHRASE }}
         OPENSSL_KEY: ${{ secrets.OPENSSL_KEY }}
         OPENSSL_IV: ${{ secrets.OPENSSL_IV }}
+
     - name: Run Tests
       working-directory: ./mapswipe_workers
       env:
@@ -73,3 +81,27 @@ jobs:
         docker-compose run --rm mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/
         docker-compose run --rm mapswipe_workers_creation bash -c 'pip install pytest && pytest -ra -v --durations=10 tests/integration/'
         docker-compose run --rm django pytest -ra -v --durations=10
+
+    - name: Django Graphql Schema Check
+      env:
+        SOURCE_SCHEMA: './django/schema.graphql'
+        LATEST_SCHEMA: './django-data/schema-latest.graphql'
+      run: |
+        docker-compose run --rm django bash -c 'wait-for-it postgres:5432 && ./manage.py graphql_schema --out /django-data/schema-latest.graphql' &&
+        cmp --silent $SOURCE_SCHEMA $LATEST_SCHEMA || {
+          echo 'The schema.graphql is not up to date with the latest changes. Please update and push latest';
+          diff $SOURCE_SCHEMA $LATEST_SCHEMA;
+          exit 1;
+        }
+
+    - name: Django Database Migration Check
+      env:
+        POSTGRES_PASSWORD: postgres
+        POSTGRES_USER: postgres
+        POSTGRES_DB: postgres
+        DJANGO_SECRET_KEY: test-django-secret-key
+      run: |
+        docker-compose run --rm django bash -c 'wait-for-it postgres:5432 && ./manage.py makemigrations --check --dry-run' || {
+          echo 'There are some changes to be reflected in the migration. Make sure to run makemigrations';
+          exit 1;
+        }

django/apps/existing_database/factories.py

@@ -73,7 +73,7 @@ def _create(cls, model_class, *args, **kwargs):
 
 
 class UserGroupFactory(DjangoModelFactory):
-    user_group_id = factory.Sequence(lambda n: f"dummy-user-group-id-{n}")
+    user_group_id = factory.Sequence(lambda n: f"dummy-user-group-id-{n:02d}")
     name = factory.Sequence(lambda n: f"UserGroup-{n}")
     description = factory.Faker("sentence", nb_words=20)
     archived_by_id = factory.SubFactory(UserFactory)

django/apps/existing_database/management/commands/graphql_schema.py

@@ -0,0 +1,22 @@
+import argparse
+
+from django.core.management.base import BaseCommand
+from mapswipe.graphql import schema
+from strawberry.printer import print_schema
+
+
+class Command(BaseCommand):
+    help = "Create schema.graphql file"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--out",
+            type=argparse.FileType("w"),
+            default="schema.graphql",
+        )
+
+    def handle(self, *args, **options):
+        file = options["out"]
+        file.write(print_schema(schema))
+        file.close()
+        self.stdout.write(self.style.SUCCESS(f"{file.name} file generated"))

django/apps/existing_database/query.py

@@ -9,8 +9,9 @@
 from django.db import models
 from django.utils import timezone
 from django_cte import With
-from mapswipe.paginations import CountList, StrawberryDjangoCountList
+from mapswipe.paginations import CountList, pagination_field
 from mapswipe.types import AreaSqKm
+from strawberry.types import Info
 
 from .filters import ProjectFilter, UserFilter, UserGroupFilter
 from .models import Project
@@ -225,22 +226,26 @@ class Query:
         description=get_community_stats_latest.__doc__,
     )
 
-    projects: CountList[ProjectType] = StrawberryDjangoCountList(
+    projects: CountList[ProjectType] = pagination_field(
         pagination=True,
         filters=ProjectFilter,
     )
 
-    users: CountList[UserType] = StrawberryDjangoCountList(
-        pagination=True, filters=UserFilter
+    users: CountList[UserType] = pagination_field(
+        pagination=True,
+        filters=UserFilter,
     )
-    user: UserType = strawberry_django.field()
+    user: UserType | None = strawberry_django.field()
 
-    user_groups: CountList[UserGroupType] = StrawberryDjangoCountList(
+    user_groups: CountList[UserGroupType] = pagination_field(
         pagination=True,
         filters=UserGroupFilter,
         order=UserGroupOrder,
     )
-    user_group: UserGroupType = strawberry_django.field()
+
+    @strawberry_django.field
+    async def user_group(self, info: Info, pk: strawberry.ID) -> UserGroupType | None:
+        return await UserGroupType.get_queryset(None, None, info).filter(pk=pk).afirst()
 
     @strawberry.field
     async def filtered_stats(

django/apps/existing_database/test_queries.py

@@ -15,7 +15,7 @@
     UserGroupFactory,
     UserGroupMembershipFactory,
 )
-from apps.existing_database.models import Project
+from apps.existing_database.models import Project, UserGroup
 from django.utils import timezone
 from mapswipe.tests import TestCase
 
@@ -160,6 +160,8 @@ def test_user_group_query(self):
                 isArchived
                 userMemberships(pagination: $pagination) {
                   count
+                  offset
+                  limit
                   items {
                     userId
                     username
@@ -245,12 +247,63 @@ def test_user_group_query(self):
                         "name": user_group.name,
                         "userMemberships": {
                             "count": 3,
+                            "offset": offset,
+                            "limit": 2,
                             "items": expected_memberships,
                         },
                     },
                 },
             }
 
+    def test_user_groups_query(self):
+        query = """
+            query MyQuery($pagination: OffsetPaginationInput!) {
+              userGroups(pagination: $pagination, order: {userGroupId: ASC}) {
+                count
+                offset
+                limit
+                items {
+                    name
+                    createdAt
+                    archivedAt
+                    isArchived
+                  }
+              }
+            }
+        """
+        existing_user_groups_count = UserGroup.objects.count()
+        # UserGroup with None name should not be filtered out
+        UserGroupFactory.create_batch(3, name=None)
+
+        offset = 0
+        resp = self.query_check(
+            query,
+            variables=dict(
+                pagination=dict(
+                    limit=2,
+                    offset=offset,
+                ),
+            ),
+        )
+        assert resp == {
+            "data": {
+                "userGroups": {
+                    "count": existing_user_groups_count,
+                    "limit": 2,
+                    "offset": offset,
+                    "items": [
+                        {
+                            "archivedAt": user_group.archived_at,
+                            "createdAt": user_group.created_at,
+                            "isArchived": user_group.is_archived,
+                            "name": user_group.name,
+                        }
+                        for user_group in self.user_groups[:2]
+                    ],
+                },
+            },
+        }
+
     def test_user_query(self):
         # TODO:
         query = """
@@ -260,6 +313,8 @@ def test_user_query(self):
                 username
                 userInUserGroups(pagination: $pagination) {
                   count
+                  offset
+                  limit
                   items {
                     userGroupId
                     userGroupName
@@ -280,7 +335,7 @@ def test_user_query(self):
                     user_group=user_group,
                     user=user,
                 )
-            # Additinal users
+            # Additional users
             for additional_user in additional_users[:index]:
                 UserGroupMembershipFactory.create(
                     user_group=user_group, user=additional_user
@@ -340,6 +395,8 @@ def test_user_query(self):
                         "username": user.username,
                         "userInUserGroups": {
                             "count": 3,
+                            "offset": offset,
+                            "limit": 2,
                             "items": expected_memberships,
                         },
                     },

django/apps/existing_database/types.py

@@ -9,6 +9,7 @@
 from django.utils import timezone
 from mapswipe.paginations import CountList, apply_pagination
 from mapswipe.types import AreaSqKm, GenericJSON, TimeInSeconds
+from mapswipe.utils import get_queryset_for_model
 from strawberry.types import Info
 
 from .enums import ProjectTypeEnum
@@ -469,6 +470,7 @@ async def user_in_user_groups(
                 ).values("user_group_id")
             )
             .annotate(
+                user_group_name=models.F("name"),
                 members_count=models.functions.Coalesce(
                     models.Subquery(
                         UserGroupUserMembership.objects.filter(
@@ -485,19 +487,13 @@ async def user_in_user_groups(
             )
             .order_by("user_group_id")
         )
+
         paginated_qs = apply_pagination(pagination, qs)
         return CountList[UserUserGroupMembershipType](
-            node=dict(
-                count_callback=lambda: qs.acount(),
-                queryset=[
-                    UserUserGroupMembershipType(**data)
-                    async for data in paginated_qs.values(
-                        "user_group_id",
-                        "members_count",
-                        user_group_name=models.F("name"),
-                    )
-                ],
-            )
+            limit=pagination.limit,
+            offset=pagination.offset,
+            get_count=lambda: qs.acount(),
+            queryset=paginated_qs,
         )
 
 
@@ -576,25 +572,28 @@ def _subquery_generator(agg_func):
             )
             .order_by("user_id")
         )
+
         paginated_qs = apply_pagination(pagination, qs)
         return CountList[UserGroupUserMembershipType](
-            node=dict(
-                count_callback=lambda: qs.acount(),
-                queryset=[
-                    UserGroupUserMembershipType(**data)
-                    async for data in paginated_qs.values(
-                        "user_id",
-                        "is_active",
-                        # Annotate fields
-                        "username",
-                        "total_mapping_projects",
-                        "total_swipes",
-                        "total_swipe_time",
-                    )
-                ],
-            )
+            limit=pagination.limit,
+            offset=pagination.offset,
+            get_count=lambda: qs.acount(),
+            queryset=[
+                UserGroupUserMembershipType(**item)
+                async for item in paginated_qs.values(
+                    # NOTE: Defining manual select fields since DB doesn't have field id but Django assumes it has
+                    "user_id",
+                    "is_active",
+                    # Annotate fields
+                    "username",
+                    "total_mapping_projects",
+                    "total_swipes",
+                    "total_swipe_time",
+                )
+            ],
         )
 
-    def get_queryset(self, queryset, info, **kwargs):
+    @staticmethod
+    def get_queryset(_, queryset: models.QuerySet | None, info: Info):
         # Filter out user group without name. They aren't sync yet.
-        return UserGroup.objects.exclude(name__isnull=True).all()
+        return get_queryset_for_model(UserGroup, queryset).exclude(name__isnull=True)