From ad107aa40972a50fc01e914ae486fcdbbad4aab4 Mon Sep 17 00:00:00 2001 From: Bradley Oesch Date: Thu, 16 May 2024 16:22:27 -0500 Subject: [PATCH 01/11] Support async queryset evaluation --- cursor_pagination.py | 43 +++++++++++++++++++++++++++++++++---------- tests/tests.py | 22 ++++++++++++++++++++++ 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/cursor_pagination.py b/cursor_pagination.py index fa29638..3f61e79 100644 --- a/cursor_pagination.py +++ b/cursor_pagination.py @@ -1,3 +1,4 @@ +from asgiref.sync import sync_to_async from base64 import b64decode, b64encode from collections.abc import Sequence @@ -68,17 +69,17 @@ def _nulls_ordering(self, ordering, from_last=False): return nulls_ordering - - - def page(self, first=None, last=None, after=None, before=None): - qs = self.queryset + def _apply_paginator_arguments(self, qs, first=None, last=None, after=None, before=None): + """ + Apply first/after, last/before filtering to the queryset + """ page_size = first or last if page_size is None: return CursorPage(qs, self) from_last = last is not None if from_last and first is not None: - raise ValueError('Cannot process first and last') + raise ValueError('Cannot process first and last') if after is not None: qs = self.apply_cursor(after, qs, from_last=from_last) @@ -89,11 +90,10 @@ def page(self, first=None, last=None, after=None, before=None): if last is not None: qs = qs.order_by(*self._nulls_ordering(reverse_ordering(self.ordering), from_last=True))[:last + 1] - qs = list(qs) - items = qs[:page_size] - if last is not None: - items.reverse() - has_additional = len(qs) > len(items) + def _get_cursor_page(self, items, has_additional, first, last, after, before): + """ + Create and return the cursor page for the given items + """ additional_kwargs = {} if first is not None: additional_kwargs['has_next'] = has_additional @@ -103,6 +103,29 @@ def page(self, first=None, last=None, after=None, before=None): additional_kwargs['has_next'] = bool(before) return CursorPage(items, self, **additional_kwargs) + def page(self, first=None, last=None, after=None, before=None): + qs = self.queryset + self._apply_paginator_arguments(qs, first, last, after, before) + + qs = list(qs) + items = qs[:page_size] + if last is not None: + items.reverse() + has_additional = len(qs) > len(items) + + return self._get_cursor_page(items, has_additional, first, last, after, before) + + async def apage(self, first=None, last=None, after=None, before=None): + qs = self.queryset + self._apply_paginator_arguments(qs, first, last, after, before) + + items = await sync_to_async(list)(qs[:page_size]) + if last is not None: + items.reverse() + has_additional = (await qs.acount()) > len(items) + + return self._get_cursor_page(items, has_additional, first, last, after, before) + def apply_cursor(self, cursor, queryset, from_last, reverse=False): position = self.decode_cursor(cursor) diff --git a/tests/tests.py b/tests/tests.py index 238dbd5..4bd4912 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -18,6 +18,13 @@ def test_empty(self): self.assertFalse(page.has_next) self.assertFalse(page.has_previous) + async def test_async_empty(self): + paginator = CursorPaginator(Post.objects.all(), ('id',)) + page = await paginator.apage() + self.assertEqual(len(page), 0) + self.assertFalse(page.has_next) + self.assertFalse(page.has_previous) + def test_with_items(self): for i in range(20): Post.objects.create(name='Name %s' % i) @@ -27,6 +34,15 @@ def test_with_items(self): self.assertFalse(page.has_next) self.assertFalse(page.has_previous) + async def test_async_with_items(self): + for i in range(20): + await Post.objects.acreate(name='Name %s' % i) + paginator = CursorPaginator(Post.objects.all(), ('id',)) + page = await paginator.apage() + self.assertEqual(len(page), 20) + self.assertFalse(page.has_next) + self.assertFalse(page.has_previous) + class TestForwardPagination(TestCase): @@ -45,6 +61,12 @@ def test_first_page(self): self.assertTrue(page.has_next) self.assertFalse(page.has_previous) + async def test_async_first_page(self): + page = await self.paginator.apage(first=2) + self.assertSequenceEqual(page, [self.items[0], self.items[1]]) + self.assertTrue(page.has_next) + self.assertFalse(page.has_previous) + def test_second_page(self): previous_page = self.paginator.page(first=2) cursor = self.paginator.cursor(previous_page[-1]) From d8e171675a688efe9099a8fcbbc7a607a48695e2 Mon Sep 17 00:00:00 2001 From: Bradley Oesch Date: Thu, 16 May 2024 16:33:06 -0500 Subject: [PATCH 02/11] Add async example in README --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 3b8499e..834d2a5 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,19 @@ def posts_api(request, after=None): 'last_cursor': paginator.cursor(page[-1]) } return data + + +async def posts_api_async(request, after=None): + qs = Post.objects.all() + page_size = 10 + paginator = CursorPaginator(qs, ordering=('-created', '-id')) + page = await paginator.apage(first=page_size, after=after) + data = { + 'objects': [serialize_page(p) for p in page], + 'has_next_page': page.has_next, + 'last_cursor': paginator.cursor(page[-1]) + } + return data ``` Reverse pagination can be achieved by using the `last` and `before` arguments From 7f77477b4eb74ce7e4819b29d958d4210f2c3750 Mon Sep 17 00:00:00 2001 From: Bradley Oesch Date: Mon, 20 May 2024 16:50:07 -0500 Subject: [PATCH 03/11] Fix missing page_size variable bug --- cursor_pagination.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cursor_pagination.py b/cursor_pagination.py index 3f61e79..8403fc9 100644 --- a/cursor_pagination.py +++ b/cursor_pagination.py @@ -108,6 +108,7 @@ def page(self, first=None, last=None, after=None, before=None): self._apply_paginator_arguments(qs, first, last, after, before) qs = list(qs) + page_size = first or last items = qs[:page_size] if last is not None: items.reverse() @@ -119,6 +120,7 @@ async def apage(self, first=None, last=None, after=None, before=None): qs = self.queryset self._apply_paginator_arguments(qs, first, last, after, before) + page_size = first or last items = await sync_to_async(list)(qs[:page_size]) if last is not None: items.reverse() From 4e3e45c8db4bf1e8b0a87671041bca33de80ae78 Mon Sep 17 00:00:00 2001 From: Bradley Oesch Date: Mon, 20 May 2024 16:56:14 -0500 Subject: [PATCH 04/11] Use async for in favor of sync_to_async --- cursor_pagination.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cursor_pagination.py b/cursor_pagination.py index 8403fc9..8074925 100644 --- a/cursor_pagination.py +++ b/cursor_pagination.py @@ -1,4 +1,3 @@ -from asgiref.sync import sync_to_async from base64 import b64decode, b64encode from collections.abc import Sequence @@ -121,7 +120,9 @@ async def apage(self, first=None, last=None, after=None, before=None): self._apply_paginator_arguments(qs, first, last, after, before) page_size = first or last - items = await sync_to_async(list)(qs[:page_size]) + items = [] + async for item in qs[:page_size]: + items.append(item) if last is not None: items.reverse() has_additional = (await qs.acount()) > len(items) From 28f6afa7343c3a0f2664e0d07947938c69db5eb1 Mon Sep 17 00:00:00 2001 From: Bradley Oesch Date: Mon, 20 May 2024 16:59:56 -0500 Subject: [PATCH 05/11] Add some more basic async tests --- tests/tests.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 4bd4912..a769dd1 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -75,6 +75,14 @@ def test_second_page(self): self.assertTrue(page.has_next) self.assertTrue(page.has_previous) + async def test_async_second_page(self): + previous_page = await self.paginator.apage(first=2) + cursor = self.paginator.cursor(previous_page[-1]) + page = await self.paginator.apage(first=2, after=cursor) + self.assertSequenceEqual(page, [self.items[2], self.items[3]]) + self.assertTrue(page.has_next) + self.assertTrue(page.has_previous) + def test_last_page(self): previous_page = self.paginator.page(first=18) cursor = self.paginator.cursor(previous_page[-1]) @@ -83,6 +91,14 @@ def test_last_page(self): self.assertFalse(page.has_next) self.assertTrue(page.has_previous) + async def test_async_last_page(self): + previous_page = await self.paginator.apage(first=18) + cursor = self.paginator.cursor(previous_page[-1]) + page = await self.paginator.apage(first=2, after=cursor) + self.assertSequenceEqual(page, [self.items[18], self.items[19]]) + self.assertFalse(page.has_next) + self.assertTrue(page.has_previous) + def test_incomplete_last_page(self): previous_page = self.paginator.page(first=18) cursor = self.paginator.cursor(previous_page[-1]) @@ -91,6 +107,14 @@ def test_incomplete_last_page(self): self.assertFalse(page.has_next) self.assertTrue(page.has_previous) + async def test_async_incomplete_last_page(self): + previous_page = await self.paginator.apage(first=18) + cursor = self.paginator.cursor(previous_page[-1]) + page = await self.paginator.apage(first=100, after=cursor) + self.assertSequenceEqual(page, [self.items[18], self.items[19]]) + self.assertFalse(page.has_next) + self.assertTrue(page.has_previous) + class TestBackwardsPagination(TestCase): @@ -109,6 +133,12 @@ def test_first_page(self): self.assertTrue(page.has_previous) self.assertFalse(page.has_next) + async def test_async_first_page(self): + page = await self.paginator.apage(last=2) + self.assertSequenceEqual(page, [self.items[18], self.items[19]]) + self.assertTrue(page.has_previous) + self.assertFalse(page.has_next) + def test_second_page(self): previous_page = self.paginator.page(last=2) cursor = self.paginator.cursor(previous_page[0]) @@ -117,6 +147,14 @@ def test_second_page(self): self.assertTrue(page.has_previous) self.assertTrue(page.has_next) + async def test_async_second_page(self): + previous_page = await self.paginator.apage(last=2) + cursor = self.paginator.cursor(previous_page[0]) + page = await self.paginator.apage(last=2, before=cursor) + self.assertSequenceEqual(page, [self.items[16], self.items[17]]) + self.assertTrue(page.has_previous) + self.assertTrue(page.has_next) + def test_last_page(self): previous_page = self.paginator.page(last=18) cursor = self.paginator.cursor(previous_page[0]) @@ -125,6 +163,14 @@ def test_last_page(self): self.assertFalse(page.has_previous) self.assertTrue(page.has_next) + async def test_async_last_page(self): + previous_page = await self.paginator.apage(last=18) + cursor = self.paginator.cursor(previous_page[0]) + page = await self.paginator.apage(last=2, before=cursor) + self.assertSequenceEqual(page, [self.items[0], self.items[1]]) + self.assertFalse(page.has_previous) + self.assertTrue(page.has_next) + def test_incomplete_last_page(self): previous_page = self.paginator.page(last=18) cursor = self.paginator.cursor(previous_page[0]) @@ -133,6 +179,14 @@ def test_incomplete_last_page(self): self.assertFalse(page.has_previous) self.assertTrue(page.has_next) + async def test_async_incomplete_last_page(self): + previous_page = await self.paginator.apage(last=18) + cursor = self.paginator.cursor(previous_page[0]) + page = await self.paginator.apage(last=100, before=cursor) + self.assertSequenceEqual(page, [self.items[0], self.items[1]]) + self.assertFalse(page.has_previous) + self.assertTrue(page.has_next) + class TestTwoFieldPagination(TestCase): From 0131668adfd0f7b9215521eea246b3e80941335c Mon Sep 17 00:00:00 2001 From: Bradley Oesch Date: Wed, 12 Jun 2024 11:43:11 -0500 Subject: [PATCH 06/11] Be explicit about aiterator() --- cursor_pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cursor_pagination.py b/cursor_pagination.py index 8074925..dbdd417 100644 --- a/cursor_pagination.py +++ b/cursor_pagination.py @@ -121,7 +121,7 @@ async def apage(self, first=None, last=None, after=None, before=None): page_size = first or last items = [] - async for item in qs[:page_size]: + async for item in qs[:page_size].aiterator(): items.append(item) if last is not None: items.reverse() From 9a87cd018fc180d5b4c0d70c89ee82d6d3b95a7f Mon Sep 17 00:00:00 2001 From: Patrick Arminio Date: Fri, 14 Jun 2024 10:43:45 +0200 Subject: [PATCH 07/11] Apply querysets changes --- cursor_pagination.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cursor_pagination.py b/cursor_pagination.py index dbdd417..e5b45b3 100644 --- a/cursor_pagination.py +++ b/cursor_pagination.py @@ -89,6 +89,8 @@ def _apply_paginator_arguments(self, qs, first=None, last=None, after=None, befo if last is not None: qs = qs.order_by(*self._nulls_ordering(reverse_ordering(self.ordering), from_last=True))[:last + 1] + return qs + def _get_cursor_page(self, items, has_additional, first, last, after, before): """ Create and return the cursor page for the given items @@ -104,7 +106,7 @@ def _get_cursor_page(self, items, has_additional, first, last, after, before): def page(self, first=None, last=None, after=None, before=None): qs = self.queryset - self._apply_paginator_arguments(qs, first, last, after, before) + qs = self._apply_paginator_arguments(qs, first, last, after, before) qs = list(qs) page_size = first or last @@ -117,7 +119,7 @@ def page(self, first=None, last=None, after=None, before=None): async def apage(self, first=None, last=None, after=None, before=None): qs = self.queryset - self._apply_paginator_arguments(qs, first, last, after, before) + qs = self._apply_paginator_arguments(qs, first, last, after, before) page_size = first or last items = [] From a98e8df2a57e87d247947d0e6d7ad6c7f5521cdd Mon Sep 17 00:00:00 2001 From: Patrick Arminio Date: Fri, 14 Jun 2024 10:46:40 +0200 Subject: [PATCH 08/11] Add acount method on CursorPage --- cursor_pagination.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cursor_pagination.py b/cursor_pagination.py index e5b45b3..3caf6ea 100644 --- a/cursor_pagination.py +++ b/cursor_pagination.py @@ -36,6 +36,9 @@ def __getitem__(self, key): def __repr__(self): return '' % (', '.join(repr(i) for i in self.items[:21]), ' (remaining truncated)' if len(self.items) > 21 else '') + async def acount(self): + return await self.items.acount() + class CursorPaginator(object): delimiter = '|' From 543ad6d9ad60a2b0edf3182abb8e42df911c1e20 Mon Sep 17 00:00:00 2001 From: Patrick Arminio Date: Fri, 14 Jun 2024 10:48:36 +0200 Subject: [PATCH 09/11] Bump django --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e639155..6ba0198 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: python-version: [3.8, 3.9, "3.10", "3.11"] - django-version: [4.0, 3.2] + django-version: [5.0, 4.2] services: postgres: From ccc959fb8db6d3a084fe41dbfdc743f7a5f95a3e Mon Sep 17 00:00:00 2001 From: Patrick Arminio Date: Fri, 14 Jun 2024 10:51:15 +0200 Subject: [PATCH 10/11] Drop tests on older version --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6ba0198..9086ee9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - python-version: [3.8, 3.9, "3.10", "3.11"] + python-version: ["3.10", "3.11"] django-version: [5.0, 4.2] services: From aa9c7d8cee6c80e1de2900ceebd683c533e97399 Mon Sep 17 00:00:00 2001 From: Bradley Oesch Date: Fri, 21 Jun 2024 14:22:00 -0400 Subject: [PATCH 11/11] Remove acount() and fix 0 first/last bug --- cursor_pagination.py | 11 ++--------- tests/tests.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/cursor_pagination.py b/cursor_pagination.py index 3caf6ea..dbcc1d7 100644 --- a/cursor_pagination.py +++ b/cursor_pagination.py @@ -36,9 +36,6 @@ def __getitem__(self, key): def __repr__(self): return '' % (', '.join(repr(i) for i in self.items[:21]), ' (remaining truncated)' if len(self.items) > 21 else '') - async def acount(self): - return await self.items.acount() - class CursorPaginator(object): delimiter = '|' @@ -75,10 +72,6 @@ def _apply_paginator_arguments(self, qs, first=None, last=None, after=None, befo """ Apply first/after, last/before filtering to the queryset """ - page_size = first or last - if page_size is None: - return CursorPage(qs, self) - from_last = last is not None if from_last and first is not None: raise ValueError('Cannot process first and last') @@ -112,7 +105,7 @@ def page(self, first=None, last=None, after=None, before=None): qs = self._apply_paginator_arguments(qs, first, last, after, before) qs = list(qs) - page_size = first or last + page_size = first if first is not None else last items = qs[:page_size] if last is not None: items.reverse() @@ -124,7 +117,7 @@ async def apage(self, first=None, last=None, after=None, before=None): qs = self.queryset qs = self._apply_paginator_arguments(qs, first, last, after, before) - page_size = first or last + page_size = first if first is not None else last items = [] async for item in qs[:page_size].aiterator(): items.append(item) diff --git a/tests/tests.py b/tests/tests.py index a769dd1..0603b44 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -55,6 +55,18 @@ def setUpTestData(cls): cls.items.append(post) cls.paginator = CursorPaginator(Post.objects.all(), ('-created',)) + def test_first_page_zero(self): + page = self.paginator.page(first=0) + self.assertSequenceEqual(page, []) + self.assertTrue(page.has_next) + self.assertFalse(page.has_previous) + + async def test_async_first_page_zero(self): + page = await self.paginator.apage(first=0) + self.assertSequenceEqual(page, []) + self.assertTrue(page.has_next) + self.assertFalse(page.has_previous) + def test_first_page(self): page = self.paginator.page(first=2) self.assertSequenceEqual(page, [self.items[0], self.items[1]]) @@ -127,6 +139,18 @@ def setUpTestData(cls): cls.items.append(post) cls.paginator = CursorPaginator(Post.objects.all(), ('-created',)) + def test_first_page_zero(self): + page = self.paginator.page(last=0) + self.assertSequenceEqual(page, []) + self.assertTrue(page.has_previous) + self.assertFalse(page.has_next) + + async def test_async_first_page_zero(self): + page = await self.paginator.apage(last=0) + self.assertSequenceEqual(page, []) + self.assertTrue(page.has_previous) + self.assertFalse(page.has_next) + def test_first_page(self): page = self.paginator.page(last=2) self.assertSequenceEqual(page, [self.items[18], self.items[19]])