From 8e466ef15c7ec5945c7ae10cbf32d02c26ac3d2f Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 6 Jan 2025 21:44:25 +0100 Subject: [PATCH] Create keyset pagination function --- api/src/controllers/movies.ts | 47 +++++++++++-------------- api/src/db/schema/utils.ts | 2 +- api/src/models/utils/index.ts | 2 ++ api/src/models/utils/keyset-paginate.ts | 47 +++++++++++++++++++++++++ api/src/models/utils/sort.ts | 8 ++--- 5 files changed, 75 insertions(+), 31 deletions(-) create mode 100644 api/src/models/utils/keyset-paginate.ts diff --git a/api/src/controllers/movies.ts b/api/src/controllers/movies.ts index 093ff70c4..3633ea855 100644 --- a/api/src/controllers/movies.ts +++ b/api/src/controllers/movies.ts @@ -1,20 +1,23 @@ import { and, desc, eq, sql } from "drizzle-orm"; import { Elysia, t } from "elysia"; import { KError } from "~/models/error"; -import { - type FilterDef, - Genre, - isUuid, - processLanguages, -} from "~/models/utils"; import { comment } from "~/utils"; import { db } from "../db"; import { shows, showTranslations } from "../db/schema/shows"; import { getColumns } from "../db/schema/utils"; import { bubble } from "../models/examples"; import { Movie, MovieStatus, MovieTranslation } from "../models/movie"; -import { Filter, type Page } from "~/models/utils"; -import { Sort } from "~/models/utils/sort"; +import { + Filter, + Sort, + type FilterDef, + Genre, + isUuid, + keysetPaginate, + processLanguages, + type Page, + createPage, +} from "~/models/utils"; // drizzle is bugged and doesn't allow js arrays to be used in raw sql. export function sqlarr(array: unknown[]) { @@ -156,6 +159,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) async ({ query: { limit, after, sort, filter }, headers: { "accept-language": languages }, + request: { url }, }) => { const langs = processLanguages(languages); const [transQ, transCol] = getTranslationQuery(langs); @@ -171,33 +175,24 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) }) .from(shows) .innerJoin(transQ, eq(shows.pk, transQ.pk)) - .where(filter) + .where(and(filter, keysetPaginate({ table: shows, after, sort }))) .orderBy( ...sort.map((x) => (x.desc ? desc(shows[x.key]) : shows[x.key])), shows.pk, ) .limit(limit); - return { items, next: "", prev: "", this: "" }; + return createPage(items, { url, sort }); }, { detail: { description: "Get all movies" }, query: t.Object({ - sort: Sort( - [ - "slug", - "rating", - "airDate", - "createdAt", - "nextRefresh", - ], - { - // TODO: Add random - remap: { airDate: "startAir" }, - default: ["slug"], - description: "How to sort the query", - }, - ), + sort: Sort(["slug", "rating", "airDate", "createdAt", "nextRefresh"], { + // TODO: Add random + remap: { airDate: "startAir" }, + default: ["slug"], + description: "How to sort the query", + }), filter: t.Optional(Filter({ def: movieFilters })), limit: t.Integer({ minimum: 1, @@ -207,7 +202,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) }), after: t.Optional( t.String({ - format: "uuid", + format: "byte", description: comment` Id of the cursor in the pagination. You can ignore this and only use the prev/next field in the response. diff --git a/api/src/db/schema/utils.ts b/api/src/db/schema/utils.ts index e7963287d..dae8a8011 100644 --- a/api/src/db/schema/utils.ts +++ b/api/src/db/schema/utils.ts @@ -69,7 +69,7 @@ export function conflictUpdateAllExcept< return updateColumns.reduce( (acc, [colName, col]) => { - // @ts-ignore: drizzle internal + // @ts-expect-error: drizzle internal const name = (db.dialect.casing as CasingCache).getColumnCasing(col); acc[colName as keyof typeof acc] = sql.raw(`excluded."${name}"`); return acc; diff --git a/api/src/models/utils/index.ts b/api/src/models/utils/index.ts index 31076b9a4..70c086260 100644 --- a/api/src/models/utils/index.ts +++ b/api/src/models/utils/index.ts @@ -5,3 +5,5 @@ export * from "./language"; export * from "./resource"; export * from "./filters"; export * from "./page"; +export * from "./sort"; +export * from "./keyset-paginate"; diff --git a/api/src/models/utils/keyset-paginate.ts b/api/src/models/utils/keyset-paginate.ts new file mode 100644 index 000000000..f083d92af --- /dev/null +++ b/api/src/models/utils/keyset-paginate.ts @@ -0,0 +1,47 @@ +import type { NonEmptyArray, Sort } from "./sort"; +import { eq, or, type Column, and, gt, lt } from "drizzle-orm"; + +type Table = Record; + +// Create a filter (where) expression on the query to skip everything before/after the referenceID. +// The generalized expression for this in pseudocode is: +// (x > a) OR +// (x = a AND y > b) OR +// (x = a AND y = b AND z > c) OR... +// +// Of course, this will be a bit more complex when ASC and DESC are mixed. +// Assume x is ASC, y is DESC, and z is ASC: +// (x > a) OR +// (x = a AND y < b) OR +// (x = a AND y = b AND z > c) OR... +export const keysetPaginate = < + const T extends NonEmptyArray, + const Remap extends Partial>, +>({ + table, + sort, + after, +}: { + table: Table<"pk" | Sort[number]["key"]>; + after: string | undefined; + sort: Sort; +}) => { + if (!after) return undefined; + const cursor: Record = JSON.parse( + Buffer.from(after, "base64").toString("utf-8"), + ); + + // TODO: Add an outer query >= for perf + // PERF: See https://use-the-index-luke.com/sql/partial-results/fetch-next-page#sb-equivalent-logic + let where = undefined; + let previous = undefined; + for (const by of [...sort, { key: "pk" as const, desc: false }]) { + const cmp = by.desc ? lt : gt; + where = or(where, and(previous, cmp(table[by.key], cursor[by.key]))); + previous = and(previous, eq(table[by.key], cursor[by.key])); + } + + return where; +}; + + diff --git a/api/src/models/utils/sort.ts b/api/src/models/utils/sort.ts index 5dcf2c964..fe4358f07 100644 --- a/api/src/models/utils/sort.ts +++ b/api/src/models/utils/sort.ts @@ -1,14 +1,14 @@ import { t } from "elysia"; -type Sort< +export type Sort< T extends string[], Remap extends Partial>, > = { - key: Exclude | Remap[keyof Remap]; + key: Exclude | NonNullable; desc: boolean; }[]; -type NonEmptyArray = [T, ...T[]]; +export type NonEmptyArray = [T, ...T[]]; export const Sort = < const T extends NonEmptyArray, @@ -44,7 +44,7 @@ export const Sort = < return sort.map((x) => { const desc = x[0] === "-"; const key = (desc ? x.substring(1) : x) as T[number]; - if (key in remap) return { key: remap[key], desc }; + if (key in remap) return { key: remap[key]!, desc }; return { key: key as Exclude, desc }; }); })