From 7b17f8ebe84b61244ede5e98bef092e3e5c8d1c1 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 22 Dec 2024 21:50:51 +0100 Subject: [PATCH] Add filter to drizzle converter --- api/src/controllers/movies.ts | 46 ++++++++-- api/src/db/schema/utils.ts | 2 +- api/src/models/error.ts | 2 +- api/src/models/utils/filters-sql.ts | 133 ++++++++++++++++++++++++++++ api/src/models/utils/filters.ts | 22 +---- 5 files changed, 177 insertions(+), 28 deletions(-) create mode 100644 api/src/models/utils/filters-sql.ts diff --git a/api/src/controllers/movies.ts b/api/src/controllers/movies.ts index 8a5d4220c..0905eaadc 100644 --- a/api/src/controllers/movies.ts +++ b/api/src/controllers/movies.ts @@ -1,14 +1,15 @@ import { and, desc, eq, sql } from "drizzle-orm"; import { Elysia, t } from "elysia"; import { KError } from "~/models/error"; -import { isUuid, processLanguages } from "~/models/utils"; +import { Genre, isUuid, processLanguages } from "~/models/utils"; import { comment, RemovePrefix } from "~/utils"; import { db } from "../db"; import { shows, showTranslations } from "../db/schema/shows"; import { getColumns } from "../db/schema/utils"; import { bubble } from "../models/examples"; -import { Movie, type MovieStatus, MovieTranslation } from "../models/movie"; +import { Movie, MovieStatus, MovieTranslation } from "../models/movie"; import { Page } from "~/models/utils/page"; +import { type Filter, parseFilters } from "~/models/utils/filter-sql"; // drizzle is bugged and doesn't allow js arrays to be used in raw sql. export function sqlarr(array: unknown[]) { @@ -37,6 +38,20 @@ const getTranslationQuery = (languages: string[]) => { const { pk: _, kind, startAir, endAir, ...moviesCol } = getColumns(shows); +const movieFilters: Filter = { + genres: { + column: shows.genres, + type: "enum", + values: Genre.enum, + isArray: true, + }, + rating: { column: shows.rating, type: "int" }, + status: { column: shows.status, type: "enum", values: MovieStatus.enum }, + runtime: { column: shows.runtime, type: "float" }, + airDate: { column: shows.startAir, type: "date" }, + originalLanguage: { column: shows.originalLanguage, type: "string" }, +}; + export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) .model({ movie: Movie, @@ -100,8 +115,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) example: "en-us, ja;q=0.5", description: comment` List of languages you want the data in. - This follows the Accept-Language offical specification - (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language). + This follows the [Accept-Language offical specification](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language). `, }), }), @@ -135,7 +149,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) .get( "", async ({ - query: { limit, after, sort }, + query: { limit, after, sort, filter }, headers: { "accept-language": languages }, }) => { const langs = processLanguages(languages); @@ -146,6 +160,9 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) if (key === "airDate") return { key: "startAir" as const, desc }; return { key, desc }; }); + const filters = parseFilters(filter, movieFilters); + + // TODO: Add sql indexes on order keys const items = await db .select({ @@ -156,6 +173,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) }) .from(shows) .innerJoin(transQ, eq(shows.pk, transQ.pk)) + .where(filters) .orderBy( ...order.map((x) => (x.desc ? desc(shows[x.key]) : shows[x.key])), shows.pk, @@ -168,6 +186,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) detail: { description: "Get all movies" }, query: t.Object({ sort: t.Array( + // TODO: Add random t.UnionEnum([ "slug", "-slug", @@ -183,6 +202,18 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) // TODO: support explode: true (allow sort=slug,-createdAt). needs a pr to elysia { explode: false, default: ["slug"] }, ), + filter: t.Optional( + t.String({ + description: comment` + Filters to apply to the query. + This is based on [odata's filter specification](https://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#sec_SystemQueryOptionfilter). + + Filters available: ${Object.keys(movieFilters).join(", ")} + `, + example: + "(rating gt 75 and genres has action) or status eq planned", + }), + ), limit: t.Integer({ minimum: 1, maximum: 250, @@ -205,10 +236,9 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) example: "en-us, ja;q=0.5", description: comment` List of languages you want the data in. - This follows the Accept-Language offical specification - (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language). + This follows the [Accept-Language offical specification](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language). - In this request, * is always implied (if no language could satisfy the request, kyoo will use any language available). + In this request, * is always implied (if no language could satisfy the request, kyoo will use any language available.) `, }), }), diff --git a/api/src/db/schema/utils.ts b/api/src/db/schema/utils.ts index 9ba146da0..e7963287d 100644 --- a/api/src/db/schema/utils.ts +++ b/api/src/db/schema/utils.ts @@ -19,7 +19,7 @@ import { import type { AnySQLiteSelect } from "drizzle-orm/sqlite-core"; import type { WithSubquery } from "drizzle-orm/subquery"; import { db } from ".."; -import { CasingCache } from "drizzle-orm/casing"; +import type { CasingCache } from "drizzle-orm/casing"; export const schema = pgSchema("kyoo"); diff --git a/api/src/models/error.ts b/api/src/models/error.ts index c795ad123..a2bf9822f 100644 --- a/api/src/models/error.ts +++ b/api/src/models/error.ts @@ -3,6 +3,6 @@ import { t } from "elysia"; export const KError = t.Object({ status: t.Integer(), message: t.String(), - details: t.Any(), + details: t.Optional(t.Any()), }); export type KError = typeof KError.static; diff --git a/api/src/models/utils/filters-sql.ts b/api/src/models/utils/filters-sql.ts new file mode 100644 index 000000000..ad79ce9e6 --- /dev/null +++ b/api/src/models/utils/filters-sql.ts @@ -0,0 +1,133 @@ +import { + and, + type Column, + eq, + gt, + gte, + lt, + lte, + ne, + not, + or, + type SQL, + sql, +} from "drizzle-orm"; +import { comment } from "~/utils"; +import type { KError } from "../error"; +import { type Expression, expression, type Operator } from "./filters"; + +export type Filter = { + [key: string]: + | { + column: Column; + type: "int" | "float" | "date" | "string"; + isArray?: boolean; + } + | { column: Column; type: "enum"; values: string[]; isArray?: boolean }; +}; + +export const parseFilters = (filter: string | undefined, config: Filter) => { + if (!filter) return undefined; + const ret = expression.parse(filter); + if (!ret.isOk) { + throw new Error("todo"); + // return { status: 422, message: `Invalid filter: ${filter}.`, details: ret } + } + + return toDrizzle(ret.value, config); +}; + +const opMap: Record = { + eq: eq, + ne: ne, + gt: gt, + ge: gte, + lt: lt, + le: lte, + has: eq, +}; + +const toDrizzle = (expr: Expression, config: Filter): SQL | KError => { + switch (expr.type) { + case "op": { + const where = `${expr.property} ${expr.operator} ${expr.value}`; + const prop = config[expr.property]; + + if (!prop) { + return { + status: 422, + message: comment` + Invalid property: ${expr.property}. + Expected one of ${Object.keys(config).join(", ")}. + `, + details: { in: where }, + }; + } + + if (prop.type !== expr.value.type) { + return { + status: 422, + message: comment` + Invalid value for property ${expr.property}. + Got ${expr.value.type} but expected ${prop.type}. + `, + details: { in: where }, + }; + } + if ( + prop.type === "enum" && + (expr.value.type === "enum" || expr.value.type === "string") && + !prop.values.includes(expr.value.value) + ) { + return { + status: 422, + message: comment` + Invalid value ${expr.value.value} for property ${expr.property}. + Expected one of ${prop.values.join(", ")} but got ${expr.value.value}. + `, + details: { in: where }, + }; + } + + if (prop.isArray) { + if (expr.operator !== "has" && expr.operator !== "eq") { + return { + status: 422, + message: comment` + Property ${expr.property} is an array but you wanted to use the + operator ${expr.operator}. Only "has" is supported ("eq" is also aliased to "has") + `, + details: { in: where }, + }; + } + return sql`${expr.value.value} = any(${prop.column})`; + } + return opMap[expr.operator](prop.column, expr.value.value); + } + case "and": { + const lhs = toDrizzle(expr.lhs, config); + const rhs = toDrizzle(expr.rhs, config); + if ("status" in lhs) return lhs; + if ("status" in rhs) return rhs; + return and(lhs, rhs)!; + } + case "or": { + const lhs = toDrizzle(expr.lhs, config); + const rhs = toDrizzle(expr.rhs, config); + if ("status" in lhs) return lhs; + if ("status" in rhs) return rhs; + return or(lhs, rhs)!; + } + case "not": { + const lhs = toDrizzle(expr.expression, config); + if ("status" in lhs) return lhs; + return not(lhs); + } + default: + return exhaustiveCheck(expr); + } +}; + +function exhaustiveCheck(v: never): never { + return v; +} diff --git a/api/src/models/utils/filters.ts b/api/src/models/utils/filters.ts index a1e17fb18..35ccce00d 100644 --- a/api/src/models/utils/filters.ts +++ b/api/src/models/utils/filters.ts @@ -22,21 +22,16 @@ import { between, recover, } from "parjs/combinators"; -import type { KError } from "../error"; -export type Filter = { - [key: string]: any; -}; - -type Property = string; -type Value = +export type Property = string; +export type Value = | { type: "int"; value: number } | { type: "float"; value: number } | { type: "date"; value: string } | { type: "string"; value: string } | { type: "enum"; value: string }; -const operators = ["eq", "ne", "gt", "ge", "lt", "le", "has", "in"] as const; -type Operator = (typeof operators)[number]; +const operators = ["eq", "ne", "gt", "ge", "lt", "le", "has"] as const; +export type Operator = (typeof operators)[number]; export type Expression = | { type: "op"; operator: Operator; property: Property; value: Value } | { type: "and"; lhs: Expression; rhs: Expression } @@ -133,12 +128,3 @@ const not = t(string("not")).pipe( const brackets = expression.pipe(between("(", ")")); expr.init(not.pipe(or(brackets, operation))); - -export const parseFilter = ( - filter: string, - config: Filter, -): Expression | KError => { - const ret = expression.parse(filter); - if (ret.isOk) return ret.value; - return { status: 422, message: `Invalid filter: ${filter}.`, details: ret }; -};