From 59a6d486acadc9becffa28d223ca6b3288587d5b Mon Sep 17 00:00:00 2001 From: Katrix Date: Sun, 4 Aug 2024 11:08:18 +0200 Subject: [PATCH] Test fixes, array agg, docs --- .../platform/sql/query/SqlQueries.scala | 3 - .../platform/sql/query/SqlQueriesBase.scala | 5 +- .../platform/sql/value/SqlArrays.scala | 7 + .../platform/sql/value/SqlDbValues.scala | 2 +- .../platform/sql/value/SqlDbValuesBase.scala | 4 +- .../dataprism/sharedast/AstRenderer.scala | 1 + .../dataprism/sharedast/SharedSqlAst.scala | 1 + .../scala/dataprism/PlatformArraysSuite.scala | 21 +- docs/_docs/guide/02_queries.md | 18 +- docs/_docs/guide/05_dbvalue_expressions.md | 190 +++++++++++++++++- docs/_docs/guide/10_fs2_effects.md | 4 +- 11 files changed, 219 insertions(+), 37 deletions(-) diff --git a/common/src/main/scala/dataprism/platform/sql/query/SqlQueries.scala b/common/src/main/scala/dataprism/platform/sql/query/SqlQueries.scala index c47244a9..9504cf85 100644 --- a/common/src/main/scala/dataprism/platform/sql/query/SqlQueries.scala +++ b/common/src/main/scala/dataprism/platform/sql/query/SqlQueries.scala @@ -892,8 +892,5 @@ trait SqlQueries extends SqlQueriesBase { platform: SqlQueryPlatform => } extension [A](query: Query[[F[_]] =>> F[A]]) - // TODO: Make use of an implicit conversion here? - @targetName("queryAsMany") override def asMany: Many[A] = query.asDbValue.unsafeDbValAsMany - @targetName("queryAsDbValue") override def asDbValue: DbValue[A] = SqlDbValue.SubSelect(query).lift } diff --git a/common/src/main/scala/dataprism/platform/sql/query/SqlQueriesBase.scala b/common/src/main/scala/dataprism/platform/sql/query/SqlQueriesBase.scala index 3aa04225..f44210b0 100644 --- a/common/src/main/scala/dataprism/platform/sql/query/SqlQueriesBase.scala +++ b/common/src/main/scala/dataprism/platform/sql/query/SqlQueriesBase.scala @@ -130,13 +130,10 @@ trait SqlQueriesBase extends SqlQueryPlatformBase, SqlDbValuesBase { platform => end SqlQueryCompanion extension [A](query: Query[[F[_]] =>> F[A]]) - // TODO: Make use of an implicit conversion here? - @targetName("queryAsMany") def asMany: Many[A] - @targetName("queryAsDbValue") def asDbValue: DbValue[A] type Api <: SqlQueryApi & SqlDbValueApi & QueryApi trait SqlQueryApi { - export platform.{asDbValue, asMany} + export platform.asDbValue } } diff --git a/common/src/main/scala/dataprism/platform/sql/value/SqlArrays.scala b/common/src/main/scala/dataprism/platform/sql/value/SqlArrays.scala index 15229e90..5f56d03a 100644 --- a/common/src/main/scala/dataprism/platform/sql/value/SqlArrays.scala +++ b/common/src/main/scala/dataprism/platform/sql/value/SqlArrays.scala @@ -128,6 +128,12 @@ trait SqlArrays extends SqlDbValuesBase { platform => } } + extension [A](many: Many[A]) + // TODO: Check that the return type is indeed Long on all platforms + @targetName("sqlArrayArrayAgg") def arrayAgg: DbValue[Seq[A]] = + val v = Many.unsafeAsDbValue(many) + Impl.function(SqlExpr.FunctionName.ArrayAgg, Seq(v.asAnyDbVal), arrayOfType(v.tpe)) + type Impl <: SqlArraysImpl & SqlValuesBaseImpl & SqlBaseImpl trait SqlArraysImpl { def queryFunction[A[_[_]]: ApplyKC: TraverseKC]( @@ -141,6 +147,7 @@ trait SqlArrays extends SqlDbValuesBase { platform => trait SqlArraysApi { export platform.DbArrayLike export platform.DbArrayLike.given + export platform.arrayAgg type DbArrayCompanion = platform.DbArrayCompanion inline def DbArray: DbArrayCompanion = platform.DbArray diff --git a/common/src/main/scala/dataprism/platform/sql/value/SqlDbValues.scala b/common/src/main/scala/dataprism/platform/sql/value/SqlDbValues.scala index 5fd1b2c6..338e1606 100644 --- a/common/src/main/scala/dataprism/platform/sql/value/SqlDbValues.scala +++ b/common/src/main/scala/dataprism/platform/sql/value/SqlDbValues.scala @@ -582,7 +582,7 @@ trait SqlDbValues extends SqlDbValuesBase { platform: SqlQueryPlatform => .Function(SqlExpr.FunctionName.Count, Seq(many.unsafeAsDbValue.asAnyDbVal), AnsiTypes.bigint.notNull) .lift - inline def unsafeAsDbValue: DbValue[A] = many.asInstanceOf[DbValue[A]] + def unsafeAsDbValue: DbValue[A] = many.asInstanceOf[DbValue[A]] def map[B](f: DbValue[A] => DbValue[B]): Many[B] = f(many.asInstanceOf[DbValue[A]]).asInstanceOf[DbValue[Any]] } diff --git a/common/src/main/scala/dataprism/platform/sql/value/SqlDbValuesBase.scala b/common/src/main/scala/dataprism/platform/sql/value/SqlDbValuesBase.scala index 32bbe67c..06136734 100644 --- a/common/src/main/scala/dataprism/platform/sql/value/SqlDbValuesBase.scala +++ b/common/src/main/scala/dataprism/platform/sql/value/SqlDbValuesBase.scala @@ -254,7 +254,7 @@ trait SqlDbValuesBase extends SqlQueryPlatformBase { platform => // TODO: Check that the return type is indeed Long on all platforms def count: DbValue[Long] - inline def unsafeAsDbValue: DbValue[A] + def unsafeAsDbValue: DbValue[A] def map[B](f: DbValue[A] => DbValue[B]): Many[B] } @@ -283,7 +283,7 @@ trait SqlDbValuesBase extends SqlQueryPlatformBase { platform => val Case: CaseCompanion type CaseCompanion <: SqlCaseCompanion - trait SqlCaseCompanion { + trait SqlCaseCompanion { def apply[A](v: DbValue[A]): ValueCase0[A] def when[A](whenCond: DbValue[Boolean])(thenV: DbValue[A]): ConditionCase[A] } diff --git a/common/src/main/scala/dataprism/sharedast/AstRenderer.scala b/common/src/main/scala/dataprism/sharedast/AstRenderer.scala index 54532609..df1e3282 100644 --- a/common/src/main/scala/dataprism/sharedast/AstRenderer.scala +++ b/common/src/main/scala/dataprism/sharedast/AstRenderer.scala @@ -190,6 +190,7 @@ class AstRenderer[Codec[_]](ansiTypes: AnsiTypes[Codec], getCodecTypeName: [A] = case SqlExpr.FunctionName.ArrayContains => normal("array_contains") case SqlExpr.FunctionName.TrimArray => normal("trim_array") case SqlExpr.FunctionName.Unnest => normal("unnest") + case SqlExpr.FunctionName.ArrayAgg => normal("array_agg") case SqlExpr.FunctionName.Custom(f) => normal(f) diff --git a/common/src/main/scala/dataprism/sharedast/SharedSqlAst.scala b/common/src/main/scala/dataprism/sharedast/SharedSqlAst.scala index 95bb247a..4aaf55c8 100644 --- a/common/src/main/scala/dataprism/sharedast/SharedSqlAst.scala +++ b/common/src/main/scala/dataprism/sharedast/SharedSqlAst.scala @@ -166,6 +166,7 @@ object SqlExpr { case ArrayContains case TrimArray case Unnest + case ArrayAgg case Custom(f: String) diff --git a/common/src/test/scala/dataprism/PlatformArraysSuite.scala b/common/src/test/scala/dataprism/PlatformArraysSuite.scala index 4e4986b3..2ba857cc 100644 --- a/common/src/test/scala/dataprism/PlatformArraysSuite.scala +++ b/common/src/test/scala/dataprism/PlatformArraysSuite.scala @@ -116,21 +116,28 @@ trait PlatformArraysSuite[Codec0[_], Platform <: SqlQueryPlatform { type Codec[A val arr2Type: Type[Seq[Seq[A]]] = platform.arrayOfType(arr1Type) Select( Query.of( - // Seq().as(arr1Type), - // Seq(v1).as(arr1Type), - // Seq(v1, v2).as(arr1Type), + Seq().as(arr1Type), + Seq(v1).as(arr1Type), + Seq(v1, v2).as(arr1Type), Seq(Seq(v1, v2), Seq(v3, v4)).as(arr2Type) ) ) .runOne[F] - .map: /*r1, r2, r3,*/ r4 => + .map: (r1, r2, r3, r4) => expect.all( - // r1 == Seq(), - // r2 == Seq(v1), - // r3 == Seq(v1, v2), + r1 == Seq(), + r2 == Seq(v1), + r3 == Seq(v1, v2), r4 == Seq(Seq(v1, v2), Seq(v3, v4)) ) + typeTest("ArrayAgg", tpe): + configuredForall(genNel(gen)): (v, vs) => + Select(Query.values(tpe)(v, vs*).mapSingleGrouped(v => v.arrayAgg)) + .runOne[F] + .map: r => + expect(r == (v +: vs)) + end testArrays def testArrayUnnest[A: Show](tpe: Type[A], gen: Gen[A], opV: A => A, opDb: DbValue[A] => DbValue[A])( diff --git a/docs/_docs/guide/02_queries.md b/docs/_docs/guide/02_queries.md index 38a083ea..4a581e22 100644 --- a/docs/_docs/guide/02_queries.md +++ b/docs/_docs/guide/02_queries.md @@ -133,15 +133,9 @@ traditional `groupBy` function, as a `groupMap` function maps better to DataPris The second function does the aggregation given both the extracted value, and the values of the query, which are now wrapped in `Many`. Here are some examples: -Arrays are currently out of commission. - ```scala 3 sc-compile-with:User.scala -/* import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} -//Needed for arrayAgg currently -import dataprism.jdbc.sql.PostgresJdbcTypes.ArrayMapping.given_ArrayMapping_A - val q: Query[UserK] = Query.from(UserK.table) val q1: Query[[F[_]] =>> (F[String], F[Seq[String]])] = @@ -154,7 +148,6 @@ val q2: Query[[F[_]] =>> (F[Option[String]], F[String], F[Seq[String]])] = (t: (DbValue[Option[String]], DbValue[String]), v: UserK[Many]) => (t._1, t._2, v.email.arrayAgg) ) -*/ ``` Note how you don't have to directly return a column from the grouping function. For example, in `q3` @@ -173,8 +166,8 @@ want flatMap for a database which does not support `LATERAL`, create your own pl import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} //TODO: Does not compile for some reason. Fix MapRes -//val q1: Query[[F[_]] =>> (UserK[F], UserK[F])] = -// Query.from(UserK.table).flatMap(u1 => Query.from(UserK.table).map(u2 => (u1, u2))) +val q1: Query[[F[_]] =>> (UserK[F], UserK[F])] = + Query.from(UserK.table).flatMap(u1 => Query.from(UserK.table).map(u2 => (u1, u2))) val q2: Query[UserK] = for u <- Query.from(UserK.table) @@ -190,15 +183,9 @@ and similar. The HKD used to define a table is not special. Any HKD (or not even with `perspective.ApplyKC` and `perspective.TraverseKC` instances can be used as a result type in functions like `map`. Here's one example: -Arrays are currently out of commission. - ```scala 3 sc-compile-with:User.scala -/* import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} -//Needed for arrayAgg currently -import dataprism.jdbc.sql.PostgresJdbcTypes.ArrayMapping.given_ArrayMapping_A - case class UsersWithEmailK[F[_]](email: F[String], usernames: F[Seq[String]]) object UsersWithEmailK: @@ -209,7 +196,6 @@ val q1: Query[UsersWithEmailK] = Query.from(UserK.table).groupMap((v: UserK[DbValue]) => v.email)( (email: DbValue[String], v: UserK[Many]) => UsersWithEmailK(email, v.username.arrayAgg) ) -*/ ``` For more info, see [MapRes and Exotic data](07_mapres_exotic_data.md) diff --git a/docs/_docs/guide/05_dbvalue_expressions.md b/docs/_docs/guide/05_dbvalue_expressions.md index e704c399..adac0e23 100644 --- a/docs/_docs/guide/05_dbvalue_expressions.md +++ b/docs/_docs/guide/05_dbvalue_expressions.md @@ -19,22 +19,208 @@ do `Some("a").as(text.nullable)` or more simply `"a".asNullable(text)`. ## Expressions -TODO +Here are some operations that can be done on expressions. This is not a comprehensive list. + +### Primitive operators and functions + +Here are some of the operators and functions found on all db values: + +All DbValues can be compared for equality. This is done using the `===` and `!==` operators. + +```scala 3 +import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} + +val a: DbValue[Boolean] = ??? +val b: DbValue[Boolean] = ??? + +val eq = a === b +val neq = a !== b +``` + +Casting and converting to Some. For casting you need a `CastType`. For most platforms, this is the normal types used +elsewhere in DataPrism. For MySQL platforms, they are special types found at `MySqlJdbcTypes.castType`. + +``` +import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} +import dataprism.jdbc.sql.PostgresJdbcTypes +val a: DbValue[Boolean] = ??? + +val asSome = a.asSome //Wrapping in Some +val casted = a.cast(PostgresJdbcTypes.integer) +``` ### Booleans +Normal boolean operations work with DbValues + +```scala 3 +import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} + +val a: DbValue[Boolean] = ??? +val b: DbValue[Boolean] = ??? + +val and = a && b +val or = a || b +val not = !a +``` + ### Numerics +Most dataprism numeric operators are found in the `SqlNumeric` typeclass + +```scala 3 +import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} + +val a: DbValue[Double] = ??? +val b: DbValue[Double] = ??? + +val plus = a + b +val minus = a - b +val times = a * b +val div = a / b +val mod = a % b +val neg = -a +``` + +### Math functions + +DataPrism puts a lot of math functions in the `DbMath` object. + +```scala 3 +import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} +import dataprism.jdbc.sql.PostgresJdbcTypes + +val a: DbValue[Double] = ??? +val b: DbValue[Double] = ??? + +val pow = DbMath.pow(a, b) +val sqrt = DbMath.sqrt(a) +val ceil = DbMath.ceil(a) +val floor = DbMath.floor(a) +val log = DbMath.log(a, b) +val sign = DbMath.sign(a) +val pi = DbMath.pi(PostgresJdbcTypes.doublePrecision) //Cast type here +val random = DbMath.random(PostgresJdbcTypes.doublePrecision) //Cast type here + +val sin = DbMath.sin(a) +val cos = DbMath.cos(a) +val tan = DbMath.tan(a) +``` + ### Nullable values +Here are operations that can be used on nullable values. + +```scala 3 +import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} + +val a: DbValue[Option[Double]] = ??? +val b: DbValue[Option[Double]] = ??? +val c: DbValue[Double] = ??? + +val map = a.map(v => v + c) +val filter = a.filter(v => v > c) +val flatMap = a.flatMap(_ => b) + +val isEmpty = a.isEmpty +val isDefined = a.isDefined + +val orElse = a.orElse(b) +val getOrElse = a.getOrElse(c) + +val mapN = (a, b).mapNullableN((v1, v2) => v1 + v2) +``` ### Many +Many is the type used to represent a group of values, usually from groupBy or having. + +```scala 3 +import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} + +val v: DbValue[Double] = ??? +val m1: Many[Double] = ??? +val m2: Many[Double] = ??? + +val map = m1.map(_ + v) +val count = m2.count + +val mapN = (m1, m2).mapManyN((a, b) => a + b) +``` ### String operations +Here are some SQL string operations. These are found in the `SqlString` typeclass. + +```scala 3 +import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} +import dataprism.jdbc.sql.PostgresJdbcTypes + +val a: DbValue[String] = ??? +val b: DbValue[String] = ??? +val c: DbValue[String] = ??? + +val concat = a ++ b +val repeat = a * 5.as(PostgresJdbcTypes.integer) +val length = a.length +val lower = a.toLowerCase +val upper = a.toUpperCase +val like = a.like(b) +val startsWith = a.startsWith(b) +val endsWith = a.endsWith(b) +val replace = a.replace(b, c) + +val concat2 = SqlString.concat(a, b) +val concatWs = SqlString.concatWs(" ".as(PostgresJdbcTypes.text), a, b) +``` ### In +SQL IN looks like this. + +```scala 3 +import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} +import dataprism.jdbc.sql.PostgresJdbcTypes + +val a: DbValue[Boolean] = ??? +val b: DbValue[Boolean] = ??? +val c: DbValue[Boolean] = ??? + +val in1 = a.in(b, c) +val in2 = a.notIn(b, c) +val in3 = a.in(Query.of(b)) +val in4 = a.notIn(Query.of(b)) +val in5 = a.inAs(Seq(true, false), PostgresJdbcTypes.boolean) +val in6 = a.notInAs(Seq(true, false), PostgresJdbcTypes.boolean) +``` ### Case +DataPrism supports case both as many if checks, and as a pattern match like operator. + +```scala 3 +import dataprism.jdbc.platform.PostgresJdbcPlatform.Api.{*, given} + +val v: DbValue[Double] = ??? +val w1: DbValue[Double] = ??? +val w2: DbValue[Double] = ??? + +val t1: DbValue[Int] = ??? +val t2: DbValue[Int] = ??? +val t3: DbValue[Int] = ??? + +Case(v) + .when(w1)(t1) + .when(w2)(t2) + .otherwise(t3) + +Case + .when(v === w1)(t1) + .when(v === w2)(t2) + .otherwise(t3) +``` + +### Custom SQL + +Custom SQL can be created using the functions `DbValue.raw` and `DbValue.rawK`. ### Custom SQL functions -### Custom SQL strings +When a custom SQL function is needed, there exists helpers called `DbValue.function` and `DbValue.functionK` to help +create these functions. + diff --git a/docs/_docs/guide/10_fs2_effects.md b/docs/_docs/guide/10_fs2_effects.md index 5cdf9bed..bd6ba438 100644 --- a/docs/_docs/guide/10_fs2_effects.md +++ b/docs/_docs/guide/10_fs2_effects.md @@ -17,8 +17,8 @@ object MyPlatform extends PostgresJdbcPlatform, Fs2SqlPlatform { override type Api = PostgresApi & Fs2Api object Api extends PostgresApi, Fs2Api - override type Impl = DefaultCompleteImpl - object Impl extends DefaultCompleteImpl + override type Impl = DefaultCompleteImpl & SqlArraysImpl + object Impl extends DefaultCompleteImpl, SqlArraysImpl } ```