From 744859cf70f04373c53ed197ee8aab9049b0cf33 Mon Sep 17 00:00:00 2001 From: gs-jp1 <80327721+gs-jp1@users.noreply.github.com> Date: Thu, 28 Sep 2023 15:42:55 +0100 Subject: [PATCH] resolveSchema - support calls to Tabular functions (#2317) --- .../resources/core/pure/tds/tdsSchema.pure | 24 ++- .../core/pure/tds/testTdsSchema.pure | 186 ++++++++++-------- 2 files changed, 118 insertions(+), 92 deletions(-) diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure index 0bd43d676fa..cff7447b91a 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tdsSchema.pure @@ -168,7 +168,7 @@ function meta::pure::tds::schema::resolveSchemaImpl(vs : ValueSpecification[1], iv:InstanceValue[1]|$iv.values->match([ t:TabularDataSet[1]|createSchemaState($t.columns); ]), - v:VariableExpression[1]|$openVars->get($v.name).values->evaluateAndDeactivate()->match([ + v:VariableExpression[1]| $openVars->get($v.name).values->evaluateAndDeactivate()->match([ t:TabularDataSet[1]|createSchemaState($t.columns), vs2:ValueSpecification[1]|resolveSchemaImpl($vs2, $openVars, $extensions) ]), @@ -176,12 +176,17 @@ function meta::pure::tds::schema::resolveSchemaImpl(vs : ValueSpecification[1], ]); } -function meta::pure::tds::schema::resolveSchemaImpl(f : FunctionDefinition[1], extensions:meta::pure::extension::Extension[*]) : SchemaState[1] +function meta::pure::tds::schema::resolveSchemaImpl(f : FunctionDefinition[1], vars:Map>[1], extensions:meta::pure::extension::Extension[*]) : SchemaState[1] { let es = $f->evaluateAndDeactivate().expressionSequence->last(); assertEquals(1, $es->size()); - resolveSchemaImpl($es->toOne(), $f->openVariableValues(), $extensions); + resolveSchemaImpl($es->toOne(), $f->openVariableValues()->putAll($vars), $extensions); +} + +function meta::pure::tds::schema::resolveSchemaImpl(f : FunctionDefinition[1], extensions:meta::pure::extension::Extension[*]) : SchemaState[1] +{ + resolveSchemaImpl($f, ^Map>(), $extensions); } function <> meta::pure::tds::schema::resolveSchemaImpl(fe : FunctionExpression[1], openVars : Map>[1], extensions:meta::pure::extension::Extension[*]) : SchemaState[1] @@ -224,7 +229,7 @@ function <> meta::pure::tds::schema::resolveSchemaImpl(fe : Func ^TDSColumn(name = $p.second, type = $p.first->functionReturnType().rawType->toOne()->cast(@DataType)) ); createSchemaState($cols); - }), + }), pair(restrict_TabularDataSet_1__String_MANY__TabularDataSet_1_->cast(@Function), {| let tdsSchema = resolveSchemaImpl($fe.parametersValues->at(0), $openVars, $extensions); let colNamesToRestrict = $fe.parametersValues->at(1)->reactivate($openVars)->cast(@String)->toOneMany(); @@ -310,18 +315,23 @@ function <> meta::pure::tds::schema::resolveSchemaImpl(fe : Func ]->map(f| pair($f->cast(@Function), {| $fe.parametersValues->at(0)->reactivate()->cast(@FunctionDefinition).expressionSequence->evaluateAndDeactivate()->last()->toOne()->resolveSchemaImpl($openVars, $extensions); - }) + }) )); let handlerMatches = $handlers->filter(p|$p.first == $fe.func).second; if ($handlerMatches->isEmpty() && $fe.func->functionReturnType().rawType == TabularDataSet, | //this allows us to handle calls to custom functions that return TDS. - $fe.func->cast(@FunctionDefinition)->resolveSchemaImpl($extensions), + let zip = $fe.func.classifierGenericType.typeArguments.rawType->cast(@FunctionType).parameters->evaluateAndDeactivate()->zip($fe.parametersValues); + let vars = $zip->map(p | + pair($p.first.name, list($p.second)); + )->newMap(); + + $fe.func->cast(@FunctionDefinition)->resolveSchemaImpl($vars, $extensions);, | assertNotEmpty($handlerMatches, 'Failed to find handler for function ' + $fe.func->elementToPath()); assertEquals(1, $handlerMatches->size()); $handlerMatches->toOne()->eval(); - ); + ); } function meta::pure::tds::schema::resolveProject(colSpecs : ColumnSpecification[*], openVars : Map>[1]) : SchemaState[1] diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/testTdsSchema.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/testTdsSchema.pure index 0c193fa5a8e..d72eaa86a4b 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/testTdsSchema.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/testTdsSchema.pure @@ -28,6 +28,13 @@ function <> meta::pure::tds::schema::tests::tdsFunc(date:Date[1] ]) } +function <> meta::pure::tds::schema::tests::addFullName(objects: meta::pure::tds::TabularDataSet[1]): meta::pure::tds::TabularDataSet[1] +{ + $objects->extend( + col(r: TDSRow[1]|$r.getString('firstName') + ' ' + $r.getString('lastName'), 'fullName') + ) +} + function <> meta::pure::tds::schema::tests::resolveSchemaTest() : Boolean[1] { assertSchemaRoundTripEquality({| @@ -35,10 +42,19 @@ function <> meta::pure::tds::schema::tests::resolveSchemaTest() : Bo ->project( col(p|$p.firstName,'firstName') ) - }); + }); assertSchemaRoundTripEquality({| meta::pure::tds::schema::tests::tdsFunc(%2023-01-01)}); + assertSchemaRoundTripEquality({| + Person.all() + ->project([ + col(p|$p.firstName,'firstName') , + col(p|$p.lastName,'lastName') + ]) + ->meta::pure::tds::schema::tests::addFullName() + }); + assertSchemaRoundTripEquality( [ ^TDSColumn(name='firstName', offset = 0, type = String), @@ -53,8 +69,8 @@ function <> meta::pure::tds::schema::tests::resolveSchemaTest() : Bo col(p|$const, 'const'), col(p|$p.firstName,'firstName') ]); - }); - + }); + assertSchemaRoundTripEquality({| Person.all() ->project([ @@ -62,7 +78,7 @@ function <> meta::pure::tds::schema::tests::resolveSchemaTest() : Bo col(p|$p.lastName,'lastName') ]) }); - + assertSchemaRoundTripEquality({| Person.all() ->project([ @@ -70,10 +86,10 @@ function <> meta::pure::tds::schema::tests::resolveSchemaTest() : Bo col(p|$p.lastName,'lastName') ]) ->meta::pure::tds::groupBy(['firstName'], [agg('count',r|$r.getString('firstName'), c|$c->size())]) - }); - - assertSchemaRoundTripEquality({| - Person.all()->projectWithColumnSubset([ + }); + + assertSchemaRoundTripEquality({| + Person.all()->projectWithColumnSubset([ col(x|$x.firstName, 'first'), col(x|$x.lastName, 'last'), col(x|$x.age, 'age'), @@ -90,7 +106,7 @@ function <> meta::pure::tds::schema::tests::resolveSchemaTest() : Bo ->limit(10) ->filter(row|$row.getString('firstName') != 'hello') }); - + assertSchemaRoundTripEquality({| Person.all() ->project([ @@ -101,29 +117,29 @@ function <> meta::pure::tds::schema::tests::resolveSchemaTest() : Bo ]) ->limit(10) ->filter(row|$row.getString('firstName') != 'hello') - }); - + }); + assertSchemaRoundTripEquality({| Firm.all()->project([f|$f.employees.age->average()], ['a']) - }); - + }); + assertSchemaRoundTripEquality({| Firm.all()->project([ f|$f.employees.lastName->count(), f|$f.employees.age->average() ], ['b','a']) - }); - - assertSchemaRoundTripEquality({| + }); + + assertSchemaRoundTripEquality({| Firm.all()->project([ col(f|$f.legalName,'a'), col(f|$f.employees->map(e|$e.lastName),'b'), col(f|$f.employees->map(e|2 + $e.locations.place->count()),'c') ]) }); - - assertSchemaRoundTripEquality({| + + assertSchemaRoundTripEquality({| Firm.all()->project([ col(f|$f.employees->map(e|2 + $e.locations.place->count()),'c') ]) @@ -131,51 +147,51 @@ function <> meta::pure::tds::schema::tests::resolveSchemaTest() : Bo ->restrict('c2') ->extend(col(row:TDSRow[1]| $row.getString('c2'), 'c')) ->restrict('c') - }); + }); - assertSchemaRoundTripEquality({| + assertSchemaRoundTripEquality({| Person.all() ->project(col(p|$p.age, 'age')) ->sort(asc('age')) ->limit(3) - ->filter(r|$r.getInteger('age') > 25) - }); - - assertSchemaRoundTripEquality({| + ->filter(r|$r.getInteger('age') > 25) + }); + + assertSchemaRoundTripEquality({| Person.all() ->project(col(p|$p.age->first(), 'age')) ->sort(asc('age')) ->limit(3) - ->filter(r|$r.getInteger('age')->toOne() > 25) - }); - - assertSchemaRoundTripEquality({| + ->filter(r|$r.getInteger('age')->toOne() > 25) + }); + + assertSchemaRoundTripEquality({| Person.all() ->project(col(p|$p.age, 'age')) ->sort(asc('age')) ->limit(3) - ->filter(r|$r.getInteger('age') > 25) + ->filter(r|$r.getInteger('age') > 25) }); - + assertSchemaRoundTripEquality({| Address.all() ->project([a|$a.name, a|$a.type],['name', 'type']) ->extend(col(row:TDSRow[1]| DurationUnit.DAYS, 'typeduplicated')) }); - + assertSchemaRoundTripEquality({| Address.all() ->project([a|$a.name, a|$a.type],['name', 'type']) ->extend(^BasicColumnSpecification(name='typeduplicated', func={row:TDSRow[1]|DurationUnit.DAYS})) }); - + assertSchemaRoundTripEquality({| Address.all() ->project([a|$a.name, a|$a.type],['name', 'type']) ->extend(^BasicColumnSpecification(name='typeduplicated', func={row:TDSRow[1]|DurationUnit.DAYS})) ->restrict(['typeduplicated', 'name']) }); - + assertSchemaRoundTripEquality({| Person.all() ->project([ @@ -189,105 +205,105 @@ function <> meta::pure::tds::schema::tests::resolveSchemaTest() : Bo assertSchemaRoundTripEquality( [ - ^TDSColumn(name = 'constString', offset= 0, type = String), + ^TDSColumn(name = 'constString', offset= 0, type = String), ^TDSColumn(name = 'constDate', offset= 1, type = StrictDate) - ], - {| + ], + {| Person.all() ->project([ col(p|'hello','constString'), col(p|%2018-12-12,'constDate') ]) ->distinct() - }); + }); assertSchemaRoundTripEquality({| Trade.all() - ->groupBy([x|$x.date->adjust(0, DurationUnit.DAYS)], - [ agg(x | $x.quantity, y | $y->sum()), agg(x | $x.quantity, y | $y->sum())], + ->groupBy([x|$x.date->adjust(0, DurationUnit.DAYS)], + [ agg(x | $x.quantity, y | $y->sum()), agg(x | $x.quantity, y | $y->sum())], ['tradeDate', 'quantityA', 'quantityB'] - ) - }); + ) + }); - assertSchemaRoundTripEquality({| + assertSchemaRoundTripEquality({| Person.all() ->project([ col(p|$p.age->first(), 'age1'), col(p|$p.age->first(), 'age2') ]) ->renameColumns(pair('age1', 'ageOne')) - }); + }); assertSchemaRoundTripEquality( [ - ^TDSColumn(name = 'constString', offset= 0, type = String), - ^TDSColumn(name = 'constDate', offset= 1, type = StrictDate), + ^TDSColumn(name = 'constString', offset= 0, type = String), + ^TDSColumn(name = 'constDate', offset= 1, type = StrictDate), ^TDSColumn(name = 'rowNumber', offset= 2, type = Integer) - ], - {| + ], + {| Person.all() ->project([ col(p|'hello','constString'), col(p|%2018-12-12,'constDate') ]) ->olapGroupBy('constString',asc('constString'),y|$y->meta::pure::functions::math::olap::rowNumber(),'rowNumber') - }); + }); let t = ^TabularDataSet(rows = [], columns = ^TDSColumn(name = 'aCol', offset= 0, type = String)); assertSchemaRoundTripEquality( [ ^TDSColumn(name = 'aCol', offset= 0, type = String) - ], - {| + ], + {| $t - }); + }); - assertSchemaRoundTripEquality({| + assertSchemaRoundTripEquality({| {| Person.all() ->project([col(p|$p.age->first(), 'age1')]) - }->eval()}); + }->eval()}); - assertSchemaRoundTripEquality({| + assertSchemaRoundTripEquality({| {a:String[1] | Person.all() ->project([col(p|$p.age->first(), 'age1')]) - }->eval('a')}); - - assertSchemaRoundTripEquality({| + }->eval('a')}); + + assertSchemaRoundTripEquality({| {a:String[1], b:String[1] | Person.all() ->project([col(p|$p.age->first(), 'age1')]) - }->eval('a', 'b')}); + }->eval('a', 'b')}); - assertSchemaRoundTripEquality({| + assertSchemaRoundTripEquality({| {a:String[1], b:String[1], c:String[1] | Person.all() ->project([col(p|$p.age->first(), 'age1')]) - }->eval('a', 'b', 'c')}); + }->eval('a', 'b', 'c')}); - assertSchemaRoundTripEquality({| + assertSchemaRoundTripEquality({| {a:String[1], b:String[1], c:String[1], d:String[1] | Person.all() ->project([col(p|$p.age->first(), 'age1')]) - }->eval('a', 'b', 'c', 'd')}); + }->eval('a', 'b', 'c', 'd')}); - assertSchemaRoundTripEquality({| + assertSchemaRoundTripEquality({| {a:String[1], b:String[1], c:String[1], d:String[1], e:String[1] | Person.all() ->project([col(p|$p.age->first(), 'age1')]) - }->eval('a', 'b', 'c', 'd', 'e')}); - - assertSchemaRoundTripEquality({| + }->eval('a', 'b', 'c', 'd', 'e')}); + + assertSchemaRoundTripEquality({| {a:String[1], b:String[1], c:String[1], d:String[1], e:String[1], f:String[1] | Person.all() ->project([col(p|$p.age->first(), 'age1')]) - }->eval('a', 'b', 'c', 'd', 'e', 'f')}); + }->eval('a', 'b', 'c', 'd', 'e', 'f')}); - assertSchemaRoundTripEquality({| + assertSchemaRoundTripEquality({| {a:String[1], b:String[1], c:String[1], d:String[1], e:String[1], f:String[1], g:String[1] | Person.all() ->project([col(p|$p.age->first(), 'age1')]) - }->eval('a', 'b', 'c', 'd', 'e', 'f', 'g')}); + }->eval('a', 'b', 'c', 'd', 'e', 'f', 'g')}); assertSchemaRoundTripEquality( ^TDSColumn(name='age1', offset = 0, type = Integer), {a:String[1], b:String[1], c:String[1], d:String[1], e:String[1], f:String[1], g:String[1] | Person.all() ->project([col(p|$p.age->first(), 'age1')]) - }); + }); } function meta::pure::tds::schema::tests::assertSchemaRoundTripEquality(query : FunctionDefinition<{->TabularDataSet[1]}>[1]) : Boolean[1] @@ -300,39 +316,39 @@ function meta::pure::tds::schema::tests::assertSchemaRoundTripEquality(query : F let expected = $query->eval().columns;//->map(c|if($c.offset->isEmpty(), |^$c(offset = 0), | $c)); assertSchemaRoundTripEquality($expected, $query, $extensions); } - + function meta::pure::tds::schema::tests::assertSchemaRoundTripEquality(expected:TDSColumn[*], query : FunctionDefinition[1]) : Boolean[1] -{ +{ assertSchemaRoundTripEquality($expected, $query, defaultExtensions()); -} +} function meta::pure::tds::schema::tests::assertSchemaRoundTripEquality(expected:TDSColumn[*], query : FunctionDefinition[1], extensions:Extension[*]) : Boolean[1] { let actual = $query->meta::pure::tds::schema::resolveSchema($extensions); - - assertSchemaEquality($expected, $actual); -} + + assertSchemaEquality($expected, $actual); +} function meta::pure::tds::schema::tests::assertSchemaEquality(expected:TDSColumn[*], actual : TDSColumn[*]) : Boolean[1] -{ +{ let areEqual = ($expected->size() == $actual->size()) && ($expected->zip($actual)->forAll(p| - + let theProperties = TDSColumn->getAllTypeGeneralisations()->remove(Any)->map(t|$t->match([c:Class[1]|$c, a:Any[*]|[]])).properties->removeAll(Class.properties); - + $theProperties->forAll(property| $property->eval($p.first) == $property->eval($p.second); ); - )); - - if ($areEqual, - | $areEqual, - | + )); + + if ($areEqual, + | $areEqual, + | println('expected:'); println($expected->map(c|$c->simpleToString())->joinStrings('[', ',', ']')); println('actual:'); println($actual->map(c|$c->simpleToString())->joinStrings('[', ',', ']')); - + assertEquals($expected, $actual); ); } \ No newline at end of file