From b8a5bfa7080944be2dae5ee1ae83237d3f7d2dd3 Mon Sep 17 00:00:00 2001 From: Prateek Garg Date: Thu, 12 Sep 2024 09:06:19 -0400 Subject: [PATCH] Enhance duckDB Extension and add tests -fixup varbinary dataype translation for duck db --- ...ational_DuckDB_EssentialFunctions_PCT.java | 25 ++----- .../sqlQueryToString/duckdbExtension.pure | 72 ++++++++++++++++--- .../testSuite/dynaFunctions/numeric.pure | 7 ++ .../testSuite/dynaFunctions/string.pure | 1 + .../selectSubClauses/aggregationDynaFns.pure | 57 ++++++++++++++- .../sqlQueryToString/testSuite/testDDL.pure | 59 +++++++++++++++ 6 files changed, 192 insertions(+), 29 deletions(-) create mode 100644 legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/testDDL.pure diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_EssentialFunctions_PCT.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_EssentialFunctions_PCT.java index ea9f80a5f33..c9d693a92dc 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_EssentialFunctions_PCT.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-PCT/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_EssentialFunctions_PCT.java @@ -91,7 +91,7 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf one("meta::pure::functions::collection::tests::forall::testforAllOnNonEmptySetIsTrue_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'forAll_T_MANY__Function_1__Boolean_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), // Indexof - one("meta::pure::functions::collection::tests::indexof::testIndexOfOneElement_Function_1__Boolean_1_", "\"[unsupported-api] The function 'indexOf' (state: [Select, false]) is not supported yet\""), + one("meta::pure::functions::collection::tests::indexof::testIndexOfOneElement_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: 0\nactual: 1\""), one("meta::pure::functions::collection::tests::indexof::testIndexOf_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'indexOf_T_MANY__T_1__Integer_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), // Init @@ -180,7 +180,7 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf // JoinStrings one("meta::pure::functions::string::tests::joinStrings::testJoinStringsNoStrings_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'tail_T_MANY__T_MANY_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), - one("meta::pure::functions::string::tests::joinStrings::testJoinStringsSingleString_Function_1__Boolean_1_", "\"The database type 'DuckDB' is not supported yet!\""), + one("meta::pure::functions::string::tests::joinStrings::testJoinStringsSingleString_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: '[a]'\nactual: '['\""), one("meta::pure::functions::string::tests::joinStrings::testJoinStringsUsingGenericArrow_Function_1__Boolean_1_", "\"\nexpected: '[a,b,c]'\nactual: '[,a,b,c,]'\""), one("meta::pure::functions::string::tests::joinStrings::testJoinStrings_Function_1__Boolean_1_", "\"\nexpected: '[a,b,c]'\nactual: '[,a,b,c,]'\""), @@ -189,8 +189,8 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf one("meta::pure::functions::string::tests::split::testSplit_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'split_String_1__String_1__String_MANY_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), // SubString - one("meta::pure::functions::string::tests::substring::testStartEnd_Function_1__Boolean_1_", "\"[unsupported-api] The function 'substring' (state: [Select, false]) is not supported yet\""), - one("meta::pure::functions::string::tests::substring::testStart_Function_1__Boolean_1_", "\"[unsupported-api] The function 'substring' (state: [Select, false]) is not supported yet\""), + one("meta::pure::functions::string::tests::substring::testStartEnd_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: 'the quick brown fox jumps over the lazy dog'\nactual: 'the quick brown fox jumps over the lazy do'\""), + one("meta::pure::functions::string::tests::substring::testStart_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: 'he quick brown fox jumps over the lazy dog'\nactual: 'the quick brown fox jumps over the lazy dog'\""), // ToString one("meta::pure::functions::string::tests::toString::testClassToString_Function_1__Boolean_1_", "\"Cannot cast a collection of size 0 to multiplicity [1]\""), @@ -215,14 +215,7 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf one("meta::pure::functions::math::tests::round::testNegativeFloatRoundHalfEvenUp_Function_1__Boolean_1_", "\"\nexpected: -16\nactual: -17\""), // toDecimal - one("meta::pure::functions::math::tests::toDecimal::testDecimalToDecimal_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toDecimal' (state: [Select, false]) is not supported yet\""), - one("meta::pure::functions::math::tests::toDecimal::testDoubleToDecimal_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toDecimal' (state: [Select, false]) is not supported yet\""), - one("meta::pure::functions::math::tests::toDecimal::testIntToDecimal_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toDecimal' (state: [Select, false]) is not supported yet\""), - - // toFloat - one("meta::pure::functions::math::tests::toFloat::testDecimalToFloat_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toFloat' (state: [Select, false]) is not supported yet\""), - one("meta::pure::functions::math::tests::toFloat::testDoubleToFloat_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toFloat' (state: [Select, false]) is not supported yet\""), - one("meta::pure::functions::math::tests::toFloat::testIntToFloat_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toFloat' (state: [Select, false]) is not supported yet\""), + one("meta::pure::functions::math::tests::toDecimal::testIntToDecimal_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: 8D\nactual: 8.0D\""), // Is one("meta::pure::functions::boolean::tests::testIsEnum_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'is_Any_1__Any_1__Boolean_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), @@ -297,7 +290,7 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf // IndexOf one("meta::pure::functions::string::tests::indexOf::testFromIndex_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'indexOf_String_1__String_1__Integer_1__Integer_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""), - one("meta::pure::functions::string::tests::indexOf::testSimple_Function_1__Boolean_1_", "\"[unsupported-api] The function 'indexOf' (state: [Select, false]) is not supported yet\""), + one("meta::pure::functions::string::tests::indexOf::testSimple_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: 4\nactual: 5\""), // ParseBoolean one("meta::pure::functions::string::tests::parseBoolean::testParseFalse_Function_1__Boolean_1_", "\"[unsupported-api] The function 'parseBoolean' (state: [Select, false]) is not supported yet\""), @@ -314,11 +307,7 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf one("meta::pure::functions::string::tests::parseDecimal::testParseZero_Function_1__Boolean_1_", "\"\nexpected: 0.000D\nactual: 0.0D\""), // ParseInteger - one("meta::pure::functions::string::tests::parseInteger::testParseInteger_Function_1__Boolean_1_", "java.sql.SQLException: Conversion Error: Could not convert string '9999999999999992' to INT32\nLINE 1: select cast('9999999999999992' as integer)\n ^"), - - // ReverseString - one("meta::pure::functions::string::tests::reverse::testReverseString_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Catalog Error: Scalar Function with name legend_h2_extension_reverse_string does not exist!\nDid you mean \"list_reverse_sort\"?\nLINE 1: select legend_h2_extension_reverse_string('')\n ^") - + one("meta::pure::functions::string::tests::parseInteger::testParseInteger_Function_1__Boolean_1_", "java.sql.SQLException: Conversion Error: Could not convert string '9999999999999992' to INT32\nLINE 1: select cast('9999999999999992' as integer)\n ^") ); public static Test suite() diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure index 1cb4b916fd6..ece73fe8b93 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-pure/src/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure @@ -26,7 +26,7 @@ function <> meta::relational::functions::sqlQueryToString::duckD isBooleanLiteralSupported = true, isDbReservedIdentifier = {str:String[1]| $str->toLower()->in($reservedWords); }, // check case insensitive windowColumnProcessor = processWindowColumn_WindowColumn_1__SqlGenerationContext_1__String_1_, - joinStringsProcessor = processJoinStringsOperationWithConcatCall_JoinStrings_1__SqlGenerationContext_1__String_1_, + joinStringsProcessor = processJoinStringsOperationForDuckDB_JoinStrings_1__SqlGenerationContext_1__String_1_, literalProcessor = $literalProcessor, selectSQLQueryProcessor = processSelectSQLQueryForDuckDB_SelectSQLQuery_1__SqlGenerationContext_1__Boolean_1__String_1_, selectSQLQueryWithCTEsProcessor = processSelectSQLQueryWithCTEsDefault_SelectSQLQueryWithCommonTableExpressions_1__SqlGenerationContext_1__Boolean_1__String_1_, @@ -42,7 +42,7 @@ function <> meta::relational::functions::sqlQueryToString::duckD ^RelationalDDLCommandsTranslator( createSchema = translateCreateSchemaStatementDefault_CreateSchemaSQL_1__DbConfig_1__String_1_, dropSchema = translateDropSchemaStatementDefault_DropSchemaSQL_1__DbConfig_1__String_1_, - createTable = translateCreateTableStatementDefault_CreateTableSQL_1__DbConfig_1__String_1_, + createTable = translateCreateTableStatementDuckDB_CreateTableSQL_1__DbConfig_1__String_1_, dropTable = translateDropTableStatementDefault_DropTableSQL_1__DbConfig_1__String_1_, loadTable = loadValuesToDbTableDefault_LoadTableSQL_1__DbConfig_1__String_MANY_ ); @@ -78,14 +78,60 @@ function meta::relational::functions::sqlQueryToString::duckDB::convertDateToSql ) ); } + +function meta::relational::functions::sqlQueryToString::duckDB::translateCreateTableStatementDuckDB(createTableSQL:CreateTableSQL[1], dbConfig:DbConfig[1]) : String[1] +{ + let t= $createTableSQL.table; + let applyConstraints = $createTableSQL.applyConstraints; + 'Create Table '+if($t.schema.name == 'default',|'',|$t.schema.name+'.')+$t.name+ + + '(' + + $t.columns->cast(@meta::relational::metamodel::Column) + ->map(c | $c.name->processColumnName($dbConfig) + ' ' + dataTypeToSqlTextDuckDB($c.type) + if($c.nullable->isEmpty() || $applyConstraints == false, | '', | if($c.nullable == true , | ' NULL', | ' NOT NULL' ))) + ->joinStrings(',') + + if ($t.primaryKey->isEmpty() || $applyConstraints == false, | '', | ', PRIMARY KEY(' + $t.primaryKey->map(c | $c.name)->joinStrings(',') + ')') + +');'; +} + + +function meta::relational::functions::sqlQueryToString::duckDB::dataTypeToSqlTextDuckDB(type: meta::relational::metamodel::datatype::DataType[1]):String[1] +{ + $type->match([ + s : meta::relational::metamodel::datatype::SemiStructured[1] | 'VARCHAR(4000)', + i : meta::relational::metamodel::datatype::Integer[1] | 'INT', + f : meta::relational::metamodel::datatype::Float[1] | 'FLOAT', + v : meta::relational::metamodel::datatype::Varchar[1] | format('VARCHAR(%d)', $v.size), + c : meta::relational::metamodel::datatype::Char[1] | format('CHAR(%d)', $c.size), + d : meta::relational::metamodel::datatype::Decimal[1] | format('DECIMAL(%d, %d)', [$d.precision, $d.scale]), + t : meta::relational::metamodel::datatype::Timestamp[1] | 'TIMESTAMP', + d : meta::relational::metamodel::datatype::Date[1] | 'DATE', + b : meta::relational::metamodel::datatype::BigInt[1] | 'BIGINT', + s : meta::relational::metamodel::datatype::SmallInt[1] | 'SMALLINT', + t : meta::relational::metamodel::datatype::TinyInt[1] | 'TINYINT', + d : meta::relational::metamodel::datatype::Double[1] | 'DOUBLE', + n : meta::relational::metamodel::datatype::Numeric[1] | format('NUMERIC(%d, %d)', [$n.precision, $n.scale]), + d : meta::relational::metamodel::datatype::Distinct[1] | 'DISTINCT', + o : meta::relational::metamodel::datatype::Other[1] | 'OTHER', + b : meta::relational::metamodel::datatype::Bit[1] | 'BIT', + b : meta::relational::metamodel::datatype::Binary[1] | 'BINARY', + r : meta::relational::metamodel::datatype::Real[1] | 'REAL', + a : meta::relational::metamodel::datatype::Array[1] | 'ARRAY', + v : meta::relational::metamodel::datatype::Varbinary[1] | 'VARBINARY', + s : meta::relational::metamodel::datatype::SemiStructured[1] | 'SEMISTRUCTURED', + j : meta::relational::metamodel::datatype::Json[1] | 'JSON', + d : meta::relational::metamodel::datatype::DbSpecificDataType[1] | $d.dbSpecificSql + ]); +} + + + function <> meta::relational::functions::sqlQueryToString::duckDB::getDynaFunctionToSqlForDuckDB(): DynaFunctionToSql[*] { let allStates = allGenerationStates(); [ dynaFnToSql('adjust', $allStates, ^ToSql(format='date_add(%s)', transform={p:String[3] | $p->at(0) + ',' + constructIntervalFunction($p->at(2), $p->at(1)) })), - dynaFnToSql('booland', $allStates, ^ToSql(format='every(%s)')), - dynaFnToSql('boolor', $allStates, ^ToSql(format='any(%s)')), + dynaFnToSql('booland', $allStates, ^ToSql(format='bool_and(%s)')), //aggregate function across rows (not and on conditions) + dynaFnToSql('boolor', $allStates, ^ToSql(format='bool_or(%s)')), dynaFnToSql('castBoolean', $allStates, ^ToSql(format='cast(%s as boolean)')), dynaFnToSql('chr', $allStates, ^ToSql(format='char(%s)')), dynaFnToSql('concat', $allStates, ^ToSql(format='concat%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})), @@ -114,7 +160,7 @@ function <> meta::relational::functions::sqlQueryToString::duckD dynaFnToSql('firstMinuteOfHour', $allStates, ^ToSql(format='date_trunc(\'hour\', %s)', transform={p:String[1] | $p->repeat(1)})), dynaFnToSql('firstSecondOfMinute', $allStates, ^ToSql(format='date_trunc(\'minute\', %s)', transform={p:String[1] | $p->repeat(1)})), dynaFnToSql('hour', $allStates, ^ToSql(format='hour(%s)')), -// dynaFnToSql('indexOf', $allStates, ^ToSql(format='position(%s IN %s)', transform={p:String[2] | [$p->at(1), $p->at(0)]})), + dynaFnToSql('indexOf', $allStates, ^ToSql(format='instr(%s, %s)', transform={p:String[2] | [$p->at(0), $p->at(1)]})), // TODO - pure uses 0-based indexing, duck db returns location with 1-based index , keeping this as H2 also returns 1-based currently, many user tests need to be fixed dynaFnToSql('isNumeric', $allStates, ^ToSql(format='(lower(%s) = upper(%s))')), dynaFnToSql('isAlphaNumeric', $allStates, ^ToSql(format='regexp_matches(%s,\'^[a-zA-Z0-9]*$\')', transform={p:String[1]|$p})), dynaFnToSql('joinStrings', $allStates, ^ToSql(format='string_agg(%s,%s)')), @@ -139,19 +185,19 @@ function <> meta::relational::functions::sqlQueryToString::duckD // dynaFnToSql('previousDayOfWeek', $allStates, ^ToSql(format='date_add(DAY, case when %s - DAY_OF_WEEK(%s) >= 0 then %s - DAY_OF_WEEK(%s) - 7 else %s - DAY_OF_WEEK(%s) end, %s)', transform={p:String[1..2] | $p->formatMostRecentH2('current_date()')}, parametersWithinWhenClause = [false, false])), dynaFnToSql('quarter', $allStates, ^ToSql(format='quarter(%s)')), dynaFnToSql('quarterNumber', $allStates, ^ToSql(format='quarter(%s)')), - dynaFnToSql('reverseString', $allStates, ^ToSql(format='legend_h2_extension_reverse_string(%s)')), -// dynaFnToSql('round', $allStates, ^ToSql(format='round(%s, %s)', transform=transformRound_String_MANY__String_MANY_)), + dynaFnToSql('reverseString', $allStates, ^ToSql(format='reverse(%s)')), + dynaFnToSql('round', $allStates, ^ToSql(format='round(%s, %s)', transform=transformRound_String_MANY__String_MANY_)), dynaFnToSql('rpad', $allStates, ^ToSql(format='rpad(%s,%s,%s)', transform=processPaddingParams_String_MANY__String_MANY_)), dynaFnToSql('second', $allStates, ^ToSql(format='second(%s)')), dynaFnToSql('sha1', $allStates, ^ToSql(format='sha1(%s)')), dynaFnToSql('sha256', $allStates, ^ToSql(format='sha256(%s)')), dynaFnToSql('splitPart', $allStates, ^ToSql(format='split_part(%s, %s, %s)')), -// dynaFnToSql('substring', $allStates, ^ToSql(format='substring%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})), + dynaFnToSql('substring', $allStates, ^ToSql(format='substring%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})), // TODO - pure uses 0-based indexing, duck db returns location with 1-based index , keeping this as H2 also returns 1-based currently, many user tests need to be fixed dynaFnToSql('stdDevPopulation', $allStates, ^ToSql(format='stddev_pop(%s)')), dynaFnToSql('stdDevSample', $allStates, ^ToSql(format='stddev_samp(%s)')), dynaFnToSql('today', $allStates, ^ToSql(format='cast(today() as timestamp_s)')), -// dynaFnToSql('toDecimal', $allStates, ^ToSql(format='cast(%s as decimal)')), -// dynaFnToSql('toFloat', $allStates, ^ToSql(format='cast(%s as double precision)')), + dynaFnToSql('toDecimal', $allStates, ^ToSql(format='cast(%s as decimal)')), + dynaFnToSql('toFloat', $allStates, ^ToSql(format='cast(%s as double)')), dynaFnToSql('toString', $allStates, ^ToSql(format='cast(%s as varchar)')), // dynaFnToSql('toTimestamp', $allStates, ^ToSql(format='%s', transform={p:String[2] | $p->transformToTimestampH2()})), dynaFnToSql('weekOfYear', $allStates, ^ToSql(format='week(%s)')), @@ -274,3 +320,9 @@ function <> meta::relational::functions::sqlQueryToString::duckD 'WHERE', 'WINDOW', 'WITH' ]; } + +function meta::relational::functions::sqlQueryToString::duckDB::processJoinStringsOperationForDuckDB(js:JoinStrings[1], sgc:SqlGenerationContext[1]): String[1] +{ + processJoinStringsOperation($js, $sgc, {col, sep| 'group_concat(' + $col + if($sep == '\'\'', |'', |',' + $sep) + ' )'}, + {strs, sep| $strs->joinStrings('concat(', if('\'\'' == $sep, |', ', |',' + $sep + ',') , ')')}); +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/numeric.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/numeric.pure index 0fd66df178f..85d70fb5455 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/numeric.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/numeric.pure @@ -313,6 +313,13 @@ function <> meta::relational::tests::dbSpecificTests::sqlQueryTe runDynaFunctionDatabaseTest($dynaFunc, $expected, $config); } +function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::dynaFunctions::round::testDecimalToPrecision(config:DbTestConfig[1]):Boolean[1] +{ + let dynaFunc = ^DynaFunction(name='round', parameters=[^Literal(value=1.1113), ^Literal(value=2)]); + let expected = ^Literal(value=1.11); + runDynaFunctionDatabaseTest($dynaFunc, $expected, $config); +} + function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::dynaFunctions::round::testInt(config:DbTestConfig[1]):Boolean[1] { let dynaFunc = ^DynaFunction(name='round', parameters=[^Literal(value=2.0)]); diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure index 6adebb03cd7..fecc6ac7e8d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure @@ -94,6 +94,7 @@ function <> meta::relational::tests::dbSpecificTests::sqlQueryTe runDynaFunctionDatabaseTest($dynaFunc, $expected, $config); } +//To-fix > pure function indexOf uses 0-based indexing , whereas db function uses 1-based indexing. function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::dynaFunctions::indexOf::testString(config:DbTestConfig[1]):Boolean[1] { let dynaFunc = ^DynaFunction(name='indexOf', parameters=[^Literal(value='String Random'),^Literal(value='and')]); diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/selectSubClauses/aggregationDynaFns.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/selectSubClauses/aggregationDynaFns.pure index 016b40ed99a..cc76e184dc1 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/selectSubClauses/aggregationDynaFns.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/selectSubClauses/aggregationDynaFns.pure @@ -1,7 +1,9 @@ +import meta::relational::tests::*; import meta::relational::dbTestRunner::*; import meta::pure::test::*; import meta::relational::metamodel::*; import meta::relational::metamodel::relation::*; +import meta::relational::tests::model::simple::*; function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::selectSubClauses::aggregationDynaFns::count::testOnAllRows(config:DbTestConfig[1]):Boolean[1] { @@ -167,7 +169,7 @@ function meta::relational::tests::dbSpecificTests::sqlQueryTests::selectSubClaus let dynaFunc = ^DynaFunction(name = $func , parameters=[ ^DynaFunction(name = 'greaterThan', parameters = [ - ^TableAliasColumn(column = $column,alias = ^TableAlias(name = 'myTable', relationalElement = $table)), + ^TableAliasColumn(column = $column,alias = ^TableAlias(name = 'myTable', relationalElement = $table)), ^Literal(value = $value)])]); let sqlQuery = ^SelectSQLQuery(columns=[$dynaFunc], @@ -177,3 +179,56 @@ function meta::relational::tests::dbSpecificTests::sqlQueryTests::selectSubClaus let expected = ^Literal(value=$expectedValue); runSqlQueryTest($sqlQuery, $expected,[], $config); } + + +function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::selectSubClauses::aggregationDynaFns::joinStrings::testJoinStrings(config:DbTestConfig[1]):Boolean[1] +{ + let result = executeViaPlan(|Firm.all()->project([ + f|$f.legalName, + f|$f.employees->sortBy(#/Person/firstName#).firstName->joinStrings() + ], + [ + 'legalName', + 'employeesFirstNamePerFirm' + ]), simpleRelationalMapping, meta::relational::tests::db, $config, meta::relational::extension::relationalExtensions()); + + runDataAssertion($result, $config, + | let tds = $result.values->at(0); + assertEquals([String, String], $result.values.columns.type); + + let resultStrs = $tds.rows->map(r| if($r.values->at(0) == ^TDSNull(), + |'Null', + |$r.values->at(0)->cast(@String)->toOne() + ', ' + $r.values->at(1)->cast(@String)->toOne() + ))->sort(); + assertSameElements(['Firm A, Fabrice', + 'Firm B, Oliver', + 'Firm C, David', + 'Firm X, Anthony,John,John,Peter'], $resultStrs); + ); +} + +function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::selectSubClauses::aggregationDynaFns::joinStrings::testJoinStringsWithSep(config:DbTestConfig[1]):Boolean[1] +{ + let result = executeViaPlan(|Firm.all()->project([ + f|$f.legalName, + f|$f.employees->sortBy(#/Person/firstName#).firstName->joinStrings('|') + ], + [ + 'legalName', + 'employeesFirstNamePerFirm' + ]), simpleRelationalMapping, meta::relational::tests::db, $config, meta::relational::extension::relationalExtensions()); + + runDataAssertion($result, $config, + | let tds = $result.values->at(0); + assertEquals([String, String], $result.values.columns.type); + + let resultStrs = $tds.rows->map(r| if($r.values->at(0) == ^TDSNull(), + |'Null', + |$r.values->at(0)->cast(@String)->toOne() + ', ' + $r.values->at(1)->cast(@String)->toOne() + ))->sort(); + assertSameElements(['Firm A, Fabrice', + 'Firm B, Oliver', + 'Firm C, David', + 'Firm X, Anthony|John|John|Peter'], $resultStrs); + ); +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/testDDL.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/testDDL.pure new file mode 100644 index 00000000000..90757c9efd4 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/testDDL.pure @@ -0,0 +1,59 @@ +// Copyright 2021 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::relational::metamodel::*; +import meta::relational::dbTestRunner::*; +import meta::pure::test::*; + +//operations on booleans + +function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::ddl::testSetup(config:DbTestConfig[1]):Boolean[1] +{ + let records ='default\n'+ + 'personTable\n'+ + 'id, firstName, lastName, age, addressId, firmId, managerId\n'+ + '1,Peter,Smith,23,1,1,2'; + + meta::relational::dbTestRunner::doSetupOnConnection($records, meta::relational::tests::dbSpecificTests::sqlQueryTests::ddl::db, + $config, meta::relational::extension::relationalExtensions()); + true; +} + + +function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::ddl::testCreateTableForAllColumnDataTypes(config:DbTestConfig[1]):Boolean[1] +{ + //insert empty data > ensure create table statement is properly transl;ated for all column data types + meta::relational::dbTestRunner::doSetupOnConnection('', meta::relational::tests::dbSpecificTests::sqlQueryTests::ddl::dataTypeDB, + $config, meta::relational::extension::relationalExtensions()); + true; +} + +###Relational +Database meta::relational::tests::dbSpecificTests::sqlQueryTests::ddl::dataTypeDB +( + Table dataTypTable(i INT,bi BIGINT, si SMALLINT, ti TINYINT, + f Float,d Double, dc Decimal(1,1) , num NUMERIC(1,1), real REAL, + var VARCHAR(10) , char CHAR(10), + date DATE, ts TIMESTAMP, + bit BIT, bin BINARY(100), varbin VARBINARY(100) + // arr ARRAY , dis DISTINCT , other OTHER, + // json JSON , ss SEMISTRUCTURED + ) +) + +###Relational +Database meta::relational::tests::dbSpecificTests::sqlQueryTests::ddl::db +( + Table personTable (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT, MANAGERID INT) +) \ No newline at end of file