Skip to content

Commit

Permalink
Enhance duckDB Extension and add tests -fixup varbinary dataype trans…
Browse files Browse the repository at this point in the history 
…lation for duck db
  • Loading branch information
@PrateekGarg-gs
PrateekGarg-gs committed Sep 27, 2024
1 parent 97e1de8 commit b8a5bfa
Show file tree
Hide file tree
Showing 6 changed files with 192 additions and 29 deletions.
25 changes: 7 additions & 18 deletions ...tion/stores/relational/test/duckdb/pct/Test_Relational_DuckDB_EssentialFunctions_PCT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf
one("meta::pure::functions::collection::tests::forall::testforAllOnNonEmptySetIsTrue_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'forAll_T_MANY__Function_1__Boolean_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""),

// Indexof
one("meta::pure::functions::collection::tests::indexof::testIndexOfOneElement_Function_1__Boolean_1_", "\"[unsupported-api] The function 'indexOf' (state: [Select, false]) is not supported yet\""),
one("meta::pure::functions::collection::tests::indexof::testIndexOfOneElement_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: 0\nactual: 1\""),
one("meta::pure::functions::collection::tests::indexof::testIndexOf_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'indexOf_T_MANY__T_1__Integer_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""),

// Init
Expand Down Expand Up @@ -180,7 +180,7 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf

// JoinStrings
one("meta::pure::functions::string::tests::joinStrings::testJoinStringsNoStrings_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'tail_T_MANY__T_MANY_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""),
one("meta::pure::functions::string::tests::joinStrings::testJoinStringsSingleString_Function_1__Boolean_1_", "\"The database type 'DuckDB' is not supported yet!\""),
one("meta::pure::functions::string::tests::joinStrings::testJoinStringsSingleString_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: '[a]'\nactual: '['\""),
one("meta::pure::functions::string::tests::joinStrings::testJoinStringsUsingGenericArrow_Function_1__Boolean_1_", "\"\nexpected: '[a,b,c]'\nactual: '[,a,b,c,]'\""),
one("meta::pure::functions::string::tests::joinStrings::testJoinStrings_Function_1__Boolean_1_", "\"\nexpected: '[a,b,c]'\nactual: '[,a,b,c,]'\""),

Expand All @@ -189,8 +189,8 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf
one("meta::pure::functions::string::tests::split::testSplit_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'split_String_1__String_1__String_MANY_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""),

// SubString
one("meta::pure::functions::string::tests::substring::testStartEnd_Function_1__Boolean_1_", "\"[unsupported-api] The function 'substring' (state: [Select, false]) is not supported yet\""),
one("meta::pure::functions::string::tests::substring::testStart_Function_1__Boolean_1_", "\"[unsupported-api] The function 'substring' (state: [Select, false]) is not supported yet\""),
one("meta::pure::functions::string::tests::substring::testStartEnd_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: 'the quick brown fox jumps over the lazy dog'\nactual: 'the quick brown fox jumps over the lazy do'\""),
one("meta::pure::functions::string::tests::substring::testStart_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: 'he quick brown fox jumps over the lazy dog'\nactual: 'the quick brown fox jumps over the lazy dog'\""),

// ToString
one("meta::pure::functions::string::tests::toString::testClassToString_Function_1__Boolean_1_", "\"Cannot cast a collection of size 0 to multiplicity [1]\""),
Expand All @@ -215,14 +215,7 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf
one("meta::pure::functions::math::tests::round::testNegativeFloatRoundHalfEvenUp_Function_1__Boolean_1_", "\"\nexpected: -16\nactual: -17\""),

// toDecimal
one("meta::pure::functions::math::tests::toDecimal::testDecimalToDecimal_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toDecimal' (state: [Select, false]) is not supported yet\""),
one("meta::pure::functions::math::tests::toDecimal::testDoubleToDecimal_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toDecimal' (state: [Select, false]) is not supported yet\""),
one("meta::pure::functions::math::tests::toDecimal::testIntToDecimal_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toDecimal' (state: [Select, false]) is not supported yet\""),

// toFloat
one("meta::pure::functions::math::tests::toFloat::testDecimalToFloat_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toFloat' (state: [Select, false]) is not supported yet\""),
one("meta::pure::functions::math::tests::toFloat::testDoubleToFloat_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toFloat' (state: [Select, false]) is not supported yet\""),
one("meta::pure::functions::math::tests::toFloat::testIntToFloat_Function_1__Boolean_1_", "\"[unsupported-api] The function 'toFloat' (state: [Select, false]) is not supported yet\""),
one("meta::pure::functions::math::tests::toDecimal::testIntToDecimal_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: 8D\nactual: 8.0D\""),

// Is
one("meta::pure::functions::boolean::tests::testIsEnum_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'is_Any_1__Any_1__Boolean_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""),
Expand Down Expand Up @@ -297,7 +290,7 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf

// IndexOf
one("meta::pure::functions::string::tests::indexOf::testFromIndex_Function_1__Boolean_1_", "\"No SQL translation exists for the PURE function 'indexOf_String_1__String_1__Integer_1__Integer_1_'. \nIf you would like to add a SQL translation for the function then follow the step-by-step guide on the PURE wiki.\""),
one("meta::pure::functions::string::tests::indexOf::testSimple_Function_1__Boolean_1_", "\"[unsupported-api] The function 'indexOf' (state: [Select, false]) is not supported yet\""),
one("meta::pure::functions::string::tests::indexOf::testSimple_Function_1__Boolean_1_", "Assert failure at (resource:/platform/pure/essential/tests/assert.pure line:21 column:5), \"\nexpected: 4\nactual: 5\""),

// ParseBoolean
one("meta::pure::functions::string::tests::parseBoolean::testParseFalse_Function_1__Boolean_1_", "\"[unsupported-api] The function 'parseBoolean' (state: [Select, false]) is not supported yet\""),
Expand All @@ -314,11 +307,7 @@ public class Test_Relational_DuckDB_EssentialFunctions_PCT extends PCTReportConf
one("meta::pure::functions::string::tests::parseDecimal::testParseZero_Function_1__Boolean_1_", "\"\nexpected: 0.000D\nactual: 0.0D\""),

// ParseInteger
one("meta::pure::functions::string::tests::parseInteger::testParseInteger_Function_1__Boolean_1_", "java.sql.SQLException: Conversion Error: Could not convert string '9999999999999992' to INT32\nLINE 1: select cast('9999999999999992' as integer)\n ^"),

// ReverseString
one("meta::pure::functions::string::tests::reverse::testReverseString_Function_1__Boolean_1_", "java.sql.SQLException: java.sql.SQLException: Catalog Error: Scalar Function with name legend_h2_extension_reverse_string does not exist!\nDid you mean \"list_reverse_sort\"?\nLINE 1: select legend_h2_extension_reverse_string('')\n ^")

one("meta::pure::functions::string::tests::parseInteger::testParseInteger_Function_1__Boolean_1_", "java.sql.SQLException: Conversion Error: Could not convert string '9999999999999992' to INT32\nLINE 1: select cast('9999999999999992' as integer)\n ^")
);

public static Test suite()
Expand Down
72 changes: 62 additions & 10 deletions ...rc/main/resources/core_relational_duckdb/relational/sqlQueryToString/duckdbExtension.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::duckD
isBooleanLiteralSupported = true,
isDbReservedIdentifier = {str:String[1]| $str->toLower()->in($reservedWords); }, // check case insensitive
windowColumnProcessor = processWindowColumn_WindowColumn_1__SqlGenerationContext_1__String_1_,
joinStringsProcessor = processJoinStringsOperationWithConcatCall_JoinStrings_1__SqlGenerationContext_1__String_1_,
joinStringsProcessor = processJoinStringsOperationForDuckDB_JoinStrings_1__SqlGenerationContext_1__String_1_,
literalProcessor = $literalProcessor,
selectSQLQueryProcessor = processSelectSQLQueryForDuckDB_SelectSQLQuery_1__SqlGenerationContext_1__Boolean_1__String_1_,
selectSQLQueryWithCTEsProcessor = processSelectSQLQueryWithCTEsDefault_SelectSQLQueryWithCommonTableExpressions_1__SqlGenerationContext_1__Boolean_1__String_1_,
Expand All @@ -42,7 +42,7 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::duckD
^RelationalDDLCommandsTranslator(
createSchema = translateCreateSchemaStatementDefault_CreateSchemaSQL_1__DbConfig_1__String_1_,
dropSchema = translateDropSchemaStatementDefault_DropSchemaSQL_1__DbConfig_1__String_1_,
createTable = translateCreateTableStatementDefault_CreateTableSQL_1__DbConfig_1__String_1_,
createTable = translateCreateTableStatementDuckDB_CreateTableSQL_1__DbConfig_1__String_1_,
dropTable = translateDropTableStatementDefault_DropTableSQL_1__DbConfig_1__String_1_,
loadTable = loadValuesToDbTableDefault_LoadTableSQL_1__DbConfig_1__String_MANY_
);
Expand Down Expand Up @@ -78,14 +78,60 @@ function meta::relational::functions::sqlQueryToString::duckDB::convertDateToSql
)
);
}

function meta::relational::functions::sqlQueryToString::duckDB::translateCreateTableStatementDuckDB(createTableSQL:CreateTableSQL[1], dbConfig:DbConfig[1]) : String[1]
{
let t= $createTableSQL.table;
let applyConstraints = $createTableSQL.applyConstraints;
'Create Table '+if($t.schema.name == 'default',|'',|$t.schema.name+'.')+$t.name+
+ '('
+ $t.columns->cast(@meta::relational::metamodel::Column)
->map(c | $c.name->processColumnName($dbConfig) + ' ' + dataTypeToSqlTextDuckDB($c.type) + if($c.nullable->isEmpty() || $applyConstraints == false, | '', | if($c.nullable == true , | ' NULL', | ' NOT NULL' )))
->joinStrings(',')
+ if ($t.primaryKey->isEmpty() || $applyConstraints == false, | '', | ', PRIMARY KEY(' + $t.primaryKey->map(c | $c.name)->joinStrings(',') + ')')
+');';
}


function meta::relational::functions::sqlQueryToString::duckDB::dataTypeToSqlTextDuckDB(type: meta::relational::metamodel::datatype::DataType[1]):String[1]
{
$type->match([
s : meta::relational::metamodel::datatype::SemiStructured[1] | 'VARCHAR(4000)',
i : meta::relational::metamodel::datatype::Integer[1] | 'INT',
f : meta::relational::metamodel::datatype::Float[1] | 'FLOAT',
v : meta::relational::metamodel::datatype::Varchar[1] | format('VARCHAR(%d)', $v.size),
c : meta::relational::metamodel::datatype::Char[1] | format('CHAR(%d)', $c.size),
d : meta::relational::metamodel::datatype::Decimal[1] | format('DECIMAL(%d, %d)', [$d.precision, $d.scale]),
t : meta::relational::metamodel::datatype::Timestamp[1] | 'TIMESTAMP',
d : meta::relational::metamodel::datatype::Date[1] | 'DATE',
b : meta::relational::metamodel::datatype::BigInt[1] | 'BIGINT',
s : meta::relational::metamodel::datatype::SmallInt[1] | 'SMALLINT',
t : meta::relational::metamodel::datatype::TinyInt[1] | 'TINYINT',
d : meta::relational::metamodel::datatype::Double[1] | 'DOUBLE',
n : meta::relational::metamodel::datatype::Numeric[1] | format('NUMERIC(%d, %d)', [$n.precision, $n.scale]),
d : meta::relational::metamodel::datatype::Distinct[1] | 'DISTINCT',
o : meta::relational::metamodel::datatype::Other[1] | 'OTHER',
b : meta::relational::metamodel::datatype::Bit[1] | 'BIT',
b : meta::relational::metamodel::datatype::Binary[1] | 'BINARY',
r : meta::relational::metamodel::datatype::Real[1] | 'REAL',
a : meta::relational::metamodel::datatype::Array[1] | 'ARRAY',
v : meta::relational::metamodel::datatype::Varbinary[1] | 'VARBINARY',
s : meta::relational::metamodel::datatype::SemiStructured[1] | 'SEMISTRUCTURED',
j : meta::relational::metamodel::datatype::Json[1] | 'JSON',
d : meta::relational::metamodel::datatype::DbSpecificDataType[1] | $d.dbSpecificSql
]);
}



function <<access.private>> meta::relational::functions::sqlQueryToString::duckDB::getDynaFunctionToSqlForDuckDB(): DynaFunctionToSql[*]
{
let allStates = allGenerationStates();

[
dynaFnToSql('adjust', $allStates, ^ToSql(format='date_add(%s)', transform={p:String[3] | $p->at(0) + ',' + constructIntervalFunction($p->at(2), $p->at(1)) })),
dynaFnToSql('booland', $allStates, ^ToSql(format='every(%s)')),
dynaFnToSql('boolor', $allStates, ^ToSql(format='any(%s)')),
dynaFnToSql('booland', $allStates, ^ToSql(format='bool_and(%s)')), //aggregate function across rows (not and on conditions)
dynaFnToSql('boolor', $allStates, ^ToSql(format='bool_or(%s)')),
dynaFnToSql('castBoolean', $allStates, ^ToSql(format='cast(%s as boolean)')),
dynaFnToSql('chr', $allStates, ^ToSql(format='char(%s)')),
dynaFnToSql('concat', $allStates, ^ToSql(format='concat%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})),
Expand Down Expand Up @@ -114,7 +160,7 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::duckD
dynaFnToSql('firstMinuteOfHour', $allStates, ^ToSql(format='date_trunc(\'hour\', %s)', transform={p:String[1] | $p->repeat(1)})),
dynaFnToSql('firstSecondOfMinute', $allStates, ^ToSql(format='date_trunc(\'minute\', %s)', transform={p:String[1] | $p->repeat(1)})),
dynaFnToSql('hour', $allStates, ^ToSql(format='hour(%s)')),
// dynaFnToSql('indexOf', $allStates, ^ToSql(format='position(%s IN %s)', transform={p:String[2] | [$p->at(1), $p->at(0)]})),
dynaFnToSql('indexOf', $allStates, ^ToSql(format='instr(%s, %s)', transform={p:String[2] | [$p->at(0), $p->at(1)]})), // TODO - pure uses 0-based indexing, duck db returns location with 1-based index , keeping this as H2 also returns 1-based currently, many user tests need to be fixed
dynaFnToSql('isNumeric', $allStates, ^ToSql(format='(lower(%s) = upper(%s))')),
dynaFnToSql('isAlphaNumeric', $allStates, ^ToSql(format='regexp_matches(%s,\'^[a-zA-Z0-9]*$\')', transform={p:String[1]|$p})),
dynaFnToSql('joinStrings', $allStates, ^ToSql(format='string_agg(%s,%s)')),
Expand All @@ -139,19 +185,19 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::duckD
// dynaFnToSql('previousDayOfWeek', $allStates, ^ToSql(format='date_add(DAY, case when %s - DAY_OF_WEEK(%s) >= 0 then %s - DAY_OF_WEEK(%s) - 7 else %s - DAY_OF_WEEK(%s) end, %s)', transform={p:String[1..2] | $p->formatMostRecentH2('current_date()')}, parametersWithinWhenClause = [false, false])),
dynaFnToSql('quarter', $allStates, ^ToSql(format='quarter(%s)')),
dynaFnToSql('quarterNumber', $allStates, ^ToSql(format='quarter(%s)')),
dynaFnToSql('reverseString', $allStates, ^ToSql(format='legend_h2_extension_reverse_string(%s)')),
// dynaFnToSql('round', $allStates, ^ToSql(format='round(%s, %s)', transform=transformRound_String_MANY__String_MANY_)),
dynaFnToSql('reverseString', $allStates, ^ToSql(format='reverse(%s)')),
dynaFnToSql('round', $allStates, ^ToSql(format='round(%s, %s)', transform=transformRound_String_MANY__String_MANY_)),
dynaFnToSql('rpad', $allStates, ^ToSql(format='rpad(%s,%s,%s)', transform=processPaddingParams_String_MANY__String_MANY_)),
dynaFnToSql('second', $allStates, ^ToSql(format='second(%s)')),
dynaFnToSql('sha1', $allStates, ^ToSql(format='sha1(%s)')),
dynaFnToSql('sha256', $allStates, ^ToSql(format='sha256(%s)')),
dynaFnToSql('splitPart', $allStates, ^ToSql(format='split_part(%s, %s, %s)')),
// dynaFnToSql('substring', $allStates, ^ToSql(format='substring%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})),
dynaFnToSql('substring', $allStates, ^ToSql(format='substring%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})), // TODO - pure uses 0-based indexing, duck db returns location with 1-based index , keeping this as H2 also returns 1-based currently, many user tests need to be fixed
dynaFnToSql('stdDevPopulation', $allStates, ^ToSql(format='stddev_pop(%s)')),
dynaFnToSql('stdDevSample', $allStates, ^ToSql(format='stddev_samp(%s)')),
dynaFnToSql('today', $allStates, ^ToSql(format='cast(today() as timestamp_s)')),
// dynaFnToSql('toDecimal', $allStates, ^ToSql(format='cast(%s as decimal)')),
// dynaFnToSql('toFloat', $allStates, ^ToSql(format='cast(%s as double precision)')),
dynaFnToSql('toDecimal', $allStates, ^ToSql(format='cast(%s as decimal)')),
dynaFnToSql('toFloat', $allStates, ^ToSql(format='cast(%s as double)')),
dynaFnToSql('toString', $allStates, ^ToSql(format='cast(%s as varchar)')),
// dynaFnToSql('toTimestamp', $allStates, ^ToSql(format='%s', transform={p:String[2] | $p->transformToTimestampH2()})),
dynaFnToSql('weekOfYear', $allStates, ^ToSql(format='week(%s)')),
Expand Down Expand Up @@ -274,3 +320,9 @@ function <<access.private>> meta::relational::functions::sqlQueryToString::duckD
'WHERE', 'WINDOW', 'WITH'
];
}

function meta::relational::functions::sqlQueryToString::duckDB::processJoinStringsOperationForDuckDB(js:JoinStrings[1], sgc:SqlGenerationContext[1]): String[1]
{
processJoinStringsOperation($js, $sgc, {col, sep| 'group_concat(' + $col + if($sep == '\'\'', |'', |',' + $sep) + ' )'},
{strs, sep| $strs->joinStrings('concat(', if('\'\'' == $sep, |', ', |',' + $sep + ',') , ')')});
}
Loading

0 comments on commit b8a5bfa

Please sign in to comment.