From add891909f4ab5de0ebc7091127954073214ff59 Mon Sep 17 00:00:00 2001 From: Hadrien Kohl Date: Fri, 10 Mar 2017 13:20:51 +0100 Subject: [PATCH 01/10] Use labels to simplify the BooleanExpressionVisitor --- .../src/main/antlr4/no/ssb/vtl/parser/VTL.g4 | 31 ++++---- .../visitors/BooleanExpressionVisitor.java | 72 ++++++++++--------- 2 files changed, 55 insertions(+), 48 deletions(-) diff --git a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 index 84ceda25..165ceb98 100644 --- a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 +++ b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 @@ -100,33 +100,27 @@ aggregate : 'aggregate' ; //WS : [ \t\n\t] -> skip ; booleanExpression - : booleanExpression op=AND booleanExpression - | booleanExpression op=(OR|XOR) booleanExpression - | booleanNot - | booleanIsNull - | booleanEquality - | BOOLEAN_CONSTANT + : '(' booleanExpression ')' # BooleanPrecedence + | func=ISNULL_FUNC '(' booleanParam ')' # BooleanFunction + | left=booleanParam op=EQ_OPERATOR right=booleanParam # BooleanEquality + | booleanExpression op=AND booleanExpression # BooleanAlgebra + | booleanExpression op=(OR|XOR) booleanExpression # BooleanAlgebra + | booleanParam op=(ISNULL|ISNOTNULL) # BooleanPostfix + | op=NOT booleanExpression # BooleanNot + //| componentRef # BooleanReference + | BOOLEAN_CONSTANT # BooleanConstant ; -booleanEquality - : left=booleanParam op=( EQ | NE | LE | LT | GE | GT ) right=booleanParam - ; booleanParam : componentRef | constant ; -booleanNot - : 'not' '(' booleanExpression ')'; - -booleanIsNull - : 'isnull' '(' booleanParam ')' - ; - //datasetExpression // : 'dsTest' // ; +EQ_OPERATOR : ( EQ | NE | LE | LT | GE | GT ) ; EQ : '=' ; NE : '<>' ; LE : '<=' ; @@ -137,6 +131,11 @@ GT : '>' ; AND : 'and' ; OR : 'or' ; XOR : 'xor' ; +NOT : 'not' ; +ISNULL : 'is null' ; +ISNOTNULL : 'is not null' ; + +ISNULL_FUNC : 'isnull' ; //WS : [ \r\t\u000C] -> skip ; diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java index 53029eae..152871ec 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java @@ -22,35 +22,35 @@ public BooleanExpressionVisitor(ReferenceVisitor referenceVisitor, DataStructure this.referenceVisitor = referenceVisitor; this.dataStructure = dataStructure; } - + @Override - public VTLPredicate visitBooleanExpression(VTLParser.BooleanExpressionContext ctx) { - if (ctx.BOOLEAN_CONSTANT() != null) { - Boolean booleanConstant = Boolean.valueOf(ctx.BOOLEAN_CONSTANT().getText()); - return dataPoints -> VTLBoolean.of(booleanConstant); - }else if (ctx.op != null) { - VTLPredicate left = visit(ctx.booleanExpression(0)); - VTLPredicate right = visit(ctx.booleanExpression(1)); - switch (ctx.op.getType()) { - case VTLParser.AND: - return left.and(right); - case VTLParser.OR: - return left.or(right); - case VTLParser.XOR: - return left.xor(right); - default: - throw new ParseCancellationException("Unsupported boolean operation: " + ctx.op.getText()); - } - } else if (ctx.booleanEquality() != null) { - return visit(ctx.booleanEquality()); - } else if (ctx.booleanIsNull() != null) { - return visit(ctx.booleanIsNull()); - } else if (ctx.booleanNot() != null) { - return visit(ctx.booleanNot()); - } else { - return super.visit(ctx); + public VTLPredicate visitBooleanConstant(VTLParser.BooleanConstantContext ctx) { + Boolean booleanConstant = Boolean.valueOf(ctx.BOOLEAN_CONSTANT().getText()); + return dataPoints -> VTLBoolean.of(booleanConstant); + } + + @Override + public VTLPredicate visitBooleanAlgebra(VTLParser.BooleanAlgebraContext ctx) { + + VTLPredicate left = visit(ctx.booleanExpression(0)); + VTLPredicate right = visit(ctx.booleanExpression(1)); + + switch (ctx.op.getType()) { + case VTLParser.AND: + return left.and(right); + case VTLParser.OR: + return left.or(right); + case VTLParser.XOR: + return left.xor(right); + default: + throw new ParseCancellationException("Unsupported boolean operation: " + ctx.op.getText()); } } + + @Override + public VTLPredicate visitBooleanPrecedence(VTLParser.BooleanPrecedenceContext ctx) { + return visit(ctx.booleanExpression()); + } @Override public VTLPredicate visitBooleanEquality(VTLParser.BooleanEqualityContext ctx) { @@ -89,11 +89,11 @@ VTLPredicate getVtlPredicate(Object left, Object right, BiPredicate { VTLObject resolvedValue = getValue((Component) value, dataPoint); @@ -136,18 +136,26 @@ BiPredicate getBooleanOperation(int op) { } } - @Override - public VTLPredicate visitBooleanIsNull(VTLParser.BooleanIsNullContext ctx) { + public VTLPredicate visitBooleanPostfix(VTLParser.BooleanPostfixContext ctx) { ParamVisitor paramVisitor = new ParamVisitor(referenceVisitor); Object ref = paramVisitor.visit(ctx.booleanParam()); - return getIsNullPredicate(ref); + int op = ctx.op.getType(); + switch (op) { + case VTLParser.ISNULL: + return getIsNullPredicate(ref); + case VTLParser.ISNOTNULL: + return getNotPredicate(getIsNullPredicate(ref)); + default: + throw new ParseCancellationException("Unsupported boolean postfix operator " + op); + } + } @Override public VTLPredicate visitBooleanNot(VTLParser.BooleanNotContext ctx) { - VTLPredicate predicate = visitBooleanExpression(ctx.booleanExpression()); + VTLPredicate predicate = visit(ctx.booleanExpression()); return getNotPredicate(predicate); } From 7e88e2c022b1de4d9961db53c0be83017b282b1d Mon Sep 17 00:00:00 2001 From: Hadrien Kohl Date: Fri, 10 Mar 2017 13:25:01 +0100 Subject: [PATCH 02/10] Grammar clean up --- java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 | 7 ------- 1 file changed, 7 deletions(-) diff --git a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 index 165ceb98..201fa826 100644 --- a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 +++ b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 @@ -93,8 +93,6 @@ attrcalc : 'attrcalc' ; aggregate : 'aggregate' ; -//booleanExpression : 'booleanExpression' ; - //varID : 'varId'; //WS : [ \t\n\t] -> skip ; @@ -107,7 +105,6 @@ booleanExpression | booleanExpression op=(OR|XOR) booleanExpression # BooleanAlgebra | booleanParam op=(ISNULL|ISNOTNULL) # BooleanPostfix | op=NOT booleanExpression # BooleanNot - //| componentRef # BooleanReference | BOOLEAN_CONSTANT # BooleanConstant ; @@ -116,10 +113,6 @@ booleanParam | constant ; -//datasetExpression -// : 'dsTest' -// ; - EQ_OPERATOR : ( EQ | NE | LE | LT | GE | GT ) ; EQ : '=' ; NE : '<>' ; From f304d3fb5378732d38b3de9b55ddc4ddfbdbb40b Mon Sep 17 00:00:00 2001 From: Hadrien Kohl Date: Fri, 10 Mar 2017 13:39:11 +0100 Subject: [PATCH 03/10] Add global scope as a temporary fix for the VTLScriptContext --- .../src/main/java/no/ssb/vtl/script/VTLScriptContext.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptContext.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptContext.java index 8b0062b4..7a6c65b1 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptContext.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptContext.java @@ -19,6 +19,7 @@ public VTLScriptContext() { super(); scopes = new HashMap<>(2); scopes.put(ENGINE_SCOPE, engineScope); + scopes.put(GLOBAL_SCOPE, new SimpleBindings()); } public void addScope(int scope) { From e5f2ad0c0cdb639c8705ef13ceac8d513cf7e0a9 Mon Sep 17 00:00:00 2001 From: Hadrien Kohl Date: Fri, 10 Mar 2017 14:07:42 +0100 Subject: [PATCH 04/10] Add the ASSIGNMENT token in order to fix grammar --- .../src/main/antlr4/no/ssb/vtl/parser/VTL.g4 | 26 +++++++-------- .../no/ssb/vtl/parser/ClausesParserTest.java | 12 +++---- .../no/ssb/vtl/parser/JoinParserTest.java | 32 +++++++++---------- .../ssb/vtl/script/VTLScriptEngineTest.java | 14 ++++---- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 index 201fa826..cc5e7c75 100644 --- a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 +++ b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 @@ -21,8 +21,8 @@ grammar VTL; start : statement+ EOF; /* Assignment */ -statement : identifier ':=' datasetExpression - | identifier ':=' block +statement : identifier ASIGNMENT datasetExpression + | identifier ASIGNMENT block ; block : '{' statement+ '}' ; @@ -79,7 +79,7 @@ clause : 'rename' renameParam (',' renameParam)* #renameClause // component as string role = MEASURE, // component as string role = ATTRIBUTE // ] -renameParam : from=componentRef 'as' to=identifier ( 'role' '=' role )? ; +renameParam : from=componentRef 'as' to=identifier ( 'role' role )? ; role : ( 'IDENTIFIER' | 'MEASURE' | 'ATTRIBUTE' ) ; @@ -98,14 +98,14 @@ aggregate : 'aggregate' ; //WS : [ \t\n\t] -> skip ; booleanExpression - : '(' booleanExpression ')' # BooleanPrecedence - | func=ISNULL_FUNC '(' booleanParam ')' # BooleanFunction - | left=booleanParam op=EQ_OPERATOR right=booleanParam # BooleanEquality - | booleanExpression op=AND booleanExpression # BooleanAlgebra - | booleanExpression op=(OR|XOR) booleanExpression # BooleanAlgebra - | booleanParam op=(ISNULL|ISNOTNULL) # BooleanPostfix - | op=NOT booleanExpression # BooleanNot - | BOOLEAN_CONSTANT # BooleanConstant + : '(' booleanExpression ')' # BooleanPrecedence + | func=ISNULL_FUNC '(' booleanParam ')' # BooleanFunction + | left=booleanParam op=( EQ | NE | LE | LT | GE | GT ) right=booleanParam # BooleanEquality + | booleanExpression op=AND booleanExpression # BooleanAlgebra + | booleanExpression op=(OR|XOR) booleanExpression # BooleanAlgebra + | booleanParam op=(ISNULL|ISNOTNULL) # BooleanPostfix + | op=NOT booleanExpression # BooleanNot + | BOOLEAN_CONSTANT # BooleanConstant ; booleanParam @@ -113,7 +113,7 @@ booleanParam | constant ; -EQ_OPERATOR : ( EQ | NE | LE | LT | GE | GT ) ; +ASIGNMENT : ':=' ; EQ : '=' ; NE : '<>' ; LE : '<=' ; @@ -143,7 +143,7 @@ joinDefinition : type=( INNER | OUTER | CROSS )? datasetRef (',' datasetRef )* ( joinBody : joinClause (',' joinClause)* ; // TODO: Implement role and implicit -joinClause : role? identifier '=' joinCalcExpression # joinCalcClause +joinClause : role? identifier ASIGNMENT joinCalcExpression # joinCalcClause | joinDropExpression # joinDropClause | joinKeepExpression # joinKeepClause | joinRenameExpression # joinRenameClause diff --git a/java-vtl-parser/src/test/java/no/ssb/vtl/parser/ClausesParserTest.java b/java-vtl-parser/src/test/java/no/ssb/vtl/parser/ClausesParserTest.java index 98209df8..f9843984 100644 --- a/java-vtl-parser/src/test/java/no/ssb/vtl/parser/ClausesParserTest.java +++ b/java-vtl-parser/src/test/java/no/ssb/vtl/parser/ClausesParserTest.java @@ -26,9 +26,9 @@ public class ClausesParserTest extends GrammarTest { @Test public void testRenameWithRole() throws Exception { - parse("[rename varId as varId role = IDENTIFIER]", "clauseExpression"); - parse("[rename varId as varId role = MEASURE]", "clauseExpression"); - parse("[rename varId as varId role = ATTRIBUTE]", "clauseExpression"); + parse("[rename varId as varId role IDENTIFIER]", "clauseExpression"); + parse("[rename varId as varId role MEASURE]", "clauseExpression"); + parse("[rename varId as varId role ATTRIBUTE]", "clauseExpression"); } @Test @@ -38,9 +38,9 @@ public void testMultipleRenames() throws Exception { @Test public void testMultipleRenamesWithRoles() throws Exception { - parse("[rename varId as varId role = IDENTIFIER," + - " varId as varId role = MEASURE," + - " varId as varId role = ATTRIBUTE]", + parse("[rename varId as varId role IDENTIFIER," + + " varId as varId role MEASURE," + + " varId as varId role ATTRIBUTE]", "clauseExpression"); } diff --git a/java-vtl-parser/src/test/java/no/ssb/vtl/parser/JoinParserTest.java b/java-vtl-parser/src/test/java/no/ssb/vtl/parser/JoinParserTest.java index 9bc6f67b..d44c6421 100644 --- a/java-vtl-parser/src/test/java/no/ssb/vtl/parser/JoinParserTest.java +++ b/java-vtl-parser/src/test/java/no/ssb/vtl/parser/JoinParserTest.java @@ -8,8 +8,8 @@ public class JoinParserTest extends GrammarTest { public void testJoin() throws Exception { String expression = "" + "[varID1, varID2]{\n" + - " varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + - " varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + + " varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + + " varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + "}"; parse(expression, "joinExpression"); } @@ -18,8 +18,8 @@ public void testJoin() throws Exception { public void testJoinWithOn() throws Exception { String expression = "" + "[varID1, varID2 on dimensionExpr1, dimensionExpr2]{\n" + - " varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + - " varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + + " varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + + " varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + "}"; parse(expression, "joinExpression"); } @@ -28,8 +28,8 @@ public void testJoinWithOn() throws Exception { public void testOuterJoin() throws Exception { String expression = "" + "[outer varID1, varID2]{\n" + - " varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + - " varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + + " varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + + " varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + "}"; parse(expression, "joinExpression"); } @@ -38,8 +38,8 @@ public void testOuterJoin() throws Exception { public void testOuterJoinWithOn() throws Exception { String expression = "" + "[outer varID1, varID2 on dimensionExpr1, dimensionExpr2]{\n" + - " varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + - " varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + + " varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + + " varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + "}"; parse(expression, "joinExpression"); } @@ -48,8 +48,8 @@ public void testOuterJoinWithOn() throws Exception { public void testInnerJoin() throws Exception { String expression = "" + "[inner varID1, varID2]{\n" + - " varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + - " varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + + " varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + + " varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + "}"; parse(expression, "joinExpression"); } @@ -58,8 +58,8 @@ public void testInnerJoin() throws Exception { public void testInnerJoinWithOn() throws Exception { String expression = "" + "[inner varID1, varID2 on dimensionExpr1, dimensionExpr2]{\n" + - " varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + - " varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + + " varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + + " varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + "}"; parse(expression, "joinExpression"); } @@ -68,8 +68,8 @@ public void testInnerJoinWithOn() throws Exception { public void testCrossrJoin() throws Exception { String expression = "" + "[inner varID1, varID2]{\n" + - " varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + - " varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + + " varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + + " varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + "}"; parse(expression, "joinExpression"); } @@ -78,8 +78,8 @@ public void testCrossrJoin() throws Exception { public void testCrossJoinWithOn() throws Exception { String expression = "" + "[inner varID1, varID2 on dimensionExpr1, dimensionExpr2]{\n" + - " varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + - " varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + + " varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" + + " varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" + "}"; parse(expression, "joinExpression"); } diff --git a/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java b/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java index f220d607..f6159413 100644 --- a/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java +++ b/java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java @@ -160,9 +160,9 @@ public void testJoin() throws Exception { engine.eval("" + "ds3 := [ds1, ds2]{" + // id1, id2, ds1.m1, ds1.m2, d2.m1, d2.m2, at1, at2 " filter id1 = \"1\" and m1 = 30 or m1 = 10," + //TODO: precedence - " ident = ds1.m1 + ds2.m2 - ds1.m2 - ds2.m1," + // id1, id2, ds1.m1, ds1.m2, d2.m1, d2.m2, at1, at2, ident + " ident := ds1.m1 + ds2.m2 - ds1.m2 - ds2.m1," + // id1, id2, ds1.m1, ds1.m2, d2.m1, d2.m2, at1, at2, ident " keep ident, ds1.m1, ds2.m1, ds2.m2," + // id1, id2, ds1.m1, ds2.m1, ds2.m2, ident - " boolTest = (ds1.m1 = 10)," + + " boolTest := (ds1.m1 = 10)," + " drop ds2.m1," + // id1, id2, ds1.m1, ds2.m2, ident " rename id1 to renamedId1, ds1.m1 to m1, ds2.m2 to m2" + // renamedId1, id2, m1, m2, ident "}" + @@ -235,7 +235,7 @@ public void testJoinFold() throws Exception { bindings.put("ds1", ds1); engine.eval("ds2 := [ds1] {" + - " total = ds1.m1 + ds1.m2 + ds1.m3," + + " total := ds1.m1 + ds1.m2 + ds1.m3," + " fold ds1.m1, ds1.m2, ds1.m3, total to type, value" + "}" ); @@ -314,7 +314,7 @@ public void testJoinUnfold() throws Exception { bindings.put("ds1", ds1); engine.eval("ds2 := [ds1] {" + " unfold id2, ds1.m1 to \"one\", \"two\"," + - " onePlusTwo = one + two" + + " onePlusTwo := one + two" + "}" ); @@ -350,9 +350,9 @@ public void testRename() throws Exception { bindings.put("ds1", dataset); engine.eval("ds2 := ds1[rename id1 as id3]" + " [rename id3 as id1]" - + " [rename id1 as id1m role = MEASURE," - + " me1 as me1a role = ATTRIBUTE," - + " at1 as at1i role = IDENTIFIER]"); + + " [rename id1 as id1m role MEASURE," + + " me1 as me1a role ATTRIBUTE," + + " at1 as at1i role IDENTIFIER]"); assertThat(bindings).containsKey("ds2"); Dataset result = (Dataset) bindings.get("ds2"); From fb6984738e29ebf28557e434a2702b4f1ba5d8d3 Mon Sep 17 00:00:00 2001 From: Hadrien Kohl Date: Fri, 10 Mar 2017 14:14:40 +0100 Subject: [PATCH 05/10] Add more parser tests --- .../no/ssb/vtl/parser/BooleanParserTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/java-vtl-parser/src/test/java/no/ssb/vtl/parser/BooleanParserTest.java b/java-vtl-parser/src/test/java/no/ssb/vtl/parser/BooleanParserTest.java index 6bdb7d91..ab57e815 100644 --- a/java-vtl-parser/src/test/java/no/ssb/vtl/parser/BooleanParserTest.java +++ b/java-vtl-parser/src/test/java/no/ssb/vtl/parser/BooleanParserTest.java @@ -5,6 +5,24 @@ public class BooleanParserTest extends GrammarTest { + @Test + public void testAlgebra() throws Exception { + parse("true and false or false and true", "booleanExpression"); + parse("true and (false or false) and true", "booleanExpression"); + parse("not true and not (true or false) and not true", "booleanExpression"); + parse("not ( ( true xor not (true or false) and not true ) )", "booleanExpression"); + } + + @Test + public void testEquality() throws Exception { + parse("dataset.component = component", "booleanExpression"); + parse("dataset.component <> component", "booleanExpression"); + parse("dataset.component <= component", "booleanExpression"); + parse("dataset.component >= component", "booleanExpression"); + parse("dataset.component < component", "booleanExpression"); + parse("dataset.component > component", "booleanExpression"); + } + @Test public void testNot() throws Exception { parse("not(true)", "booleanExpression"); @@ -17,5 +35,9 @@ public void testNot() throws Exception { public void testIsNull() throws Exception { parse("isnull(a)", "booleanExpression"); parse("isnull(null)", "booleanExpression"); + parse("component is not null", "booleanExpression"); + parse("dataset.component is not null", "booleanExpression"); + parse("component is null", "booleanExpression"); + parse("dataset.component is null", "booleanExpression"); } } From dbc43aa5de78a160cc3a970bde52566e573a3ace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=B8iseth-Gilje=2C=20Eivind?= Date: Fri, 10 Mar 2017 23:28:29 +0100 Subject: [PATCH 06/10] Minor cleanup in boolean visitor and grammar --- .../src/main/antlr4/no/ssb/vtl/parser/VTL.g4 | 25 ++--- .../visitors/BooleanExpressionVisitor.java | 93 +++++++++---------- .../visitors/VTLScalarExpressionVisitor.java | 2 +- 3 files changed, 60 insertions(+), 60 deletions(-) diff --git a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 index cc5e7c75..c60e4d7c 100644 --- a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 +++ b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 @@ -21,8 +21,8 @@ grammar VTL; start : statement+ EOF; /* Assignment */ -statement : identifier ASIGNMENT datasetExpression - | identifier ASIGNMENT block +statement : identifier ASSIGNMENT datasetExpression + | identifier ASSIGNMENT block ; block : '{' statement+ '}' ; @@ -97,14 +97,15 @@ aggregate : 'aggregate' ; //WS : [ \t\n\t] -> skip ; -booleanExpression - : '(' booleanExpression ')' # BooleanPrecedence - | func=ISNULL_FUNC '(' booleanParam ')' # BooleanFunction - | left=booleanParam op=( EQ | NE | LE | LT | GE | GT ) right=booleanParam # BooleanEquality - | booleanExpression op=AND booleanExpression # BooleanAlgebra - | booleanExpression op=(OR|XOR) booleanExpression # BooleanAlgebra - | booleanParam op=(ISNULL|ISNOTNULL) # BooleanPostfix - | op=NOT booleanExpression # BooleanNot +booleanExpression //Evaluation order of the operators + : '(' booleanExpression ')' # BooleanPrecedence // I + | func=ISNULL_FUNC '(' booleanParam ')' # BooleanFunction // II All functional operators + | booleanParam op=(ISNULL|ISNOTNULL) # BooleanPostfix // ?? + | left=booleanParam op=( LE | LT | GE | GT ) right=booleanParam # BooleanEquality // VII + | op=NOT booleanExpression # BooleanNot // IV + | left=booleanParam op=( EQ | NE ) right=booleanParam # BooleanEquality // IX + | booleanExpression op=AND booleanExpression # BooleanAlgebra // X + | booleanExpression op=(OR|XOR) booleanExpression # BooleanAlgebra // XI | BOOLEAN_CONSTANT # BooleanConstant ; @@ -113,7 +114,7 @@ booleanParam | constant ; -ASIGNMENT : ':=' ; +ASSIGNMENT : ':=' ; EQ : '=' ; NE : '<>' ; LE : '<=' ; @@ -143,7 +144,7 @@ joinDefinition : type=( INNER | OUTER | CROSS )? datasetRef (',' datasetRef )* ( joinBody : joinClause (',' joinClause)* ; // TODO: Implement role and implicit -joinClause : role? identifier ASIGNMENT joinCalcExpression # joinCalcClause +joinClause : role? identifier ASSIGNMENT joinCalcExpression # joinCalcClause | joinDropExpression # joinDropClause | joinKeepExpression # joinKeepClause | joinRenameExpression # joinRenameClause diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java index 152871ec..d8d23638 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java @@ -6,14 +6,13 @@ import no.ssb.vtl.model.VTLBoolean; import no.ssb.vtl.model.VTLObject; import no.ssb.vtl.model.VTLPredicate; -import no.ssb.vtl.parser.VTLBaseVisitor; import no.ssb.vtl.parser.VTLParser; import org.antlr.v4.runtime.misc.ParseCancellationException; import java.util.Map; import java.util.function.BiPredicate; -public class BooleanExpressionVisitor extends VTLBaseVisitor { +public class BooleanExpressionVisitor extends VTLScalarExpressionVisitor { private final ReferenceVisitor referenceVisitor; private final DataStructure dataStructure; @@ -63,6 +62,49 @@ public VTLPredicate visitBooleanEquality(VTLParser.BooleanEqualityContext ctx) { return getVtlPredicate(left, right, booleanOperation); } + @Override + public VTLPredicate visitBooleanPostfix(VTLParser.BooleanPostfixContext ctx) { + ParamVisitor paramVisitor = new ParamVisitor(referenceVisitor); + Object ref = paramVisitor.visit(ctx.booleanParam()); + + int op = ctx.op.getType(); + switch (op) { + case VTLParser.ISNULL: + return getIsNullPredicate(ref); + case VTLParser.ISNOTNULL: + return getNotPredicate(getIsNullPredicate(ref)); + default: + throw new ParseCancellationException("Unsupported boolean postfix operator " + op); + } + + } + + @Override + public VTLPredicate visitBooleanNot(VTLParser.BooleanNotContext ctx) { + VTLPredicate predicate = visit(ctx.booleanExpression()); + return getNotPredicate(predicate); + } + + BiPredicate getBooleanOperation(int op) { + BiPredicate equals = ((l, r) -> l.compareTo(r) == 0); + switch (op) { + case VTLParser.EQ: + return equals; + case VTLParser.NE: + return equals.negate(); + case VTLParser.LE: + return (l, r) -> l.compareTo(r) <= 0; + case VTLParser.LT: + return (l, r) -> l.compareTo(r) < 0; + case VTLParser.GE: + return (l, r) -> l.compareTo(r) >= 0; + case VTLParser.GT: + return (l, r) -> l.compareTo(r) > 0; + default: + throw new ParseCancellationException("Unsupported boolean equality operator " + op); + } + } + VTLPredicate getVtlPredicate(Object left, Object right, BiPredicate booleanOperation) { if (right == null || left == null) { return dataPoint -> null; @@ -88,11 +130,11 @@ VTLPredicate getVtlPredicate(Object left, Object right, BiPredicate nullableResultOf(booleanOperation, VTLObject.of(left), VTLObject.of(right)); } } - + VTLPredicate getNotPredicate(VTLPredicate predicate) { return predicate.negate(); } - + VTLPredicate getIsNullPredicate(Object value) { if (isComp(value)) { return dataPoint -> { @@ -115,49 +157,6 @@ private VTLBoolean nullableResultOf(BiPredicate operation, return VTLBoolean.of(operation.test(left, right)); } } - - BiPredicate getBooleanOperation(int op) { - BiPredicate equals = ((l, r) -> l.compareTo(r) == 0); - switch (op) { - case VTLParser.EQ: - return equals; - case VTLParser.NE: - return equals.negate(); - case VTLParser.LE: - return (l, r) -> l.compareTo(r) <= 0; - case VTLParser.LT: - return (l, r) -> l.compareTo(r) < 0; - case VTLParser.GE: - return (l, r) -> l.compareTo(r) >= 0; - case VTLParser.GT: - return (l, r) -> l.compareTo(r) > 0; - default: - throw new ParseCancellationException("Unsupported boolean equality operator " + op); - } - } - - @Override - public VTLPredicate visitBooleanPostfix(VTLParser.BooleanPostfixContext ctx) { - ParamVisitor paramVisitor = new ParamVisitor(referenceVisitor); - Object ref = paramVisitor.visit(ctx.booleanParam()); - - int op = ctx.op.getType(); - switch (op) { - case VTLParser.ISNULL: - return getIsNullPredicate(ref); - case VTLParser.ISNOTNULL: - return getNotPredicate(getIsNullPredicate(ref)); - default: - throw new ParseCancellationException("Unsupported boolean postfix operator " + op); - } - - } - - @Override - public VTLPredicate visitBooleanNot(VTLParser.BooleanNotContext ctx) { - VTLPredicate predicate = visit(ctx.booleanExpression()); - return getNotPredicate(predicate); - } private VTLObject getValue(Component component, DataPoint dataPoint) { Map componentVTLObjectMap = dataStructure.asMap(dataPoint); diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/VTLScalarExpressionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/VTLScalarExpressionVisitor.java index a7232842..41e5885e 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/VTLScalarExpressionVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/VTLScalarExpressionVisitor.java @@ -6,5 +6,5 @@ import java.util.function.Function; -public class VTLScalarExpressionVisitor> extends VTLBaseVisitor { +public class VTLScalarExpressionVisitor> extends VTLBaseVisitor { } From e8a28d9e716c5e92af3464c17f4a4beebfd058c7 Mon Sep 17 00:00:00 2001 From: Hadrien Kohl Date: Mon, 13 Mar 2017 10:32:23 +0100 Subject: [PATCH 07/10] Add the missing visitBooleanFunction() method --- .../vtl/script/visitors/BooleanExpressionVisitor.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java index d8d23638..2643cf31 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java @@ -78,7 +78,14 @@ public VTLPredicate visitBooleanPostfix(VTLParser.BooleanPostfixContext ctx) { } } - + + @Override + public VTLPredicate visitBooleanFunction(VTLParser.BooleanFunctionContext ctx) { + ParamVisitor paramVisitor = new ParamVisitor(referenceVisitor); + Object ref = paramVisitor.visit(ctx.booleanParam()); + return getIsNullPredicate(ref); + } + @Override public VTLPredicate visitBooleanNot(VTLParser.BooleanNotContext ctx) { VTLPredicate predicate = visit(ctx.booleanExpression()); From 7ddd7b6f86056616df972c65197ce1e3a1bec292 Mon Sep 17 00:00:00 2001 From: Hadrien Kohl Date: Mon, 13 Mar 2017 11:56:02 +0100 Subject: [PATCH 08/10] Add documentation --- java-vtl-documentation/docs/reference.md | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/java-vtl-documentation/docs/reference.md b/java-vtl-documentation/docs/reference.md index 8a2f5d3f..d726c803 100644 --- a/java-vtl-documentation/docs/reference.md +++ b/java-vtl-documentation/docs/reference.md @@ -192,3 +192,53 @@ Sweden , 9M + +## Boolean operators + +### Null operators + +VTL adopts 3VL (three-value logic); null is not considered a "value", + but rather a marker (or placeholder) indicating the absence of value. + +| *p* | *q* | *p* OR *q* | *p* AND *q* | *p* = *q* | +|---------|---------|-------------|--------------|------------| +| True | True | True | True | True | +| True | False | True | False | False | +| True | Unknown | True | Unknown | Unknown | +| False | True | True | False | False | +| False | False | False | False | True | +| False | Unknown | Unknown | False | Unknown | +| Unknown | True | True | Unknown | Unknown | +| Unknown | False | Unknown | False | Unknown | +| Unknown | Unknown | Unknown | Unknown | Unknown | + +| *p* | NOT *p* | +|:--------|:--------| +| True | False | +| False | True | +| Unknown | Unknown | + +Null in boolean operators evaluates to false. In order to test whether + or not a value is null one can use the postfix operator `is null` or + `is not null` as well as the functional equivalents `isnull()` or + `not(isnull())`. + +
+ +isnotnull := [data] { + filter value is not null +} +isnull := { + filter value is null +} + + +country[I,String],value[M,String] +Null , null +NotNull , value + + +
+ + + From 5b789db62d853f063892133b6b8e303f833dcd4b Mon Sep 17 00:00:00 2001 From: Hadrien Kohl Date: Mon, 13 Mar 2017 12:06:21 +0100 Subject: [PATCH 09/10] Add doc in the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bbd643e8..11eef425 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Conditional|if-then-else| Conditional|nvl| Validation|Comparisons (>,<,>=,<=,=,<>)| Validation|in,not in, between| -Validation|isnull| +Validation|isnull|![done][done]|Implemented syntax are `isnull(value)`, `value is null` and `value is not null`| Validation|exist_in, not_exist_in| Validation|exist_in_all, not_exist_in_all| Validation|check| From 41adadd67320f24c8e517154b7ff2a0c8f975b23 Mon Sep 17 00:00:00 2001 From: Hadrien Kohl Date: Mon, 13 Mar 2017 12:18:15 +0100 Subject: [PATCH 10/10] Use a better label name for the isnull functional operator --- java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 | 2 +- .../no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 index c60e4d7c..e5349a12 100644 --- a/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 +++ b/java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4 @@ -99,7 +99,7 @@ aggregate : 'aggregate' ; booleanExpression //Evaluation order of the operators : '(' booleanExpression ')' # BooleanPrecedence // I - | func=ISNULL_FUNC '(' booleanParam ')' # BooleanFunction // II All functional operators + | ISNULL_FUNC '(' booleanParam ')' # BooleanIsNullFunction // II All functional operators | booleanParam op=(ISNULL|ISNOTNULL) # BooleanPostfix // ?? | left=booleanParam op=( LE | LT | GE | GT ) right=booleanParam # BooleanEquality // VII | op=NOT booleanExpression # BooleanNot // IV diff --git a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java index 2643cf31..cbde999c 100644 --- a/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java +++ b/java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/BooleanExpressionVisitor.java @@ -80,7 +80,7 @@ public VTLPredicate visitBooleanPostfix(VTLParser.BooleanPostfixContext ctx) { } @Override - public VTLPredicate visitBooleanFunction(VTLParser.BooleanFunctionContext ctx) { + public VTLPredicate visitBooleanIsNullFunction(VTLParser.BooleanIsNullFunctionContext ctx) { ParamVisitor paramVisitor = new ParamVisitor(referenceVisitor); Object ref = paramVisitor.visit(ctx.booleanParam()); return getIsNullPredicate(ref);