Skip to content

Commit

Permalink
Merge pull request #16 in CORE/java-vtl from feature/is-null-operator…
Browse files Browse the repository at this point in the history 
… to develop

* commit '354b980919e48751fdd7547bdca25a293c490804':
  Use a better label name for the isnull functional operator
  Add doc in the README
  Add documentation
  Add the missing visitBooleanFunction() method
  Minor cleanup in boolean visitor and grammar
  Add more parser tests
  Add the ASSIGNMENT token in order to fix grammar
  Add global scope as a temporary fix for the VTLScriptContext
  Grammar clean up
  Use labels to simplify the BooleanExpressionVisitor
  • Loading branch information
@hadrienk
Kohl, Hadrien authored and hadrienk committed Mar 13, 2017
2 parents eba7d58 + 354b980 commit f656916
Show file tree
Hide file tree
Showing 10 changed files with 208 additions and 128 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ Conditional|if-then-else|
Conditional|nvl|![usable](http://progressed.io/bar/50)|Dataset as input not implemented.
Validation|Comparisons (>,<,>=,<=,=,<>)|
Validation|in,not in, between|
Validation|isnull|
Validation|isnull|![done][done]|Implemented syntax are `isnull(value)`, `value is null` and `value is not null`|
Validation|exist_in, not_exist_in|
Validation|exist_in_all, not_exist_in_all|
Validation|check|
Expand Down
50 changes: 50 additions & 0 deletions java-vtl-documentation/docs/reference.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,3 +212,53 @@ Sweden , 9M
</vtl-dataset>
<vtl-data datasets="datasets" errors="errors"></vtl-data>
</div>

## Boolean operators

### Null operators

VTL adopts 3VL (three-value logic); null is not considered a "value",
but rather a marker (or placeholder) indicating the absence of value.

| *p* | *q* | *p* OR *q* | *p* AND *q* | *p* = *q* |
|---------|---------|-------------|--------------|------------|
| True | True | True | True | True |
| True | False | True | False | False |
| True | Unknown | True | Unknown | Unknown |
| False | True | True | False | False |
| False | False | False | False | True |
| False | Unknown | Unknown | False | Unknown |
| Unknown | True | True | Unknown | Unknown |
| Unknown | False | Unknown | False | Unknown |
| Unknown | Unknown | Unknown | Unknown | Unknown |

| *p* | NOT *p* |
|:--------|:--------|
| True | False |
| False | True |
| Unknown | Unknown |

Null in boolean operators evaluates to false. In order to test whether
or not a value is null one can use the postfix operator `is null` or
`is not null` as well as the functional equivalents `isnull()` or
`not(isnull())`.

<div vtl-example>
<vtl-code>
isnotnull := [data] {
filter value is not null
}
isnull := {
filter value is null
}
</vtl-code>
<vtl-dataset name="data">
country[I,String],value[M,String]
Null , null
NotNull , value
</vtl-dataset>
<vtl-data datasets="datasets" errors="errors"></vtl-data>
</div>



47 changes: 20 additions & 27 deletions java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ grammar VTL;
start : statement+ EOF;

/* Assignment */
statement : identifier ':=' datasetExpression
| identifier ':=' block
statement : identifier ASSIGNMENT datasetExpression
| identifier ASSIGNMENT block
;

block : '{' statement+ '}' ;
Expand Down Expand Up @@ -79,7 +79,7 @@ clause : 'rename' renameParam (',' renameParam)* #renameClause
// component as string role = MEASURE,
// component as string role = ATTRIBUTE
// ]
renameParam : from=componentRef 'as' to=identifier ( 'role' '=' role )? ;
renameParam : from=componentRef 'as' to=identifier ( 'role' role )? ;

role : ( 'IDENTIFIER' | 'MEASURE' | 'ATTRIBUTE' ) ;

Expand All @@ -93,40 +93,28 @@ attrcalc : 'attrcalc' ;

aggregate : 'aggregate' ;

//booleanExpression : 'booleanExpression' ;

//varID : 'varId';

//WS : [ \t\n\t] -> skip ;

booleanExpression
: booleanExpression op=AND booleanExpression
| booleanExpression op=(OR|XOR) booleanExpression
| booleanNot
| booleanIsNull
| booleanEquality
| BOOLEAN_CONSTANT
booleanExpression //Evaluation order of the operators
: '(' booleanExpression ')' # BooleanPrecedence // I
| ISNULL_FUNC '(' booleanParam ')' # BooleanIsNullFunction // II All functional operators
| booleanParam op=(ISNULL|ISNOTNULL) # BooleanPostfix // ??
| left=booleanParam op=( LE | LT | GE | GT ) right=booleanParam # BooleanEquality // VII
| op=NOT booleanExpression # BooleanNot // IV
| left=booleanParam op=( EQ | NE ) right=booleanParam # BooleanEquality // IX
| booleanExpression op=AND booleanExpression # BooleanAlgebra // X
| booleanExpression op=(OR|XOR) booleanExpression # BooleanAlgebra // XI
| BOOLEAN_CONSTANT # BooleanConstant
;

booleanEquality
: left=booleanParam op=( EQ | NE | LE | LT | GE | GT ) right=booleanParam
;
booleanParam
: componentRef
| constant
;

booleanNot
: 'not' '(' booleanExpression ')';

booleanIsNull
: 'isnull' '(' booleanParam ')'
;

//datasetExpression
// : 'dsTest'
// ;

ASSIGNMENT : ':=' ;
EQ : '=' ;
NE : '<>' ;
LE : '<=' ;
Expand All @@ -137,6 +125,11 @@ GT : '>' ;
AND : 'and' ;
OR : 'or' ;
XOR : 'xor' ;
NOT : 'not' ;
ISNULL : 'is null' ;
ISNOTNULL : 'is not null' ;

ISNULL_FUNC : 'isnull' ;

//WS : [ \r\t\u000C] -> skip ;

Expand All @@ -151,7 +144,7 @@ joinDefinition : type=( INNER | OUTER | CROSS )? datasetRef (',' datasetRef )* (
joinBody : joinClause (',' joinClause)* ;

// TODO: Implement role and implicit
joinClause : role? identifier '=' joinCalcExpression # joinCalcClause
joinClause : role? identifier ASSIGNMENT joinCalcExpression # joinCalcClause
| joinDropExpression # joinDropClause
| joinKeepExpression # joinKeepClause
| joinRenameExpression # joinRenameClause
Expand Down
22 changes: 22 additions & 0 deletions java-vtl-parser/src/test/java/no/ssb/vtl/parser/BooleanParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,24 @@

public class BooleanParserTest extends GrammarTest {

@Test
public void testAlgebra() throws Exception {
parse("true and false or false and true", "booleanExpression");
parse("true and (false or false) and true", "booleanExpression");
parse("not true and not (true or false) and not true", "booleanExpression");
parse("not ( ( true xor not (true or false) and not true ) )", "booleanExpression");
}

@Test
public void testEquality() throws Exception {
parse("dataset.component = component", "booleanExpression");
parse("dataset.component <> component", "booleanExpression");
parse("dataset.component <= component", "booleanExpression");
parse("dataset.component >= component", "booleanExpression");
parse("dataset.component < component", "booleanExpression");
parse("dataset.component > component", "booleanExpression");
}

@Test
public void testNot() throws Exception {
parse("not(true)", "booleanExpression");
Expand All @@ -17,5 +35,9 @@ public void testNot() throws Exception {
public void testIsNull() throws Exception {
parse("isnull(a)", "booleanExpression");
parse("isnull(null)", "booleanExpression");
parse("component is not null", "booleanExpression");
parse("dataset.component is not null", "booleanExpression");
parse("component is null", "booleanExpression");
parse("dataset.component is null", "booleanExpression");
}
}
12 changes: 6 additions & 6 deletions java-vtl-parser/src/test/java/no/ssb/vtl/parser/ClausesParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ public class ClausesParserTest extends GrammarTest {

@Test
public void testRenameWithRole() throws Exception {
parse("[rename varId as varId role = IDENTIFIER]", "clauseExpression");
parse("[rename varId as varId role = MEASURE]", "clauseExpression");
parse("[rename varId as varId role = ATTRIBUTE]", "clauseExpression");
parse("[rename varId as varId role IDENTIFIER]", "clauseExpression");
parse("[rename varId as varId role MEASURE]", "clauseExpression");
parse("[rename varId as varId role ATTRIBUTE]", "clauseExpression");
}

@Test
Expand All @@ -38,9 +38,9 @@ public void testMultipleRenames() throws Exception {

@Test
public void testMultipleRenamesWithRoles() throws Exception {
parse("[rename varId as varId role = IDENTIFIER," +
" varId as varId role = MEASURE," +
" varId as varId role = ATTRIBUTE]",
parse("[rename varId as varId role IDENTIFIER," +
" varId as varId role MEASURE," +
" varId as varId role ATTRIBUTE]",
"clauseExpression");
}

Expand Down
32 changes: 16 additions & 16 deletions java-vtl-parser/src/test/java/no/ssb/vtl/parser/JoinParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ public class JoinParserTest extends GrammarTest {
public void testJoin() throws Exception {
String expression = "" +
"[varID1, varID2]{\n" +
" varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
" varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
"}";
parse(expression, "joinExpression");
}
Expand All @@ -18,8 +18,8 @@ public void testJoin() throws Exception {
public void testJoinWithOn() throws Exception {
String expression = "" +
"[varID1, varID2 on dimensionExpr1, dimensionExpr2]{\n" +
" varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
" varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
"}";
parse(expression, "joinExpression");
}
Expand All @@ -28,8 +28,8 @@ public void testJoinWithOn() throws Exception {
public void testOuterJoin() throws Exception {
String expression = "" +
"[outer varID1, varID2]{\n" +
" varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
" varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
"}";
parse(expression, "joinExpression");
}
Expand All @@ -38,8 +38,8 @@ public void testOuterJoin() throws Exception {
public void testOuterJoinWithOn() throws Exception {
String expression = "" +
"[outer varID1, varID2 on dimensionExpr1, dimensionExpr2]{\n" +
" varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
" varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
"}";
parse(expression, "joinExpression");
}
Expand All @@ -48,8 +48,8 @@ public void testOuterJoinWithOn() throws Exception {
public void testInnerJoin() throws Exception {
String expression = "" +
"[inner varID1, varID2]{\n" +
" varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
" varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
"}";
parse(expression, "joinExpression");
}
Expand All @@ -58,8 +58,8 @@ public void testInnerJoin() throws Exception {
public void testInnerJoinWithOn() throws Exception {
String expression = "" +
"[inner varID1, varID2 on dimensionExpr1, dimensionExpr2]{\n" +
" varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
" varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
"}";
parse(expression, "joinExpression");
}
Expand All @@ -68,8 +68,8 @@ public void testInnerJoinWithOn() throws Exception {
public void testCrossrJoin() throws Exception {
String expression = "" +
"[inner varID1, varID2]{\n" +
" varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
" varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
"}";
parse(expression, "joinExpression");
}
Expand All @@ -78,8 +78,8 @@ public void testCrossrJoin() throws Exception {
public void testCrossJoinWithOn() throws Exception {
String expression = "" +
"[inner varID1, varID2 on dimensionExpr1, dimensionExpr2]{\n" +
" varID2 = varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 = varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
" varID2 := varID2.varID2 + 1 * 2 / varID2.varID2 - 2,\n" +
" varID2 := varID2.varID2 + 3 * 4 / varID2.varID2 - 5\n" +
"}";
parse(expression, "joinExpression");
}
Expand Down
1 change: 1 addition & 0 deletions java-vtl-script/src/main/java/no/ssb/vtl/script/VTLScriptContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ public VTLScriptContext() {
super();
scopes = new HashMap<>(2);
scopes.put(ENGINE_SCOPE, engineScope);
scopes.put(GLOBAL_SCOPE, new SimpleBindings());
}

public void addScope(int scope) {
Expand Down
Loading

0 comments on commit f656916

Please sign in to comment.