Merge branch 'develop' of https://git-adm.ssb.no/scm/core/java-vtl in…

…to feature/is-null-operator

README.md

@@ -33,7 +33,7 @@ Clauses|calc|
 Clauses|attrcalc|
 Clauses|aggregate|
 Conditional|if-then-else|
-Conditional|nvl|
+Conditional|nvl|![usable](http://progressed.io/bar/50)|Dataset as input not implemented.
 Validation|Comparisons (>,<,>=,<=,=,<>)|
 Validation|in,not in, between|
 Validation|isnull|![done][done]|Implemented syntax are `isnull(value)`, `value is null` and `value is not null`|

java-vtl-documentation/docs/reference.md

@@ -193,6 +193,26 @@ Sweden , 9M
     <vtl-data datasets="datasets" errors="errors"></vtl-data>
 </div>
 
+## Conditional operators
+
+### nvl
+The operator nvl replaces null values with a value given as a parameter.
+<div vtl-example>
+    <vtl-code>
+join := [left, left] {
+    b = nvl(population, 0)
+}
+    </vtl-code>
+    <vtl-dataset name="left">
+country[I,String],population[M,String]
+France , 64M
+Norway , 5M
+Italy  , null
+Sweden , 9M
+    </vtl-dataset>
+    <vtl-data datasets="datasets" errors="errors"></vtl-data>
+</div>
+
 ## Boolean operators
 
 ### Null operators

java-vtl-parser/src/main/antlr4/no/ssb/vtl/parser/VTL.g4

@@ -157,10 +157,17 @@ joinClause : role? identifier ASSIGNMENT joinCalcExpression # joinCalcClause
 joinFoldExpression      : 'fold' componentRef (',' componentRef)* 'to' dimension=identifier ',' measure=identifier ;
 joinUnfoldExpression    : 'unfold' dimension=componentRef ',' measure=componentRef 'to' STRING_CONSTANT (',' STRING_CONSTANT)* ;
 
+conditionalExpression
+    : nvlExpression
+    ;
+
+nvlExpression : 'nvl' '(' componentRef ',' nvlRepValue=constant ')';
+
 // Left recursive
 joinCalcExpression : leftOperand=joinCalcExpression  sign=( '*' | '/' ) rightOperand=joinCalcExpression #joinCalcProduct
                    | leftOperand=joinCalcExpression  sign=( '+' | '-' ) rightOperand=joinCalcExpression #joinCalcSummation
                    | '(' joinCalcExpression ')'                                                         #joinCalcPrecedence
+                   | conditionalExpression                                                              #joinCalcCondition
                    | componentRef                                                                       #joinCalcReference
                    | constant                                                                           #joinCalcAtom
                    | booleanExpression                                                                  #joinCalcBoolean

java-vtl-parser/src/test/java/no/ssb/vtl/parser/ConditionalParserTest.java

@@ -0,0 +1,18 @@
+package no.ssb.vtl.parser;
+
+
+import org.junit.Test;
+
+public class ConditionalParserTest extends GrammarTest {
+
+    @Test
+    public void testNvlWithNumber() throws Exception {
+        parse("nvl(m1 , 0)", "conditionalExpression");
+    }
+
+    @Test
+    public void testNvlComponentRefWithString() throws Exception {
+        parse("nvl(ds1.m2, \"constant\")", "conditionalExpression");
+    }
+
+}

java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/ConditionalExpressionVisitor.java

@@ -0,0 +1,47 @@
+package no.ssb.vtl.script.visitors;
+
+import no.ssb.vtl.model.Component;
+import no.ssb.vtl.model.DataStructure;
+import no.ssb.vtl.model.VTLExpression;
+import no.ssb.vtl.model.VTLObject;
+import no.ssb.vtl.parser.VTLBaseVisitor;
+import no.ssb.vtl.parser.VTLParser;
+import org.antlr.v4.runtime.misc.ParseCancellationException;
+
+import java.util.Map;
+import java.util.Optional;
+
+import static java.lang.String.*;
+
+public class ConditionalExpressionVisitor extends VTLBaseVisitor<VTLExpression> {
+
+    private final ReferenceVisitor referenceVisitor;
+    private final DataStructure dataStructure;
+
+    public ConditionalExpressionVisitor(ReferenceVisitor referenceVisitor, DataStructure dataStructure) {
+        this.referenceVisitor = referenceVisitor;
+        this.dataStructure = dataStructure;
+    }
+
+    @Override
+    public VTLExpression visitNvlExpression(VTLParser.NvlExpressionContext ctx) {
+        ParamVisitor paramVisitor = new ParamVisitor(referenceVisitor);
+        Component input = (Component) paramVisitor.visit(ctx.componentRef());
+        Object repValue = paramVisitor.visit(ctx.nvlRepValue);
+
+        if (input.getType() != repValue.getClass()) {
+            throw new ParseCancellationException("The value to replace null must be of type " + input.getType());
+        }
+
+        return new VTLExpression.Builder(input.getType(), dataPoint -> {
+            Map<Component, VTLObject> map = dataStructure.asMap(dataPoint);
+            Optional<VTLObject> vtlObject = Optional.ofNullable(map.get(input));
+            if (!vtlObject.isPresent() || vtlObject.get().get() == null) {
+                return VTLObject.of(repValue);
+            } else {
+                return vtlObject.get();
+            }
+
+        }).description(format("nvl(%s, %s)", input, repValue)).build();
+    }
+}

java-vtl-script/src/main/java/no/ssb/vtl/script/visitors/join/JoinCalcClauseVisitor.java

@@ -9,6 +9,7 @@
 import no.ssb.vtl.model.VTLPredicate;
 import no.ssb.vtl.parser.VTLParser;
 import no.ssb.vtl.script.visitors.BooleanExpressionVisitor;
+import no.ssb.vtl.script.visitors.ConditionalExpressionVisitor;
 import no.ssb.vtl.script.visitors.ReferenceVisitor;
 import no.ssb.vtl.script.visitors.VTLScalarExpressionVisitor;
 
@@ -24,12 +25,12 @@ public class JoinCalcClauseVisitor extends VTLScalarExpressionVisitor<VTLExpress
 
     private final ReferenceVisitor referenceVisitor;
     private final DataStructure dataStructure;
-    
+
     public JoinCalcClauseVisitor(ReferenceVisitor referenceVisitor, DataStructure dataStructure) {
         this.referenceVisitor = referenceVisitor;
         this.dataStructure = dataStructure;
     }
-    
+
     JoinCalcClauseVisitor() {
         this(null, null);
     }
@@ -76,25 +77,25 @@ public VTLExpression visitJoinCalcSummation(VTLParser.JoinCalcSummationContext c
         // TODO: Check types?
         //checkArgument(Number.class.isAssignableFrom(leftResult.getType()));
         //checkArgument(Number.class.isAssignableFrom(rightResult.getType()));
-    
+
         return new VTLExpression.Builder(Number.class, dataPoint -> {
-        
+
             Number leftNumber = (Number) leftResult.apply(dataPoint).get();
             Number rightNumber = (Number) rightResult.apply(dataPoint).get();
-        
+
             // TODO: Write test
             if (leftNumber == null || rightNumber == null) {
                 return null;
             }
-        
+
             VTLNumber left = VTLNumber.of(leftNumber);
             if (ctx.sign.getText().equals("+")) {
                 return left.add(rightNumber);
             } else {
                 return left.subtract(rightNumber);
             }
-        
-        
+
+
         }).description(format("%s %s %s", leftResult, ctx.sign.getText(), rightResult)).build();
     }
 
@@ -107,25 +108,25 @@ public VTLExpression visitJoinCalcProduct(VTLParser.JoinCalcProductContext ctx)
         // Check types?
         //checkArgument(Number.class.isAssignableFrom(leftResult.getType()));
         //checkArgument(Number.class.isAssignableFrom(rightResult.getType()));
-    
+
         return new VTLExpression.Builder(Number.class, dataPoint -> {
             Number leftNumber = (Number) leftResult.apply(dataPoint).get();
             Number rightNumber = (Number) rightResult.apply(dataPoint).get();
-        
+
             if (leftNumber == null ^ rightNumber == null) {
                 return null;
             }
-        
+
             VTLNumber left = VTLNumber.of(leftNumber);
             if (ctx.sign.getText().equals("*")) {
                 return left.multiply(rightNumber);
             } else {
                 return left.divide(rightNumber);
             }
-            
+
         }).description(format("%s %s %s", leftResult, ctx.sign.getText(), rightResult)).build();
     }
-    
+
     @Override
     public VTLExpression visitJoinCalcBoolean(VTLParser.JoinCalcBooleanContext ctx) {
         BooleanExpressionVisitor booleanVisitor = new BooleanExpressionVisitor(referenceVisitor, dataStructure);
@@ -134,4 +135,11 @@ public VTLExpression visitJoinCalcBoolean(VTLParser.JoinCalcBooleanContext ctx)
                 dataPoint -> VTLBoolean.of(predicate.apply(dataPoint)))
                 .description("boolean").build();
     }
+
+    @Override
+    public VTLExpression visitJoinCalcCondition(VTLParser.JoinCalcConditionContext ctx) {
+        ConditionalExpressionVisitor conditionalExpressionVisitor = new ConditionalExpressionVisitor(
+                referenceVisitor, dataStructure);
+        return conditionalExpressionVisitor.visit(ctx.conditionalExpression());
+    }
 }

java-vtl-script/src/test/java/no/ssb/vtl/script/VTLScriptEngineTest.java

@@ -25,16 +25,17 @@
 import no.ssb.vtl.connector.Connector;
 import no.ssb.vtl.model.Component;
 import no.ssb.vtl.model.DataPoint;
-import no.ssb.vtl.model.Order;
-import no.ssb.vtl.model.VTLObject;
 import no.ssb.vtl.model.DataStructure;
 import no.ssb.vtl.model.Dataset;
+import no.ssb.vtl.model.Order;
+import no.ssb.vtl.model.VTLObject;
 import no.ssb.vtl.script.support.VTLPrintStream;
 import org.junit.Test;
 
 import javax.script.Bindings;
 import javax.script.ScriptContext;
 import javax.script.ScriptEngine;
+import javax.script.ScriptException;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -43,14 +44,12 @@
 import java.util.Optional;
 import java.util.stream.Stream;
 
-import static no.ssb.vtl.model.Component.Role;
-import static no.ssb.vtl.test.ComponentConditions.componentWith;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.entry;
+import static no.ssb.vtl.model.Component.*;
+import static no.ssb.vtl.test.ComponentConditions.*;
+import static org.assertj.core.api.Assertions.*;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
 
 public class VTLScriptEngineTest {
 
@@ -515,6 +514,92 @@ public void testCheckSingleRuleWithJoin() throws Exception {
                 );
     }
 
+    @Test
+    public void testNvlAsClause() throws Exception {
+
+        Dataset ds1 = mock(Dataset.class);
+        DataStructure ds = DataStructure.of(
+                (o, aClass) -> o,
+                "id1", Role.IDENTIFIER, String.class,
+                "m1", Role.MEASURE, Integer.class,
+                "m2", Role.MEASURE, String.class
+        );
+        when(ds1.getDataStructure()).thenReturn(ds);
+        when(ds1.getData()).then(invocation -> Stream.of(
+                tuple(
+                        ds.wrap("id1", "1"),
+                        ds.wrap("m1", 1),
+                        ds.wrap("m2", null)
+                ),
+                tuple(
+                        ds.wrap("id1", "2"),
+                        ds.wrap("m1", null),
+                        ds.wrap("m2", "str2")
+                ),
+                tuple(
+                        ds.wrap("id1", "3"),
+                        ds.wrap("m1", null),
+                        ds.wrap("m2", null)
+                )
+        ));
+
+        bindings.put("ds1", ds1);
+        engine.eval("ds2 := [ds1] {" +
+                "   m11 = nvl(m1 , 0), " +
+                "   m22 = nvl(ds1.m2, \"constant\"), " +
+                "   drop m1, m2 " +
+                "}"
+        );
+
+        assertThat(bindings).containsKey("ds2");
+        Dataset ds2 = (Dataset) bindings.get("ds2");
+
+        assertThat(ds2.getDataStructure().getRoles()).containsOnly(
+                entry("id1", Role.IDENTIFIER),
+                entry("m11", Role.MEASURE),
+                entry("m22", Role.MEASURE)
+        );
+
+        assertThat(ds2.getDataStructure().getTypes()).containsOnly(
+                entry("id1", String.class),
+                entry("m11", Integer.class),
+                entry("m22", String.class)
+        );
+
+        assertThat(ds2.getData())
+                .flatExtracting(input -> input)
+                .extracting(VTLObject::get)
+                .containsExactly(
+                        "1", 1, "constant",
+                        "2", 0, "str2",
+                        "3", 0, "constant"
+                );
+    }
+
+    @Test(expected = ScriptException.class)
+    public void testNvlAsClauseNotEqualTypes() throws Exception {
+
+        Dataset ds1 = mock(Dataset.class);
+        DataStructure ds = DataStructure.of(
+                (o, aClass) -> o,
+                "id1", Role.IDENTIFIER, String.class,
+                "m1", Role.MEASURE, Integer.class
+        );
+        when(ds1.getDataStructure()).thenReturn(ds);
+        when(ds1.getData()).then(invocation -> Stream.of(
+                tuple(
+                        ds.wrap("id1", "1"),
+                        ds.wrap("m1", null)
+                )
+        ));
+
+        bindings.put("ds1", ds1);
+        engine.eval("ds2 := [ds1] {" +
+                "   m11 = nvl(m1 , \"constant\") " +
+                "}"
+        );
+    }
+
     private DataPoint tuple(VTLObject... components) {
         return DataPoint.create(Arrays.asList(components));
     }