diff --git a/Jenkinsfile b/Jenkinsfile index d553a69a2f..d711f95008 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,10 +52,6 @@ node { throw any //rethrow exception to prevent the build from proceeding } finally { - stage ('Clean up workspace'){ - step([$class: 'WsCleanup']) - } - stage('Reporting'){ // Notify on build failure using the Email-ext plugin emailext(body: '${DEFAULT_CONTENT}', mimeType: 'text/html', diff --git a/core/ast/src/main/java/org/overture/ast/messages/InternalException.java b/core/ast/src/main/java/org/overture/ast/messages/InternalException.java index 4480278e4e..9cb5575010 100644 --- a/core/ast/src/main/java/org/overture/ast/messages/InternalException.java +++ b/core/ast/src/main/java/org/overture/ast/messages/InternalException.java @@ -36,6 +36,12 @@ public InternalException(int number, String message) this.number = number; } + public InternalException(int number, String message, Throwable cause) + { + super(message,cause); + this.number = number; + } + @Override public String toString() { diff --git a/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java b/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java index ebe5a4ad63..8c44de8592 100644 --- a/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java +++ b/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java @@ -7,11 +7,14 @@ import org.overture.codegen.assistant.NodeAssistantIR; import org.overture.codegen.ir.IRConstants; import org.overture.codegen.ir.IRGeneratedTag; +import org.overture.codegen.ir.STypeIR; import org.overture.codegen.ir.analysis.AnalysisException; import org.overture.codegen.ir.analysis.DepthFirstAnalysisAdaptor; import org.overture.codegen.ir.declarations.ADefaultClassDeclIR; import org.overture.codegen.ir.declarations.AMethodDeclIR; import org.overture.codegen.ir.declarations.SClassDeclIR; +import org.overture.codegen.ir.statements.APlainCallStmIR; +import org.overture.codegen.ir.types.AClassTypeIR; import org.overture.codegen.trans.assistants.TransAssistantIR; import org.overture.config.Settings; @@ -30,6 +33,12 @@ public class JUnit4Trans extends DepthFirstAnalysisAdaptor public static final String TEST_SETUP_ANNOTATION = "@Before"; public static final String TEST_TEARDOWN_ANNOTATION = "@After"; public static final String JUNI4_IMPORT = "org.junit.*"; + public static final String ASSERT_MODULE = "Assert"; + public static final String ASSERT_TRUE_MSG_METHOD = "assertTrueMsg"; + public static final String ASSERT_FALSE_MSG_METHOD = "assertFalseMsg"; + public static final String JUNIT4_ASSERT_TRUE_METHOD = "assertTrue"; + public static final String JUNIT4_ASSERT_FALSE_METHOD = "assertFalse"; + public TransAssistantIR assist; private JavaCodeGen javaCg; @@ -123,6 +132,39 @@ private void adjustTestClass(ADefaultClassDeclIR copy) { for(AMethodDeclIR m : copy.getMethods()) { m.setStatic(false); + + try + { + m.apply(new DepthFirstAnalysisAdaptor() + { + @Override + public void caseAPlainCallStmIR(APlainCallStmIR node) + throws AnalysisException + { + STypeIR type = node.getClassType(); + + if (type instanceof AClassTypeIR) + { + AClassTypeIR classType = (AClassTypeIR) type; + + if (classType.getName().equals(ASSERT_MODULE)) + { + if (node.getName().equals(ASSERT_FALSE_MSG_METHOD)) + { + node.setName(JUNIT4_ASSERT_FALSE_METHOD); + } else if (node.getName().equals(ASSERT_TRUE_MSG_METHOD)) + { + node.setName(JUNIT4_ASSERT_TRUE_METHOD); + } + } + } + } + }); + } catch (AnalysisException e) + { + log.error("Got unexpected analysis error: " + e.getMessage()); + e.printStackTrace(); + } } for(int i = 0; i < copy.getMethods().size(); i++) diff --git a/core/codegen/platform/src/main/java/org/overture/codegen/ir/IRConstants.java b/core/codegen/platform/src/main/java/org/overture/codegen/ir/IRConstants.java index 5e9e4aa41c..d2c41d2066 100644 --- a/core/codegen/platform/src/main/java/org/overture/codegen/ir/IRConstants.java +++ b/core/codegen/platform/src/main/java/org/overture/codegen/ir/IRConstants.java @@ -38,7 +38,7 @@ public class IRConstants public static final String TEST_MODULE_NAME_POSTFIX = "Test"; - public static final String[] CLASS_NAMES_USED_IN_SL = {"CSV", "IO", "MATH", "VDMUtil"}; + public static final String[] CLASS_NAMES_USED_IN_SL = {"CSV", "IO", "MATH", "VDMUtil", "Assert", "TestRunner"}; public static final String[] CLASS_NAMES_USED_IN_VDM_PP_RT = (String[]) ArrayUtils.addAll(CLASS_NAMES_USED_IN_SL, new String[]{"VDMUnit", "Throwable", "Error", "AssertionFailedError", diff --git a/core/interpreter/src/main/java/TestCase.java b/core/interpreter/src/main/java/TestCase.java index 4be9432b2a..b766821687 100644 --- a/core/interpreter/src/main/java/TestCase.java +++ b/core/interpreter/src/main/java/TestCase.java @@ -1,8 +1,10 @@ +import org.overture.ast.definitions.AExplicitOperationDefinition; +import org.overture.ast.messages.InternalException; +import org.overture.ast.modules.AModuleModules; import org.overture.interpreter.runtime.*; import org.overture.interpreter.values.*; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; @@ -14,6 +16,7 @@ import javax.xml.transform.stream.StreamResult; import javax.xml.xpath.*; import java.io.*; +import java.lang.reflect.InvocationTargetException; public class TestCase { @@ -44,19 +47,17 @@ public static Value reflectionRunTest(Value obj, Value name) success = true; } catch (Exception e) { if (e instanceof ExitException) { - if(((ExitException)e).value.objectValue(null).type.getName().equals("AssertionFailedError")) - { + if (((ExitException) e).value.objectValue(null).type.getName().equals("AssertionFailedError")) { success = false; } throw e; } try { - return ClassInterpreter.getInstance().evaluate("Error`throw(\"" + return ClassInterpreter.getInstance().evaluate("Error`throw(\"" + e.getMessage().replaceAll("\\\\", "\\\\\\\\").replaceAll("\"", "\\\\\"").replaceAll("\'", "\\\'") + "\")", mainContext); - }catch(ExitException e2) - { + } catch (ExitException e2) { error = e2; throw e2; } @@ -81,7 +82,58 @@ public static Value reflectionRunTest(Value obj, Value name) } - private static void recordTestResults(String containerName, String methodName, boolean success, ExitException error, long totalExecTime) throws ParserConfigurationException, IOException, SAXException, XPathExpressionException, TransformerException { + public static boolean reflectionRunTest(AModuleModules module, AExplicitOperationDefinition opDef) throws Exception { + String moduleName = module.getName().getName(); + String testName = opDef.getName().getName(); + + long timerStart = System.nanoTime(); + boolean success = false; + Exception error = null; + try { + ModuleInterpreter.getInstance().evaluate(moduleName + "`" + testName + "()" + , ModuleInterpreter.getInstance().initialContext); + success = !TestRunner.isFailed(); + } catch(TestRunner.TestAsserException e) + { + success =false; + }catch(InternalException e) + { + if(e.getCause() instanceof InvocationTargetException && ((InvocationTargetException)e.getCause()).getTargetException() instanceof TestRunner.TestAsserException) + { + success =false; + }else + { + throw e; + } + } + catch (Exception e) { + success = false; + error = e; + throw e; + } finally { + long totalExecTime = System.nanoTime() - timerStart; + + if (System.getProperty(vdmUnitReportEnable) != null) { + recordTestResults(moduleName, testName, success, error, totalExecTime); + } + } + return success; + } + + + public static void reflectionRun(AModuleModules module, AExplicitOperationDefinition opDef) { + String moduleName = module.getName().getName(); + String testName = opDef.getName().getName(); + + try { + ModuleInterpreter.getInstance().evaluate(moduleName + "`" + testName + "()" + , ModuleInterpreter.getInstance().initialContext); + } catch (Exception e) { + // Fine + } + } + + private static void recordTestResults(String containerName, String methodName, boolean success, Exception error, long totalExecTime) throws ParserConfigurationException, IOException, SAXException, XPathExpressionException, TransformerException { File report = new File("TEST-" + containerName + ".xml"); @@ -115,8 +167,7 @@ private static void recordTestResults(String containerName, String methodName, b n = doc.createElement("testcase"); } - while(n.getFirstChild()!=null) - { + while (n.getFirstChild() != null) { n.removeChild(n.getFirstChild()); } @@ -127,21 +178,30 @@ private static void recordTestResults(String containerName, String methodName, b testSuiteNode.setAttribute("tests", String.valueOf(Integer.parseInt(testSuiteNode.getAttribute("tests")) + 1)); - if (error!=null) { + if (error != null) { testSuiteNode.setAttribute("error", String.valueOf(Integer.parseInt(testSuiteNode.getAttribute("errors")) + 1)); Element errorElement = doc.createElement("error"); - errorElement.setAttribute("message",error.number+""); - errorElement.setAttribute("type","ERROR"); + errorElement.setAttribute("message", error.getMessage() + ""); + errorElement.setAttribute("type", "ERROR"); StringWriter strOut = new StringWriter(); - error.ctxt.printStackTrace(new PrintWriter(strOut),true); + if(error instanceof ExitException) { + ((ExitException)error).ctxt.printStackTrace(new PrintWriter(strOut), true); + }else + { + error.printStackTrace(new PrintWriter(strOut)); + } errorElement.setTextContent(strOut.toString()); n.appendChild(errorElement); } else if (!success) { testSuiteNode.setAttribute("failures", String.valueOf(Integer.parseInt(testSuiteNode.getAttribute("failures")) + 1)); Element failureElement = doc.createElement("failure"); - failureElement.setAttribute("message",methodName); - failureElement.setAttribute("type","WARNING"); + failureElement.setAttribute("message", methodName); + failureElement.setAttribute("type", "WARNING"); failureElement.setAttribute("time", totalExecTime * 1E-9 + ""); + if(TestRunner.getMsg()!=null) + { + failureElement.setTextContent(TestRunner.getMsg()); + } n.appendChild(failureElement); } diff --git a/core/interpreter/src/main/java/TestRunner.java b/core/interpreter/src/main/java/TestRunner.java index 50a1aae924..d605daaf90 100644 --- a/core/interpreter/src/main/java/TestRunner.java +++ b/core/interpreter/src/main/java/TestRunner.java @@ -1,90 +1,243 @@ +import org.overture.ast.definitions.AExplicitOperationDefinition; +import org.overture.ast.definitions.PDefinition; +import org.overture.ast.definitions.SClassDefinition; +import org.overture.ast.modules.AModuleModules; +import org.overture.interpreter.messages.Console; +import org.overture.interpreter.runtime.*; +import org.overture.interpreter.values.*; + +import java.util.LinkedList; import java.util.List; import java.util.Vector; -import org.overture.ast.definitions.SClassDefinition; -import org.overture.interpreter.runtime.*; -import org.overture.interpreter.values.ObjectValue; -import org.overture.interpreter.values.SetValue; -import org.overture.interpreter.values.Value; -import org.overture.interpreter.values.ValueSet; - -public class TestRunner -{ - public static Value collectTests(Value obj) - { - List tests = new Vector(); - ObjectValue instance = (ObjectValue) obj; +public class TestRunner { - if (ClassInterpreter.getInstance() instanceof ClassInterpreter) + public static class TestAsserException extends RuntimeException + { + public TestAsserException(String message) { + super(message); + } + } + + private static boolean fail = false; + private static String msg = null; + + public static Value markFail() { + fail = true; + throw new TestAsserException("Assert message: "+getMsg()); + } + + public static boolean isFailed() + { + return fail; + } + + public static Value setMsg(Value msgVal) + { + if(msgVal instanceof SeqValue) { - for (SClassDefinition def : ((ClassInterpreter) ClassInterpreter.getInstance()).getClasses()) - { - if (def.getIsAbstract() || !isTestClass(def)) - { - continue; - } - tests.add(def.getName().getName()); + try { + msg = msgVal.stringValue(null); + } catch (ValueException e) { + + Console.out.println("\tReceived unexpected message: " + msgVal); } } + + return new VoidValue(); + } - Context mainContext = new StateContext(Interpreter.getInstance().getAssistantFactory(), instance.type.getLocation(), "reflection scope"); + public static String getMsg() + { + return msg; + } + + public static Value collectTests(Value obj) { + List tests = new Vector(); + ObjectValue instance = (ObjectValue) obj; - mainContext.putAll(ClassInterpreter.getInstance().initialContext); - mainContext.setThreadState(ClassInterpreter.getInstance().initialContext.threadState.dbgp, ClassInterpreter.getInstance().initialContext.threadState.CPU); + if (ClassInterpreter.getInstance() instanceof ClassInterpreter) { + for (SClassDefinition def : ((ClassInterpreter) ClassInterpreter.getInstance()).getClasses()) { + if (def.getIsAbstract() || !isTestClass(def)) { + continue; + } + tests.add(def.getName().getName()); + } + } - ValueSet vals = new ValueSet(); - for (String value : tests) - { - try - { - vals.add(ClassInterpreter.getInstance().evaluate("new " + value - + "()", mainContext)); - } catch (Exception e) - { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } + Context mainContext = new StateContext(Interpreter.getInstance().getAssistantFactory(), instance.type.getLocation(), "reflection scope"); - try - { - return new SetValue(vals); - } - catch (ValueException e) - { - return null; // Not reached - } - } + mainContext.putAll(ClassInterpreter.getInstance().initialContext); + mainContext.setThreadState(ClassInterpreter.getInstance().initialContext.threadState.dbgp, ClassInterpreter.getInstance().initialContext.threadState.CPU); - private static boolean isTestClass(SClassDefinition def) - { - if (def.getIsAbstract() || def.getName().getName().equals("Test") - || def.getName().getName().equals("TestCase") - || def.getName().getName().equals("TestSuite")) - { - return false; - } + ValueSet vals = new ValueSet(); + for (String value : tests) { + try { + vals.add(ClassInterpreter.getInstance().evaluate("new " + value + + "()", mainContext)); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } - if (checkForSuper(def, "TestSuite")) - { - // the implementation must be upgrade before this work. - // The upgrade should handle the static method for creatint the suire - return false; - } + try { + return new SetValue(vals); + } catch (ValueException e) { + return null; // Not reached + } + } - return checkForSuper(def, "Test"); - } + private static boolean isTestClass(SClassDefinition def) { + if (def.getIsAbstract() || def.getName().getName().equals("Test") + || def.getName().getName().equals("TestCase") + || def.getName().getName().equals("TestSuite")) { + return false; + } - private static boolean checkForSuper(SClassDefinition def, String superName) - { - for (SClassDefinition superDef : def.getSuperDefs()) + if (checkForSuper(def, "TestSuite")) { + // the implementation must be upgrade before this work. + // The upgrade should handle the static method for creatint the suire + return false; + } + + return checkForSuper(def, "Test"); + } + + private static boolean checkForSuper(SClassDefinition def, String superName) { + for (SClassDefinition superDef : def.getSuperDefs()) { + if (superDef.getName().getName().equals(superName) + || checkForSuper(superDef, superName)) { + return true; + } + } + return false; + } + + + public static Value run() { + //lets find the modules + List tests = new Vector<>(); + + if (ModuleInterpreter.getInstance() instanceof ModuleInterpreter) { + for (AModuleModules def : ((ModuleInterpreter) ModuleInterpreter.getInstance()).getModules()) { + if (!def.getName().getName().endsWith("Test")) { + continue; + } + tests.add(def); + } + } + + int testCount = 0; + int testFailCount = 0; + int testErrorCount = 0; + + for (AModuleModules module : tests) { + + AExplicitOperationDefinition setup = findLifecycleOp(module.getDefs(), "setUp"); + AExplicitOperationDefinition tearDown = findLifecycleOp(module.getDefs(), "tearDown"); + + String moduleName = module.getName().getName(); + for (PDefinition def : module.getDefs()) { + + if (def instanceof AExplicitOperationDefinition && def.getName().getName().startsWith("test")) { + + fail = false; + msg = null; + + boolean tearDownRun = false; + + try { + testCount++; + Console.out.println("Executing test: " + moduleName + "`" + def.getName().getName() + "()"); + + if(setup != null) + { + TestCase.reflectionRun(module, setup); + } + + TestCase.reflectionRunTest(module, (AExplicitOperationDefinition) def); + + tearDownRun = true; + + if(tearDown != null) + { + TestCase.reflectionRun(module, tearDown); + } + + if(fail) + { + testFailCount++; + //TODO: is this the right format? + if(msg != null) + { + Console.out.println("\tFAIL: " + msg); + } + else + { + Console.out.println("\tFAIL"); + } + } + else { + Console.out.println("\tOK"); + } + } catch (Exception e) { + testErrorCount++; + Console.out.println("\tERROR: "+e.getMessage()); + } + finally + { + if (!tearDownRun) + { + if (tearDown != null) + { + try + { + TestCase.reflectionRun(module, tearDown); + } catch (Exception e) + { + e.printStackTrace(); + } + } + } + msg = null; + fail = false; + } + } + } + } + + String header = "----------------------------------------\n" + + "| TEST RESULTS |\n" + + "|--------------------------------------|"; + Console.out.println(header); + Console.out.println(String.format("| Executed: %1$-" + 27 + "s", testCount) + "|"); + Console.out.println(String.format("| Failures: %1$-" + 27 + "s", testFailCount) + "|"); + Console.out.println(String.format("| Errors : %1$-" + 27 + "s", testErrorCount) + "|"); + Console.out.println("|______________________________________|"); + + + if (testErrorCount == 0 && testFailCount == 0) { + Console.out.println("| SUCCESS |"); + } else { + Console.out.println("| FAILURE |"); + } + Console.out.println("|______________________________________|\n"); + + + return new VoidValue(); + } + + private static AExplicitOperationDefinition findLifecycleOp(LinkedList defs, String name) { + + for(PDefinition d : defs) { - if (superDef.getName().getName().equals(superName) - || checkForSuper(superDef, superName)) + if(d instanceof AExplicitOperationDefinition && d.getName().getName().equals(name)) { - return true; + return (AExplicitOperationDefinition) d; } } - return false; + + return null; } } diff --git a/core/interpreter/src/main/java/org/overture/interpreter/util/Delegate.java b/core/interpreter/src/main/java/org/overture/interpreter/util/Delegate.java index b9510e62b5..16f139bb5d 100644 --- a/core/interpreter/src/main/java/org/overture/interpreter/util/Delegate.java +++ b/core/interpreter/src/main/java/org/overture/interpreter/util/Delegate.java @@ -355,7 +355,7 @@ public Value invokeDelegate(Object delegateObject, Context ctxt) strOut.append(e.getTargetException().getMessage()+"\n"); e.getTargetException().printStackTrace(new PrintWriter(strOut)); throw new InternalException(59, "Failed in native method: " - + strOut); + + strOut,e); } }