Merge pull request #43443 from gabilang/fix-exitcode

[2201.8.x] Fix zero exit code when init failures during test execution

compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/methodgen/MainMethodGen.java

@@ -54,8 +54,10 @@
 import static org.objectweb.asm.Opcodes.INVOKESPECIAL;
 import static org.objectweb.asm.Opcodes.INVOKESTATIC;
 import static org.objectweb.asm.Opcodes.INVOKEVIRTUAL;
+import static org.objectweb.asm.Opcodes.LCONST_1;
 import static org.objectweb.asm.Opcodes.NEW;
 import static org.objectweb.asm.Opcodes.PUTFIELD;
+import static org.objectweb.asm.Opcodes.PUTSTATIC;
 import static org.objectweb.asm.Opcodes.RETURN;
 import static org.wso2.ballerinalang.compiler.bir.codegen.JvmConstants.CLI_SPEC;
 import static org.wso2.ballerinalang.compiler.bir.codegen.JvmConstants.COMPATIBILITY_CHECKER;
@@ -90,6 +92,7 @@
 import static org.wso2.ballerinalang.compiler.bir.codegen.JvmConstants.STRING_VALUE;
 import static org.wso2.ballerinalang.compiler.bir.codegen.JvmConstants.TEST_ARGUMENTS;
 import static org.wso2.ballerinalang.compiler.bir.codegen.JvmConstants.TEST_CONFIG_ARGS;
+import static org.wso2.ballerinalang.compiler.bir.codegen.JvmConstants.TEST_EXECUTION_STATE;
 import static org.wso2.ballerinalang.compiler.bir.codegen.JvmConstants.THROWABLE;
 import static org.wso2.ballerinalang.compiler.bir.codegen.JvmConstants.TOML_DETAILS;
 import static org.wso2.ballerinalang.compiler.bir.codegen.JvmSignatures.ADD_SHUTDOWN_HOOK;
@@ -427,27 +430,28 @@ private void genSubmitToScheduler(String initClass, MethodVisitor mv, boolean is
         mv.visitFieldInsn(PUTFIELD, STRAND_CLASS, MethodGenUtils.FRAMES, STACK_FRAMES);
 
         startScheduler(indexMap.get(SCHEDULER_VAR), mv);
-        handleErrorFromFutureValue(mv, isTestFunction);
+        handleErrorFromFutureValue(mv, initClass, isTestFunction);
     }
 
     private void stopListeners(MethodVisitor mv, boolean isServiceEPAvailable) {
         mv.visitLdcInsn(isServiceEPAvailable);
         mv.visitMethodInsn(INVOKESTATIC , LAUNCH_UTILS, "stopListeners", "(Z)V", false);
     }
 
-    private void handleErrorFromFutureValue(MethodVisitor mv, boolean isTestFunction) {
+    private void handleErrorFromFutureValue(MethodVisitor mv, String initClass, boolean isTestFunction) {
         mv.visitVarInsn(ALOAD, indexMap.get(INIT_FUTURE_VAR));
         mv.visitInsn(DUP);
-        mv.visitFieldInsn(GETFIELD , FUTURE_VALUE , PANIC_FIELD, GET_THROWABLE);
+        mv.visitFieldInsn(GETFIELD, FUTURE_VALUE, PANIC_FIELD, GET_THROWABLE);
 
         // handle any runtime errors
         Label labelIf = new Label();
         mv.visitJumpInsn(IFNULL, labelIf);
+        mv.visitFieldInsn(GETFIELD, FUTURE_VALUE, PANIC_FIELD, GET_THROWABLE);
         if (isTestFunction) {
-            mv.visitFieldInsn(GETFIELD , FUTURE_VALUE , PANIC_FIELD, GET_THROWABLE);
             mv.visitMethodInsn(INVOKESTATIC, RUNTIME_UTILS, HANDLE_STOP_PANIC_METHOD, HANDLE_THROWABLE, false);
+            mv.visitInsn(LCONST_1);
+            mv.visitFieldInsn(PUTSTATIC, initClass, TEST_EXECUTION_STATE, "J");
         } else {
-            mv.visitFieldInsn(GETFIELD, FUTURE_VALUE, PANIC_FIELD, GET_THROWABLE);
             mv.visitMethodInsn(INVOKESTATIC, RUNTIME_UTILS, HANDLE_THROWABLE_METHOD, HANDLE_THROWABLE, false);
         }
         mv.visitInsn(RETURN);

tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/SingleTestExecutionWithInitFailuresTest.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com).
+ *
+ * WSO2 LLC. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.ballerinalang.testerina.test;
+
+import org.ballerinalang.test.context.BMainInstance;
+import org.ballerinalang.test.context.BallerinaTestException;
+import org.ballerinalang.testerina.test.utils.AssertionUtils;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+public class SingleTestExecutionWithInitFailuresTest extends BaseTestCase {
+
+    private BMainInstance balClient;
+    private String projectPath;
+
+    @BeforeClass
+    public void setup() throws BallerinaTestException {
+        balClient = new BMainInstance(balServer);
+        projectPath = singleFileTestsPath.resolve("single-test-execution").toString();
+    }
+
+    @Test
+    public void testSingleBalTestExecutionWithInitFailure() throws BallerinaTestException, IOException {
+        String[] args = mergeCoverageArgs(new String[]{"--tests", "testFunc", "bal-test-with-init-failure.bal"});
+        String output = balClient.runMainAndReadStdOut("test", args, new HashMap<>(), projectPath, false);
+        AssertionUtils.assertOutput("SingleFileTestExecutionWithInitFailure.txt", output);
+    }
+}

tests/testerina-integration-test/src/test/java/org/ballerinalang/testerina/test/TestExecutionWithInitFailuresTest.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com).
+ *
+ * WSO2 LLC. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.ballerinalang.testerina.test;
+
+import org.ballerinalang.test.context.BMainInstance;
+import org.ballerinalang.test.context.BallerinaTestException;
+import org.ballerinalang.testerina.test.utils.AssertionUtils;
+import org.ballerinalang.testerina.test.utils.CommonUtils;
+import org.ballerinalang.testerina.test.utils.FileUtils;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashMap;
+
+import static org.ballerinalang.testerina.test.BaseTestCase.balServer;
+import static org.ballerinalang.testerina.test.BaseTestCase.projectBasedTestsPath;
+
+public class TestExecutionWithInitFailuresTest {
+
+    private BMainInstance balClient;
+    private String projectPath;
+
+    @BeforeClass
+    public void setup() throws BallerinaTestException {
+        balClient = new BMainInstance(balServer);
+        projectPath = projectBasedTestsPath.resolve("test-execution-with-init-failure").toString();
+    }
+
+    @Test()
+    public void testModuleExecutionFlow() throws BallerinaTestException, IOException {
+        String[] args = new String[]{};
+        String output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        String firstString = "tests.test_execute-generated_";
+        String endString = "lineNumber";
+        output = CommonUtils.replaceVaryingString(firstString, endString, output);
+        AssertionUtils.assertOutput("TestExecutionWithInitFailures.txt", output);
+    }
+
+    @AfterMethod
+    public void copyExec() {
+        try {
+            FileUtils.copyBallerinaExec(Paths.get(projectPath), String.valueOf(System.currentTimeMillis()));
+        } catch (IOException e) {
+            // ignore exception
+        }
+    }
+}

tests/testerina-integration-test/src/test/resources/command-outputs/unix/SingleFileTestExecutionWithInitFailure.txt

@@ -0,0 +1,10 @@
+Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run...
+warning: ignoring --includes flag since code coverage is not enabled
+Compiling source
+	bal-test-with-init-failure.bal
+
+Running Tests
+
+	bal-test-with-init-failure.bal
+error: {ballerina}DivisionByZero {"message":" / by zero"}
+error: there are test failures

tests/testerina-integration-test/src/test/resources/command-outputs/unix/TestExecutionWithInitFailures.txt

@@ -0,0 +1,30 @@
+Compiling source
+	wso2/testExecutionWithModuleInitFailure:0.0.0
+
+Running Tests
+
+	testExecutionWithModuleInitFailure.moduleD
+		[pass] testFunc
+
+
+		1 passing
+		0 failing
+		0 skipped
+
+	testExecutionWithModuleInitFailure
+error: {ballerina}DivisionByZero {"message":" / by zero"}
+
+	testExecutionWithModuleInitFailure.moduleA
+error: {ballerina}DivisionByZero {"message":" / by zero"}
+
+	testExecutionWithModuleInitFailure.moduleC
+		[pass] testFunc
+
+
+		1 passing
+		0 failing
+		0 skipped
+
+	testExecutionWithModuleInitFailure.moduleB
+error: {ballerina}DivisionByZero {"message":" / by zero"}
+error: there are test failures

tests/testerina-integration-test/src/test/resources/command-outputs/windows/SingleFileTestExecutionWithInitFailure.txt

@@ -0,0 +1,10 @@
+Code coverage is not yet supported with single bal files. Ignoring the flag and continuing the test run...
+warning: ignoring --includes flag since code coverage is not enabled
+Compiling source
+	bal-test-with-init-failure.bal
+
+Running Tests
+
+	bal-test-with-init-failure.bal
+error: {ballerina}DivisionByZero {"message":" / by zero"}
+error: there are test failures

tests/testerina-integration-test/src/test/resources/command-outputs/windows/TestExecutionWithInitFailures.txt

@@ -0,0 +1,30 @@
+Compiling source
+	wso2/testExecutionWithModuleInitFailure:0.0.0
+
+Running Tests
+
+	testExecutionWithModuleInitFailure.moduleD
+		[pass] testFunc
+
+
+		1 passing
+		0 failing
+		0 skipped
+
+	testExecutionWithModuleInitFailure
+error: {ballerina}DivisionByZero {"message":" / by zero"}
+
+	testExecutionWithModuleInitFailure.moduleA
+error: {ballerina}DivisionByZero {"message":" / by zero"}
+
+	testExecutionWithModuleInitFailure.moduleC
+		[pass] testFunc
+
+
+		1 passing
+		0 failing
+		0 skipped
+
+	testExecutionWithModuleInitFailure.moduleB
+error: {ballerina}DivisionByZero {"message":" / by zero"}
+error: there are test failures

tests/testerina-integration-test/src/test/resources/project-based-tests/test-execution-with-init-failure/Ballerina.toml

@@ -0,0 +1,7 @@
+[package]
+org = "wso2"
+name = "testExecutionWithModuleInitFailure"
+version = "0.0.0"
+
+[build-options]
+observabilityIncluded = false

tests/testerina-integration-test/src/test/resources/project-based-tests/test-execution-with-init-failure/main.bal

@@ -0,0 +1,29 @@
+// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import testExecutionWithModuleInitFailure.moduleA;
+import testExecutionWithModuleInitFailure.moduleD;
+
+public function main() {
+}
+
+public function getGreeting(string s1, string s2) returns string {
+    return s1 + moduleA:addSpaceAndGetString(s2);
+}
+
+public function getKey(int i) returns int {
+    return i * moduleD:add(i, i*i);
+}

tests/testerina-integration-test/src/test/resources/project-based-tests/test-execution-with-init-failure/modules/moduleA/main.bal

@@ -0,0 +1,24 @@
+// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import testExecutionWithModuleInitFailure.moduleB;
+
+public function main() {
+}
+
+public function addSpaceAndGetString(string s) returns string {
+    return " " + moduleB:add(s);
+}

tests/testerina-integration-test/src/test/resources/project-based-tests/test-execution-with-init-failure/modules/moduleA/tests/test.bal

@@ -0,0 +1,22 @@
+// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import ballerina/test;
+
+@test:Config {}
+public function testFunc() {
+    test:assertEquals(addSpaceAndGetString("World"), " World!");
+}

tests/testerina-integration-test/src/test/resources/project-based-tests/test-execution-with-init-failure/modules/moduleB/main.bal

@@ -0,0 +1,26 @@
+// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import testExecutionWithModuleInitFailure.moduleC;
+
+int a = 1/0;
+
+public function main() {
+}
+
+public function add(string s) returns string {
+    return s + moduleC:foo();
+}

tests/testerina-integration-test/src/test/resources/project-based-tests/test-execution-with-init-failure/modules/moduleB/tests/test.bal

@@ -0,0 +1,22 @@
+// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import ballerina/test;
+
+@test:Config {}
+function testFunc() {
+   test:assertEquals(add("World"), "World!");
+}