[WIP] Clean Up Ctest4J Runner Interface and Merge Common code to Runner Interface

ctest4j-junit4/src/main/java/edu/illinois/CTest.java → ctest4j-common/src/main/java/edu/illinois/CTest.java

@@ -4,9 +4,10 @@
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
+
 /**
  * Author: Shuai Wang
- * Date: 10/13/23
+ * Date:  2/5/24
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.METHOD)
@@ -28,4 +29,4 @@ private None() {
     Class<? extends Throwable> expected() default None.class;
 
     long timeout() default 0L;
-}
+}

ctest4j-testng/src/main/java/edu/illinois/CTestClass.java → ctest4j-common/src/main/java/edu/illinois/CTestClass.java

@@ -7,7 +7,7 @@
 
 /**
  * Author: Shuai Wang
- * Date:  1/18/24
+ * Date:  2/5/24
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
@@ -17,4 +17,4 @@
     String[] optional() default {};
     String configMappingFile() default "";
     String regex() default "";
-}
+}

ctest4j-common/src/main/java/edu/illinois/CTestRunner.java

@@ -37,6 +37,47 @@ default void startTestClass(String testClassName) {
      */
     default void startTestMethod(String testClassName, String testMethodName) {
         Utils.setCurTestFullNameToPTid(Utils.getPTid(), testClassName, testMethodName);
+        ConfigTracker.startTestMethod(testClassName, testMethodName);
+    }
+
+    default void endTestMethod(ConfigUsage configUsage, String testClassName, String testMethodName,
+                               CTest cTest, Object testAnnotation,
+                               Set<String> classLevelParameters,
+                               Map<String, Set<String>> methodLevelParametersFromMappingFile) {
+        if (Options.mode == Modes.BASE) {
+            return;
+        }
+        ConfigUsage.bufferForUpdate(configUsage, testClassName, testMethodName);
+        if (cTest != null) {
+            checkConfigurationParameterUsage(testClassName, testMethodName, cTest.regex(), cTest.value(),
+                    cTest.expected(), classLevelParameters, methodLevelParametersFromMappingFile);
+        } else if (testAnnotation != null) {
+            Class<? extends Throwable> annotationExpected = null;
+            if (testAnnotation instanceof org.junit.Test) {
+                annotationExpected = ((org.junit.Test) testAnnotation).expected();
+            }
+            checkConfigurationParameterUsage(testClassName, testMethodName, "", new String[]{},
+                    annotationExpected, classLevelParameters, methodLevelParametersFromMappingFile);
+        }
+    }
+
+    default void checkConfigurationParameterUsage(String testClassName, String methodName,
+                                                  String annotationRegex, String[] annotationValue,
+                                                  Class<? extends Throwable> annotationExpected,
+                                                  Set<String> classLevelParameters,
+                                                  Map<String, Set<String>> methodLevelParametersFromMappingFile) throws UnUsedConfigParamException {
+        if (Options.mode == Modes.CHECKING || Options.mode == Modes.DEFAULT) {
+            for (String param : getUnionMethodParameters(testClassName, methodName, annotationRegex,
+                            classLevelParameters, methodLevelParametersFromMappingFile,
+                            new HashSet<>(Arrays.asList(annotationValue)))) {
+                if (!ConfigTracker.isParameterUsed(testClassName, methodName, param)) {
+                    if (isUnUsedParamException(annotationExpected)) {
+                        return;
+                    }
+                    throw new UnUsedConfigParamException(param + " was not used during the test.");
+                }
+            }
+        }
     }
 
     /**
@@ -60,7 +101,8 @@ default Object[] initializeParameterSet(String testClassName, String mappingFile
     }
 
     /**
-     * Resolve the mapping file path. If the mapping file path is empty, try the default mapping file path.
+     * Resolve the mapping file path.
+     * If the mapping file path is empty, try the default mapping file path under the @CONFIG_MAPPING_DIR.
      */
     private String resolveMappingFilePath(String mappingFile, String testClassName) {
         if (mappingFile.isEmpty()) {
@@ -82,10 +124,7 @@ default Set<String> getParametersFromRegex(String regex) {
     }
 
     /**
-     * Get the parameters from a configuration file.
-     * @param file the path of the configuration file
-     * @return the set of parameters that must be used
-     * @throws IOException if the parsing fails
+     * Get the configuration parameter name set from a configuration file.
      */
     default Set<String> getParametersFromMappingFile(String file) throws IOException {
         ConfigurationParser parser = getParser(Utils.getFileType(file));
@@ -95,29 +134,20 @@ default Set<String> getParametersFromMappingFile(String file) throws IOException
     /**
      * Get the class-level parameters from a configuration mapping file.
      */
-    default Set<String> getClasssParametersFromMappingFile(String configMappingFile) throws IOException {
+    default Set<String> getClassParametersFromMappingFile(String configMappingFile) throws IOException {
         ConfigurationParser parser = getParser(Utils.getFileType(configMappingFile));
         return parser.getClassLevelRequiredConfigParam(configMappingFile);
     }
 
-    default Map<String, Set<String>> getAllMethodLevelParametersFromMappingFile(String configMappingFile) throws IOException {
-        ConfigurationParser parser = getParser(Utils.getFileType(configMappingFile));
-        return parser.getMethodLevelRequiredConfigParam(configMappingFile);
-    }
-
     /**
      * Get the method-level parameters from a configuration mapping file.
      */
-    default Set<String> getMethodParametersFromMappingFile(String configMappingFile, String methodName) throws IOException {
+    default Map<String, Set<String>> getAllMethodLevelParametersFromMappingFile(String configMappingFile) throws IOException {
         ConfigurationParser parser = getParser(Utils.getFileType(configMappingFile));
-        Map<String, Set<String>> methodLevelRequiredConfigParam = parser.getMethodLevelRequiredConfigParam(configMappingFile);
-        if (methodLevelRequiredConfigParam.containsKey(methodName)) {
-            return methodLevelRequiredConfigParam.get(methodName);
-        } else {
-            return new HashSet<>();
-        }
+        return parser.getMethodLevelRequiredConfigParam(configMappingFile);
     }
 
+
     /**
      * Get the parser for a configuration file based on its file type.
      * @param configFileType the file type of the configuration file
@@ -140,18 +170,20 @@ default ConfigurationParser getParser(String configFileType) {
     }
 
     /**
+     * Ctest configuration parameter usage checking:
      * Get all the parameters for a test class that every test method in the class must use.
      * @return a set of parameters that every test method in the class must use
      */
     default Set<String> getUnionClassParameters(Set<String> classLevelParameters, String classConfigFile, String classRegex) throws IOException {
-        Set<String> result = new HashSet<>(getValueAndRegexClassParameters(classLevelParameters, classRegex));
-        if (!classConfigFile.isEmpty()) {
-            result.addAll(getParametersFromMappingFile(classConfigFile));
+        classLevelParameters.addAll(getClassParametersFromMappingFile(classConfigFile));
+        if (!classRegex.isEmpty()) {
+            classLevelParameters.addAll(getParametersFromRegex(classRegex));
         }
-        return result;
+        return classLevelParameters;
     }
 
     /**
+     * Ctest configuration parameter usage checking:
      * Get the parameters from the regex and the parameters from the class-level annotation.
      */
     default Set<String> getValueAndRegexClassParameters(Set<String> classLevelParameters, String classRegex) throws IOException {
@@ -163,11 +195,11 @@ default Set<String> getValueAndRegexClassParameters(Set<String> classLevelParame
     }
 
     /**
+     * Ctest configuration parameter usage checking:
      * Get all the parameters for a test method.
      * @return a set of parameters that the test method must use
      * @throws IOException if the parsing fails
      */
-
     default Set<String> getUnionMethodParameters(String testClassName, String methodName, String methodRegex,
                                                  Set<String> classLevelParameters, Map<String, Set<String>> methodLevelParametersFromMappingFile,
                                                  Set<String> methodLevelParamsFromAnnotation) {
@@ -190,31 +222,9 @@ default Set<String> getUnionMethodParameters(String testClassName, String method
     }
 
     /**
-     * Search whether there is a default place that specify the file
-     * @return the set of parameters that must be used
-     * @throws IOException if the parsing fails
+     * Ctest configuration mapping file creation:
+     * Write the used parameters to a JSON file.
      */
-    default Set<String> getRequiredParametersFromDefaultFile(String className, String methodName) throws IOException {
-        Set<String> params = new HashSet<>();
-        File defaultFile = new File(CONFIG_MAPPING_DIR, Utils.getTestMethodFullName(className, methodName) + ".json");
-        if (defaultFile.exists()) {
-            params.addAll(getParametersFromMappingFile(defaultFile.getAbsolutePath()));
-        }
-        return params;
-    }
-
-/*
-    default void checkCTestParameterUsage(Set<String> params) throws UnUsedConfigParamException {
-        if (Options.mode == Modes.CHECKING || Options.mode == Modes.DEFAULT) {
-            for (String param : params) {
-                if (!ConfigTracker.isParameterUsed(param)) {
-                    throw new UnUsedConfigParamException(param + " was not used during the test.");
-                }
-            }
-        }
-    }
-*/
-
     default void writeConfigUsageToJson(ConfigUsage configUsage, File targetFile) {
         if (saveUsedParamToFile) {
             ConfigUsage.updateAllConfigUsage(configUsage);
@@ -223,19 +233,28 @@ default void writeConfigUsageToJson(ConfigUsage configUsage, File targetFile) {
     }
 
     /**
+     * Ctest configuration parameter usage checking:
      * Check whether the exception is an UnUsedConfigParamException.
      */
     default boolean isUnUsedParamException(Class<? extends Throwable> expected) {
+        if (expected == null) {
+            return false;
+        }
         return expected.isAssignableFrom(UnUsedConfigParamException.class);
     }
 
     /**
+     * Ctest configuration parameter usage checking:
      * Swallow the exception thrown from test method if the exception is an Assertion Error that expects UnUsedConfigParamException.
      */
-    default boolean shouldThorwException(Throwable throwable) {
+    default boolean shouldThrowException(Throwable throwable) {
         return throwable != null && !throwable.getMessage().equals("Expected exception: edu.illinois.UnUsedConfigParamException");
     }
 
+    /**
+     * Ctest selection:
+     * Check whether the current test is ignored.
+     */
     default boolean isCurrentTestIgnored(Set<String> targetParams, Set<String> usedParams) {
         if (Names.CTEST_RUNTIME_SELECTION && !targetParams.isEmpty()) {
             // return true if none of the parameters in targetParams is used

ctest4j-junit4/src/main/java/edu/illinois/CTestJUnitRunner.java

@@ -193,7 +193,6 @@ protected Statement withBefores(FrameworkMethod method, Object target, Statement
             @Override
             public void evaluate() throws Throwable {
                 startTestMethod(method.getDeclaringClass().getName(), method.getName());
-                ConfigTracker.startTestMethod(method.getDeclaringClass().getName(), method.getName());
                 originalStatement.evaluate();
             }
         };
@@ -214,40 +213,12 @@ public void evaluate() throws Throwable {
                 } catch (Throwable throwable) {
                     fromTestThrowable = throwable;
                 } finally {
-                    if (shouldThorwException(fromTestThrowable)) {
+                    if (shouldThrowException(fromTestThrowable)) {
                         throw fromTestThrowable;
                     }
-                    ConfigUsage.bufferForUpdate(configUsage, testClassName, method.getName());
-                    if (Options.mode == Modes.CHECKING || Options.mode == Modes.DEFAULT) {
-                        CTest cTest = method.getAnnotation(CTest.class);
-                        if (cTest != null) {
-                            for (String param :
-                                    getUnionMethodParameters(testClassName, method.getName(), cTest.regex(),
-                                            classLevelParameters, methodLevelParametersFromMappingFile,
-                                            new HashSet<>(Arrays.asList(cTest.value())))) {
-                                if (!ConfigTracker.isParameterUsed(testClassName, method.getName(), param)) {
-                                    if (isUnUsedParamException(cTest.expected())) {
-                                        return;
-                                    }
-                                    throw new UnUsedConfigParamException(param + " was not used during the test.");
-                                }
-                            }
-                        }
-                        Test testAnnotation = method.getAnnotation(Test.class);
-                        if (testAnnotation != null) {
-                            for (String param :
-                                    getUnionMethodParameters(testClassName, method.getName(), "",
-                                            classLevelParameters, methodLevelParametersFromMappingFile,
-                                            new HashSet<>())) {
-                                if (!ConfigTracker.isParameterUsed(testClassName, method.getName(), param)) {
-                                    if (isUnUsedParamException(testAnnotation.expected())) {
-                                        return;
-                                    }
-                                    throw new UnUsedConfigParamException(param + " was not used during the test.");
-                                }
-                            }
-                        }
-                    }
+                    endTestMethod(configUsage, testClassName, method.getName(),
+                            method.getAnnotation(CTest.class), method.getAnnotation(Test.class),
+                            classLevelParameters, methodLevelParametersFromMappingFile);
                 }
             }
         };
@@ -272,14 +243,4 @@ public void evaluate() throws Throwable {
             }
         };
     }
-
-    @Override
-    public Set<String> getUnionClassParameters(Set<String> classLevelParameters, String classConfigFile, String classRegex) throws IOException {
-        classLevelParameters.addAll(getClasssParametersFromMappingFile(classConfigFile));
-        if (!classRegex.isEmpty()) {
-            classLevelParameters.addAll(getParametersFromRegex(classRegex));
-        }
-        return classLevelParameters;
-    }
-
 }

ctest4j-junit5/src/main/java/edu/illinois/CTest.java

@@ -29,4 +29,4 @@ private None() {
     }
 
     Class<? extends Throwable> expected() default None.class;
-}
+}