diff --git a/ctest4j-junit4/src/main/java/edu/illinois/CTest.java b/ctest4j-common/src/main/java/edu/illinois/CTest.java similarity index 97% rename from ctest4j-junit4/src/main/java/edu/illinois/CTest.java rename to ctest4j-common/src/main/java/edu/illinois/CTest.java index ebfe7be..d2c19a4 100644 --- a/ctest4j-junit4/src/main/java/edu/illinois/CTest.java +++ b/ctest4j-common/src/main/java/edu/illinois/CTest.java @@ -4,9 +4,10 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; + /** * Author: Shuai Wang - * Date: 10/13/23 + * Date: 2/5/24 */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD) @@ -28,4 +29,4 @@ private None() { Class expected() default None.class; long timeout() default 0L; -} \ No newline at end of file +} diff --git a/ctest4j-testng/src/main/java/edu/illinois/CTestClass.java b/ctest4j-common/src/main/java/edu/illinois/CTestClass.java similarity index 96% rename from ctest4j-testng/src/main/java/edu/illinois/CTestClass.java rename to ctest4j-common/src/main/java/edu/illinois/CTestClass.java index 51e7f6c..d73bba4 100644 --- a/ctest4j-testng/src/main/java/edu/illinois/CTestClass.java +++ b/ctest4j-common/src/main/java/edu/illinois/CTestClass.java @@ -7,7 +7,7 @@ /** * Author: Shuai Wang - * Date: 1/18/24 + * Date: 2/5/24 */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) @@ -17,4 +17,4 @@ String[] optional() default {}; String configMappingFile() default ""; String regex() default ""; -} \ No newline at end of file +} diff --git a/ctest4j-common/src/main/java/edu/illinois/CTestRunner.java b/ctest4j-common/src/main/java/edu/illinois/CTestRunner.java index 6917745..368c59f 100644 --- a/ctest4j-common/src/main/java/edu/illinois/CTestRunner.java +++ b/ctest4j-common/src/main/java/edu/illinois/CTestRunner.java @@ -37,6 +37,47 @@ default void startTestClass(String testClassName) { */ default void startTestMethod(String testClassName, String testMethodName) { Utils.setCurTestFullNameToPTid(Utils.getPTid(), testClassName, testMethodName); + ConfigTracker.startTestMethod(testClassName, testMethodName); + } + + default void endTestMethod(ConfigUsage configUsage, String testClassName, String testMethodName, + CTest cTest, Object testAnnotation, + Set classLevelParameters, + Map> methodLevelParametersFromMappingFile) { + if (Options.mode == Modes.BASE) { + return; + } + ConfigUsage.bufferForUpdate(configUsage, testClassName, testMethodName); + if (cTest != null) { + checkConfigurationParameterUsage(testClassName, testMethodName, cTest.regex(), cTest.value(), + cTest.expected(), classLevelParameters, methodLevelParametersFromMappingFile); + } else if (testAnnotation != null) { + Class annotationExpected = null; + if (testAnnotation instanceof org.junit.Test) { + annotationExpected = ((org.junit.Test) testAnnotation).expected(); + } + checkConfigurationParameterUsage(testClassName, testMethodName, "", new String[]{}, + annotationExpected, classLevelParameters, methodLevelParametersFromMappingFile); + } + } + + default void checkConfigurationParameterUsage(String testClassName, String methodName, + String annotationRegex, String[] annotationValue, + Class annotationExpected, + Set classLevelParameters, + Map> methodLevelParametersFromMappingFile) throws UnUsedConfigParamException { + if (Options.mode == Modes.CHECKING || Options.mode == Modes.DEFAULT) { + for (String param : getUnionMethodParameters(testClassName, methodName, annotationRegex, + classLevelParameters, methodLevelParametersFromMappingFile, + new HashSet<>(Arrays.asList(annotationValue)))) { + if (!ConfigTracker.isParameterUsed(testClassName, methodName, param)) { + if (isUnUsedParamException(annotationExpected)) { + return; + } + throw new UnUsedConfigParamException(param + " was not used during the test."); + } + } + } } /** @@ -60,7 +101,8 @@ default Object[] initializeParameterSet(String testClassName, String mappingFile } /** - * Resolve the mapping file path. If the mapping file path is empty, try the default mapping file path. + * Resolve the mapping file path. + * If the mapping file path is empty, try the default mapping file path under the @CONFIG_MAPPING_DIR. */ private String resolveMappingFilePath(String mappingFile, String testClassName) { if (mappingFile.isEmpty()) { @@ -82,10 +124,7 @@ default Set getParametersFromRegex(String regex) { } /** - * Get the parameters from a configuration file. - * @param file the path of the configuration file - * @return the set of parameters that must be used - * @throws IOException if the parsing fails + * Get the configuration parameter name set from a configuration file. */ default Set getParametersFromMappingFile(String file) throws IOException { ConfigurationParser parser = getParser(Utils.getFileType(file)); @@ -95,29 +134,20 @@ default Set getParametersFromMappingFile(String file) throws IOException /** * Get the class-level parameters from a configuration mapping file. */ - default Set getClasssParametersFromMappingFile(String configMappingFile) throws IOException { + default Set getClassParametersFromMappingFile(String configMappingFile) throws IOException { ConfigurationParser parser = getParser(Utils.getFileType(configMappingFile)); return parser.getClassLevelRequiredConfigParam(configMappingFile); } - default Map> getAllMethodLevelParametersFromMappingFile(String configMappingFile) throws IOException { - ConfigurationParser parser = getParser(Utils.getFileType(configMappingFile)); - return parser.getMethodLevelRequiredConfigParam(configMappingFile); - } - /** * Get the method-level parameters from a configuration mapping file. */ - default Set getMethodParametersFromMappingFile(String configMappingFile, String methodName) throws IOException { + default Map> getAllMethodLevelParametersFromMappingFile(String configMappingFile) throws IOException { ConfigurationParser parser = getParser(Utils.getFileType(configMappingFile)); - Map> methodLevelRequiredConfigParam = parser.getMethodLevelRequiredConfigParam(configMappingFile); - if (methodLevelRequiredConfigParam.containsKey(methodName)) { - return methodLevelRequiredConfigParam.get(methodName); - } else { - return new HashSet<>(); - } + return parser.getMethodLevelRequiredConfigParam(configMappingFile); } + /** * Get the parser for a configuration file based on its file type. * @param configFileType the file type of the configuration file @@ -140,18 +170,20 @@ default ConfigurationParser getParser(String configFileType) { } /** + * Ctest configuration parameter usage checking: * Get all the parameters for a test class that every test method in the class must use. * @return a set of parameters that every test method in the class must use */ default Set getUnionClassParameters(Set classLevelParameters, String classConfigFile, String classRegex) throws IOException { - Set result = new HashSet<>(getValueAndRegexClassParameters(classLevelParameters, classRegex)); - if (!classConfigFile.isEmpty()) { - result.addAll(getParametersFromMappingFile(classConfigFile)); + classLevelParameters.addAll(getClassParametersFromMappingFile(classConfigFile)); + if (!classRegex.isEmpty()) { + classLevelParameters.addAll(getParametersFromRegex(classRegex)); } - return result; + return classLevelParameters; } /** + * Ctest configuration parameter usage checking: * Get the parameters from the regex and the parameters from the class-level annotation. */ default Set getValueAndRegexClassParameters(Set classLevelParameters, String classRegex) throws IOException { @@ -163,11 +195,11 @@ default Set getValueAndRegexClassParameters(Set classLevelParame } /** + * Ctest configuration parameter usage checking: * Get all the parameters for a test method. * @return a set of parameters that the test method must use * @throws IOException if the parsing fails */ - default Set getUnionMethodParameters(String testClassName, String methodName, String methodRegex, Set classLevelParameters, Map> methodLevelParametersFromMappingFile, Set methodLevelParamsFromAnnotation) { @@ -190,31 +222,9 @@ default Set getUnionMethodParameters(String testClassName, String method } /** - * Search whether there is a default place that specify the file - * @return the set of parameters that must be used - * @throws IOException if the parsing fails + * Ctest configuration mapping file creation: + * Write the used parameters to a JSON file. */ - default Set getRequiredParametersFromDefaultFile(String className, String methodName) throws IOException { - Set params = new HashSet<>(); - File defaultFile = new File(CONFIG_MAPPING_DIR, Utils.getTestMethodFullName(className, methodName) + ".json"); - if (defaultFile.exists()) { - params.addAll(getParametersFromMappingFile(defaultFile.getAbsolutePath())); - } - return params; - } - -/* - default void checkCTestParameterUsage(Set params) throws UnUsedConfigParamException { - if (Options.mode == Modes.CHECKING || Options.mode == Modes.DEFAULT) { - for (String param : params) { - if (!ConfigTracker.isParameterUsed(param)) { - throw new UnUsedConfigParamException(param + " was not used during the test."); - } - } - } - } -*/ - default void writeConfigUsageToJson(ConfigUsage configUsage, File targetFile) { if (saveUsedParamToFile) { ConfigUsage.updateAllConfigUsage(configUsage); @@ -223,19 +233,28 @@ default void writeConfigUsageToJson(ConfigUsage configUsage, File targetFile) { } /** + * Ctest configuration parameter usage checking: * Check whether the exception is an UnUsedConfigParamException. */ default boolean isUnUsedParamException(Class expected) { + if (expected == null) { + return false; + } return expected.isAssignableFrom(UnUsedConfigParamException.class); } /** + * Ctest configuration parameter usage checking: * Swallow the exception thrown from test method if the exception is an Assertion Error that expects UnUsedConfigParamException. */ - default boolean shouldThorwException(Throwable throwable) { + default boolean shouldThrowException(Throwable throwable) { return throwable != null && !throwable.getMessage().equals("Expected exception: edu.illinois.UnUsedConfigParamException"); } + /** + * Ctest selection: + * Check whether the current test is ignored. + */ default boolean isCurrentTestIgnored(Set targetParams, Set usedParams) { if (Names.CTEST_RUNTIME_SELECTION && !targetParams.isEmpty()) { // return true if none of the parameters in targetParams is used diff --git a/ctest4j-junit4/src/main/java/edu/illinois/CTestClass.java b/ctest4j-junit4/src/main/java/edu/illinois/CTestClass.java deleted file mode 100644 index f8e6a0d..0000000 --- a/ctest4j-junit4/src/main/java/edu/illinois/CTestClass.java +++ /dev/null @@ -1,20 +0,0 @@ -package edu.illinois; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Author: Shuai Wang - * Date: 10/17/23 - */ -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.TYPE) -public @interface CTestClass { - /** A shared list of configuration parameter name for all methods in the test class. */ - String[] value() default {}; - String[] optional() default {}; - String configMappingFile() default ""; - String regex() default ""; -} \ No newline at end of file diff --git a/ctest4j-junit4/src/main/java/edu/illinois/CTestJUnitRunner.java b/ctest4j-junit4/src/main/java/edu/illinois/CTestJUnitRunner.java index ab1b714..d3d7418 100644 --- a/ctest4j-junit4/src/main/java/edu/illinois/CTestJUnitRunner.java +++ b/ctest4j-junit4/src/main/java/edu/illinois/CTestJUnitRunner.java @@ -193,7 +193,6 @@ protected Statement withBefores(FrameworkMethod method, Object target, Statement @Override public void evaluate() throws Throwable { startTestMethod(method.getDeclaringClass().getName(), method.getName()); - ConfigTracker.startTestMethod(method.getDeclaringClass().getName(), method.getName()); originalStatement.evaluate(); } }; @@ -214,40 +213,12 @@ public void evaluate() throws Throwable { } catch (Throwable throwable) { fromTestThrowable = throwable; } finally { - if (shouldThorwException(fromTestThrowable)) { + if (shouldThrowException(fromTestThrowable)) { throw fromTestThrowable; } - ConfigUsage.bufferForUpdate(configUsage, testClassName, method.getName()); - if (Options.mode == Modes.CHECKING || Options.mode == Modes.DEFAULT) { - CTest cTest = method.getAnnotation(CTest.class); - if (cTest != null) { - for (String param : - getUnionMethodParameters(testClassName, method.getName(), cTest.regex(), - classLevelParameters, methodLevelParametersFromMappingFile, - new HashSet<>(Arrays.asList(cTest.value())))) { - if (!ConfigTracker.isParameterUsed(testClassName, method.getName(), param)) { - if (isUnUsedParamException(cTest.expected())) { - return; - } - throw new UnUsedConfigParamException(param + " was not used during the test."); - } - } - } - Test testAnnotation = method.getAnnotation(Test.class); - if (testAnnotation != null) { - for (String param : - getUnionMethodParameters(testClassName, method.getName(), "", - classLevelParameters, methodLevelParametersFromMappingFile, - new HashSet<>())) { - if (!ConfigTracker.isParameterUsed(testClassName, method.getName(), param)) { - if (isUnUsedParamException(testAnnotation.expected())) { - return; - } - throw new UnUsedConfigParamException(param + " was not used during the test."); - } - } - } - } + endTestMethod(configUsage, testClassName, method.getName(), + method.getAnnotation(CTest.class), method.getAnnotation(Test.class), + classLevelParameters, methodLevelParametersFromMappingFile); } } }; @@ -272,14 +243,4 @@ public void evaluate() throws Throwable { } }; } - - @Override - public Set getUnionClassParameters(Set classLevelParameters, String classConfigFile, String classRegex) throws IOException { - classLevelParameters.addAll(getClasssParametersFromMappingFile(classConfigFile)); - if (!classRegex.isEmpty()) { - classLevelParameters.addAll(getParametersFromRegex(classRegex)); - } - return classLevelParameters; - } - } diff --git a/ctest4j-junit5/src/main/java/edu/illinois/CTest.java b/ctest4j-junit5/src/main/java/edu/illinois/CTest.java index 049584d..ffbc5ae 100644 --- a/ctest4j-junit5/src/main/java/edu/illinois/CTest.java +++ b/ctest4j-junit5/src/main/java/edu/illinois/CTest.java @@ -29,4 +29,4 @@ private None() { } Class expected() default None.class; -} +} \ No newline at end of file diff --git a/ctest4j-junit5/src/main/java/edu/illinois/CTestClass.java b/ctest4j-junit5/src/main/java/edu/illinois/CTestClass.java deleted file mode 100644 index 054ffbe..0000000 --- a/ctest4j-junit5/src/main/java/edu/illinois/CTestClass.java +++ /dev/null @@ -1,20 +0,0 @@ -package edu.illinois; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Author: Shuai Wang - * Date: 11/1/23 - */ -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.TYPE) -public @interface CTestClass { - /** A shared list of configuration parameter name for all methods in the test class. */ - String[] value() default {}; - String[] optional() default {}; - String configMappingFile() default ""; - String regex() default ""; -} \ No newline at end of file diff --git a/ctest4j-junit5/src/main/java/edu/illinois/CTestJUnit5Extension.java b/ctest4j-junit5/src/main/java/edu/illinois/CTestJUnit5Extension.java index beb8d1d..78fb55f 100644 --- a/ctest4j-junit5/src/main/java/edu/illinois/CTestJUnit5Extension.java +++ b/ctest4j-junit5/src/main/java/edu/illinois/CTestJUnit5Extension.java @@ -51,7 +51,6 @@ public void beforeEach(ExtensionContext extensionContext) { return; } startTestMethod(extensionContext.getRequiredTestClass().getName(), extensionContext.getRequiredTestMethod().getName()); - ConfigTracker.startTestMethod(extensionContext.getRequiredTestClass().getName(), extensionContext.getRequiredTestMethod().getName()); methodName = extensionContext.getRequiredTestMethod().getName(); } @@ -62,44 +61,10 @@ public void beforeEach(ExtensionContext extensionContext) { */ @Override public void afterEach(ExtensionContext extensionContext) throws IOException { - if (Options.mode == Modes.BASE) { - return; - } - ConfigUsage.bufferForUpdate(configUsage, className, methodName); - if (Options.mode == Modes.CHECKING || Options.mode == Modes.DEFAULT) { - // Retrieve method-level parameters - CTest cTest = extensionContext.getRequiredTestMethod().getAnnotation(CTest.class); - if (cTest != null) { - boolean hasUnusedExpected = isUnUsedParamException(cTest.expected()); - boolean meetUnusedException = false; - for (String param : getUnionMethodParameters(className, methodName, cTest.regex(), - classLevelParameters, methodLevelParametersFromMappingFile, - new HashSet<>(Arrays.asList(cTest.value())))) { - if (!ConfigTracker.isParameterUsed(className, methodName, param)) { - if (hasUnusedExpected) { - meetUnusedException = true; - break; - } - throw new UnUsedConfigParamException(param + " was not used during the test."); - } - } - if (hasUnusedExpected && !meetUnusedException) { - throw new RuntimeException("The test method " + methodName + " does not meet the expected exception " + cTest.expected()); - } - } - Test test = extensionContext.getRequiredTestMethod().getAnnotation(Test.class); - if (test != null) { - Log.INFO(TRACKING_LOG_PREFIX, className + "#" + methodName, - "uses configuration parameters: " + ConfigTracker.getAllUsedParams(className, methodName) + " and set parameters: " + - ConfigTracker.getAllSetParams(className, methodName)); - for (String param : getUnionMethodParameters(className, methodName, "", - classLevelParameters, methodLevelParametersFromMappingFile, new HashSet<>())) { - if (!ConfigTracker.isParameterUsed(className, methodName, param)) { - throw new UnUsedConfigParamException(param + " was not used during the test."); - } - } - } - } + endTestMethod(configUsage, className, methodName, + extensionContext.getRequiredTestMethod().getAnnotation(CTest.class), + extensionContext.getRequiredTestMethod().getAnnotation(Test.class), + classLevelParameters, methodLevelParametersFromMappingFile); } @Override @@ -107,16 +72,6 @@ public void afterAll(ExtensionContext extensionContext) throws Exception { writeConfigUsageToJson(configUsage, new File(CONFIG_SAVE_DIR, className + ".json")); } - @Override - public Set getUnionClassParameters(Set classLevelParameters, String classConfigFile, String classRegex) throws IOException { - classLevelParameters.addAll(getClasssParametersFromMappingFile(classConfigFile)); - if (!classRegex.isEmpty()) { - classLevelParameters.addAll(getParametersFromRegex(classRegex)); - } - return classLevelParameters; - } - - @Override public void initializeRunner(Object context) throws AnnotationFormatError, IOException { ExtensionContext extensionContext = (ExtensionContext) context; diff --git a/ctest4j-testng/src/main/java/edu/illinois/CTest.java b/ctest4j-testng/src/main/java/edu/illinois/CTest.java deleted file mode 100644 index 0337429..0000000 --- a/ctest4j-testng/src/main/java/edu/illinois/CTest.java +++ /dev/null @@ -1,30 +0,0 @@ -package edu.illinois; - - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Author: Shuai Wang - * Date: 1/18/24 - */ -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.METHOD) -public @interface CTest { - /** A list of configuration parameter name as input for a single test method */ - String[] value() default {}; - String[] optional() default {}; - String configMappingFile() default ""; - String regex() default ""; - - static class None extends Throwable { - private static final long serialVersionUID = 1L; - - private None() { - } - } - - Class expected() default None.class; -} \ No newline at end of file diff --git a/ctest4j-testng/src/main/java/edu/illinois/CTestListener.java b/ctest4j-testng/src/main/java/edu/illinois/CTestListener.java index 6e7ee5b..3be46cb 100644 --- a/ctest4j-testng/src/main/java/edu/illinois/CTestListener.java +++ b/ctest4j-testng/src/main/java/edu/illinois/CTestListener.java @@ -46,51 +46,18 @@ public void beforeInvocation(IInvokedMethod method, ITestResult testResult) { return; } startTestMethod(method.getTestMethod().getTestClass().getName(), method.getTestMethod().getMethodName()); - ConfigTracker.startTestMethod(method.getTestMethod().getTestClass().getName(), method.getTestMethod().getMethodName()); methodName = method.getTestMethod().getMethodName(); } @Override public void afterInvocation(IInvokedMethod method, ITestResult testResult) { - if (!method.isTestMethod() || Options.mode == Modes.BASE) { + if (!method.isTestMethod()) { return; } - ConfigUsage.bufferForUpdate(configUsage, className, methodName); - if (Options.mode == Modes.CHECKING || Options.mode == Modes.DEFAULT) { - // Retrieve method-level parameters - CTest cTest = method.getTestMethod().getConstructorOrMethod().getMethod().getAnnotation(CTest.class); - if (cTest != null) { - boolean hasUnusedExpected = isUnUsedParamException(cTest.expected()); - boolean meetUnusedException = false; - for (String param : getUnionMethodParameters(className, methodName, cTest.regex(), - classLevelParameters, methodLevelParametersFromMappingFile, - new HashSet<>(Arrays.asList(cTest.value())))) { - if (!ConfigTracker.isParameterUsed(className, methodName, param)) { - if (hasUnusedExpected) { - meetUnusedException = true; - break; - } - throw new UnUsedConfigParamException(param + " was not used during the test."); - } - } - if (hasUnusedExpected && !meetUnusedException) { - throw new RuntimeException("The test method " + methodName + " does not meet the expected exception " + cTest.expected()); - } - } else { - Test test = method.getTestMethod().getConstructorOrMethod().getMethod().getAnnotation(Test.class); - if (test != null) { - Log.INFO(TRACKING_LOG_PREFIX, className + "#" + methodName, - "uses configuration parameters: " + ConfigTracker.getAllUsedParams(className, methodName) + " and set parameters: " + - ConfigTracker.getAllSetParams(className, methodName)); - for (String param : getUnionMethodParameters(className, methodName, "", - classLevelParameters, methodLevelParametersFromMappingFile, new HashSet<>())) { - if (!ConfigTracker.isParameterUsed(className, methodName, param)) { - throw new UnUsedConfigParamException(param + " was not used during the test."); - } - } - } - } - } + endTestMethod(configUsage, className, methodName, + method.getTestMethod().getConstructorOrMethod().getMethod().getAnnotation(CTest.class), + method.getTestMethod().getConstructorOrMethod().getMethod().getAnnotation(Test.class), + classLevelParameters, methodLevelParametersFromMappingFile); } @Override @@ -98,15 +65,6 @@ public void onAfterClass(ITestClass testClass) { writeConfigUsageToJson(configUsage, new File(CONFIG_SAVE_DIR, className + ".json")); } - @Override - public Set getUnionClassParameters(Set classLevelParameters, String classConfigFile, String classRegex) throws IOException { - classLevelParameters.addAll(getClasssParametersFromMappingFile(classConfigFile)); - if (!classRegex.isEmpty()) { - classLevelParameters.addAll(getParametersFromRegex(classRegex)); - } - return classLevelParameters; - } - @Override public void initializeRunner(Object context) throws AnnotationFormatError, IOException { ITestClass testClass = (ITestClass) context;