Support regex in feature tag

- modified testInfo parser - added new tests for feature tag - fix issue with test return value Signed-off-by: renfeiw <[email protected]>
adoptium · Sep 13, 2023 · 260cdcb · 260cdcb
1 parent cf7bb69
commit 260cdcb
Show file tree

Hide file tree

Showing 6 changed files with 159 additions and 21 deletions.
diff --git a/.gitignore b/.gitignore
@@ -19,6 +19,7 @@ bin
 output_*
 resources/TRSS
 scripts/testBot/__pycache__
+scripts/testTKG/__pycache__
 
 # misc auto generated files
 autoGen.mk

diff --git a/examples/feature/playlist.xml b/examples/feature/playlist.xml
@@ -0,0 +1,51 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!--
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+<playlist xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../TKG/playlist.xsd">
+	<test>
+		<testCaseName>testFeatureFips</testCaseName>
+		<command>echo "match FIPS140_3_20220501 and everything else by default"; \
+	$(TEST_STATUS)</command>
+		<features>
+			<feature>FIPS140_3_20220501:applicable</feature>
+		</features>
+	</test>
+
+	<test>
+		<testCaseName>testFeatureFipsNoApplicable</testCaseName>
+		<command>echo "test excluded on FIPS140_2"; \
+	$(TEST_STATUS)</command>
+		<features>
+			<feature>FIPS140_2:nonapplicable</feature>
+		</features>
+	</test>
+
+	<test>
+		<testCaseName>testFeatureFipsRequired</testCaseName>
+		<command>echo "only run on FIPS140_3_20220101"; \
+	$(TEST_STATUS)</command>
+		<features>
+			<feature>FIPS140_3_20220101:required</feature>
+		</features>
+	</test>
+
+	<test>
+		<testCaseName>testFeatureFipsRegexRequired</testCaseName>
+		<command>echo "only run on FIPS140_3 from 2022 Dec 01 to 2029 Dec 31"; \
+	$(TEST_STATUS)</command>
+		<features>
+			<feature>/FIPS140_3_(2022(12\d(2))|202[3-9]\d{4})/:required</feature>
+		</features>
+	</test>
+</playlist>
diff --git a/scripts/testTKG/test_base.py b/scripts/testTKG/test_base.py
@@ -23,12 +23,12 @@ def run_test():
     command = "make _all"
     print(f"\t{command}")
     result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
-    rt = checkResult(result, {'testSuccess_0'}, {'testFail_0'}, set(), set())
+    rt &= checkResult(result, {'testSuccess_0'}, {'testFail_0'}, set(), set())
 
     command = "make _testSuccess"
     print(f"\t{command}")
     result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
-    rt = checkResult(result, {'testSuccess_0'}, set(), set(), set())
+    rt &= checkResult(result, {'testSuccess_0'}, set(), set(), set())
     return rt
 
 if __name__ == "__main__":

diff --git a/scripts/testTKG/test_feature.py b/scripts/testTKG/test_feature.py
@@ -0,0 +1,56 @@
+################################################################################
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import subprocess
+from utils import *
+
+def run_test():
+    rt = True
+    printTestheader("feature")
+
+    buildList = "TKG/examples/feature"
+    command = "make _all"
+    print(f"\t{command}")
+    result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
+    rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0'}, set(), set(), set())
+
+    buildList = "TKG/examples/feature"
+    command = "make _all"
+    testFlag = "export TEST_FLAG=FIPS140_2"
+    print(f"\t{testFlag}; {command}")
+    result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
+    rt &= checkResult(result, {'testFeatureFips_0'}, set(), set(), set())
+
+    command = "make _all"
+    testFlag = "export TEST_FLAG=FIPS140_3_20220501"
+    print(f"\t{testFlag}; {command}")
+    result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
+    rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0'}, set(), set(), set())
+
+    command = "make _all"
+    testFlag = "export TEST_FLAG=FIPS140_3_20230511"
+    print(f"\t{testFlag}; {command}")
+    result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
+    rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0', 'testFeatureFipsRegexRequired_0'}, set(), set(), set())
+
+    command = "make _all"
+    testFlag = "export TEST_FLAG=FIPS140_3_20220101"
+    print(f"\t{testFlag}; {command}")
+    result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
+    rt &= checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0', 'testFeatureFipsRequired_0'}, set(), set(), set())
+
+    return rt
+
+if __name__ == "__main__":
+    run_test()
diff --git a/scripts/testTKG/test_level.py b/scripts/testTKG/test_level.py
@@ -24,27 +24,27 @@ def run_test():
     command = "make _sanity"
     print(f"\t{command}")
     result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
-    rt = checkResult(result, {'testSanity_0'}, set(), set(), set())
+    rt &= checkResult(result, {'testSanity_0'}, set(), set(), set())
 
     command = "make _extended"
     print(f"\t{command}")
     result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
-    rt = checkResult(result, {'testDefault_0', 'testExtended_0'}, set(), set(), set())
+    rt &= checkResult(result, {'testDefault_0', 'testExtended_0'}, set(), set(), set())
 
     command = "make _dev"
     print(f"\t{command}")
     result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
-    rt = checkResult(result, {'testDev_0'}, set(), set(), set())
+    rt &= checkResult(result, {'testDev_0'}, set(), set(), set())
 
     command = "make _special"
     print(f"\t{command}")
     result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
-    rt = checkResult(result, {'testSpecial_0'}, set(), set(), set())
+    rt &= checkResult(result, {'testSpecial_0'}, set(), set(), set())
 
     command = "make _all"
     print(f"\t{command}")
     result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False)
-    rt = checkResult(result, {'testSanity_0', 'testDefault_0', 'testExtended_0', 'testDev_0', 'testSpecial_0'}, set(), set(), set())
+    rt &= checkResult(result, {'testSanity_0', 'testDefault_0', 'testExtended_0', 'testDev_0', 'testSpecial_0'}, set(), set(), set())
     return rt
 
 if __name__ == "__main__":

diff --git a/src/org/testKitGen/TestInfoParser.java b/src/org/testKitGen/TestInfoParser.java
@@ -88,31 +88,37 @@ public TestInfo parse() {
 		}
 		Set<String> testFlags = new HashSet<>(arg.getTestFlag());
 		for (Map.Entry<String,String> entry : ti.getFeatures().entrySet()) {
-			if (entry.getValue().equalsIgnoreCase("required")) {
-				if (!testFlags.contains(entry.getKey())) {
+			String featureOpt = entry.getValue().toLowerCase();
+			if (featureOpt.equals("required")) {
+				if (!isFeatureInTestFlags(testFlags, entry.getKey())) {
 					return null;
-				} else if (entry.getKey().equalsIgnoreCase("aot")) {
-					ti.setAotOptions("$(AOT_OPTIONS) ");
 				}
-			} else if (entry.getValue().equalsIgnoreCase("applicable")) {
-				if (testFlags.contains("aot") && (entry.getKey().equalsIgnoreCase("aot") || entry.getKey().equalsIgnoreCase("all"))) {
-					ti.setAotOptions("$(AOT_OPTIONS) ");
-				}
-			} else if (entry.getValue().equalsIgnoreCase("nonapplicable")) {
+			} else if (featureOpt.equals("nonapplicable")) {
 				// Do not generate make target if the test is not applicable for one feature defined in TEST_FLAG
-				if (testFlags.contains(entry.getKey())) {
+				if (isFeatureInTestFlags(testFlags, entry.getKey())) {
 					return null;
 				}
-			} else if (entry.getValue().equalsIgnoreCase("explicit")) {
-				if (testFlags.contains("aot") && entry.getKey().equalsIgnoreCase("aot")) {
-					ti.setAotIterations(1);
-				}
+			} else if (featureOpt.equals("applicable") || featureOpt.equals("explicit")) {
+				// Do nothing
 			} else {
 				System.err.println("Error: Please provide a valid feature parameter in test " + ti.getTestCaseName() + ". The valid string is <feature_name>:[required|applicable|nonapplicable|explicit].");
 				System.exit(1);
 			}
 		}
 
+		if (testFlags.contains("aot")) {
+			for (Map.Entry<String,String> entry : ti.getFeatures().entrySet()) {
+				if (doesFeatureMatchTestFlag("aot", entry.getKey())) {
+					String featureOpt = entry.getValue().toLowerCase();
+					if (featureOpt.equals("required") || featureOpt.equals("applicable")) {
+						ti.setAotOptions("$(AOT_OPTIONS) ");
+					} else if (featureOpt.equals("explicit")) {
+						ti.setAotIterations(1);
+					}
+				}
+			}
+		}
+
 		String rerun = getImmediateChildContent(testEle, "rerun");
 		if (rerun != null) {
 			if (rerun.equals("true")) {
@@ -207,6 +213,30 @@ public TestInfo parse() {
 		return ti;
 	}
 
+	private boolean isFeatureInTestFlags(Set<String> testFlags, String feature) {
+		for (String testFlag : testFlags) {
+			if (doesFeatureMatchTestFlag(testFlag, feature)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	private boolean doesFeatureMatchTestFlag(String testFlag, String feature) {
+		if (feature.equals("all")) {
+			return true;
+		}
+		if (!feature.startsWith("/") || !feature.endsWith("/")) {
+			return testFlag.equalsIgnoreCase(feature);
+		}
+		Pattern pattern = Pattern.compile(feature.substring(1, feature.length() - 1));
+		Matcher matcher = pattern.matcher(testFlag);
+		if (matcher.matches()) {
+			return true;
+		}
+		return false;
+	}
+
 	private boolean checkJavaVersion(String version) {
 		if (version.equalsIgnoreCase(arg.getJdkVersion())) {
 			return true;