From 66787385fbd886eb25a85df0eae50c37d469a8c3 Mon Sep 17 00:00:00 2001 From: renfeiw Date: Wed, 6 Sep 2023 12:13:23 -0400 Subject: [PATCH] Support regex in feature tag - modified testInfo parser - added new tests for feature tag Signed-off-by: renfeiw --- examples/feature/playlist.xml | 51 ++++++++++++++++++++++ scripts/testTKG/test_feature.py | 56 +++++++++++++++++++++++++ src/org/testKitGen/TestInfoParser.java | 58 +++++++++++++++++++------- 3 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 examples/feature/playlist.xml create mode 100644 scripts/testTKG/test_feature.py diff --git a/examples/feature/playlist.xml b/examples/feature/playlist.xml new file mode 100644 index 00000000..aa5c72c8 --- /dev/null +++ b/examples/feature/playlist.xml @@ -0,0 +1,51 @@ + + + + + testFeatureFips + echo "match FIPS140_3_20220501 and everything else by default"; \ + $(TEST_STATUS) + + FIPS140_3_20220501:applicable + + + + + testFeatureFipsNoApplicable + echo "test excluded on FIPS140_2"; \ + $(TEST_STATUS) + + FIPS140_2:nonapplicable + + + + + testFeatureFipsRequired + echo "only run on FIPS140_3_20220101"; \ + $(TEST_STATUS) + + FIPS140_3_20220101:required + + + + + testFeatureFipsRegexRequired + echo "only run on FIPS140_3 from 2022 Dec 01 to 2029 Dec 31"; \ + $(TEST_STATUS) + + /FIPS140_3_(2022(12\d(2))|202[3-9]\d{4})/:required + + + diff --git a/scripts/testTKG/test_feature.py b/scripts/testTKG/test_feature.py new file mode 100644 index 00000000..9ae5ff9a --- /dev/null +++ b/scripts/testTKG/test_feature.py @@ -0,0 +1,56 @@ +################################################################################ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import subprocess +from utils import * + +def run_test(): + rt = True + printTestheader("feature") + + buildList = "TKG/examples/feature" + command = "make _all" + print(f"\t{command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + rt = checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0'}, set(), set(), set()) + + buildList = "TKG/examples/feature" + command = "make _all" + testFlag = "export TEST_FLAG=FIPS140_2" + print(f"\t{testFlag}; {command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + rt = checkResult(result, {'testFeatureFips_0'}, set(), set(), set()) + + command = "make _all" + testFlag = "export TEST_FLAG=FIPS140_3_20220501" + print(f"\t{testFlag}; {command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + rt = checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0'}, set(), set(), set()) + + command = "make _all" + testFlag = "export TEST_FLAG=FIPS140_3_20230511" + print(f"\t{testFlag}; {command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + rt = checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0', 'testFeatureFipsRegexRequired_0'}, set(), set(), set()) + + command = "make _all" + testFlag = "export TEST_FLAG=FIPS140_3_20220101" + print(f"\t{testFlag}; {command}") + result = subprocess.run(f"{EXPORT_BUILDLIST}={buildList}; {testFlag}; {CD_TKG}; {MAKE_CLEAN}; {MAKE_COMPILE}; {command}", stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, check=False) + rt = checkResult(result, {'testFeatureFips_0', 'testFeatureFipsNoApplicable_0', 'testFeatureFipsRequired_0'}, set(), set(), set()) + + return rt + +if __name__ == "__main__": + run_test() \ No newline at end of file diff --git a/src/org/testKitGen/TestInfoParser.java b/src/org/testKitGen/TestInfoParser.java index 949941e9..df0b7c67 100644 --- a/src/org/testKitGen/TestInfoParser.java +++ b/src/org/testKitGen/TestInfoParser.java @@ -88,31 +88,37 @@ public TestInfo parse() { } Set testFlags = new HashSet<>(arg.getTestFlag()); for (Map.Entry entry : ti.getFeatures().entrySet()) { - if (entry.getValue().equalsIgnoreCase("required")) { - if (!testFlags.contains(entry.getKey())) { + String featureOpt = entry.getValue().toLowerCase(); + if (featureOpt.equals("required")) { + if (!isFeatureInTestFlags(testFlags, entry.getKey())) { return null; - } else if (entry.getKey().equalsIgnoreCase("aot")) { - ti.setAotOptions("$(AOT_OPTIONS) "); } - } else if (entry.getValue().equalsIgnoreCase("applicable")) { - if (testFlags.contains("aot") && (entry.getKey().equalsIgnoreCase("aot") || entry.getKey().equalsIgnoreCase("all"))) { - ti.setAotOptions("$(AOT_OPTIONS) "); - } - } else if (entry.getValue().equalsIgnoreCase("nonapplicable")) { + } else if (featureOpt.equals("nonapplicable")) { // Do not generate make target if the test is not applicable for one feature defined in TEST_FLAG - if (testFlags.contains(entry.getKey())) { + if (isFeatureInTestFlags(testFlags, entry.getKey())) { return null; } - } else if (entry.getValue().equalsIgnoreCase("explicit")) { - if (testFlags.contains("aot") && entry.getKey().equalsIgnoreCase("aot")) { - ti.setAotIterations(1); - } + } else if (featureOpt.equals("applicable") || featureOpt.equals("explicit")) { + // Do nothing } else { System.err.println("Error: Please provide a valid feature parameter in test " + ti.getTestCaseName() + ". The valid string is :[required|applicable|nonapplicable|explicit]."); System.exit(1); } } + if (testFlags.contains("aot")) { + for (Map.Entry entry : ti.getFeatures().entrySet()) { + if (doesFeatureMatchTestFlag("aot", entry.getKey())) { + String featureOpt = entry.getValue().toLowerCase(); + if (featureOpt.equals("required") || featureOpt.equals("applicable")) { + ti.setAotOptions("$(AOT_OPTIONS) "); + } else if (featureOpt.equals("explicit")) { + ti.setAotIterations(1); + } + } + } + } + String rerun = getImmediateChildContent(testEle, "rerun"); if (rerun != null) { if (rerun.equals("true")) { @@ -207,6 +213,30 @@ public TestInfo parse() { return ti; } + private boolean isFeatureInTestFlags(Set testFlags, String feature) { + for (String testFlag : testFlags) { + if (doesFeatureMatchTestFlag(testFlag, feature)) { + return true; + } + } + return false; + } + + private boolean doesFeatureMatchTestFlag(String testFlag, String feature) { + if (feature.equals("all")) { + return true; + } + if (!feature.startsWith("/") || !feature.endsWith("/")) { + return testFlag.equalsIgnoreCase(feature); + } + Pattern pattern = Pattern.compile(feature.substring(1, feature.length() - 1)); + Matcher matcher = pattern.matcher(testFlag); + if (matcher.matches()) { + return true; + } + return false; + } + private boolean checkJavaVersion(String version) { if (version.equalsIgnoreCase(arg.getJdkVersion())) { return true;