diff --git a/testing/esp/esdlcmd/esdlcmd-test.py b/testing/esp/esdlcmd/esdlcmd-test.py index 99c258d42df..21a3d7dbb73 100755 --- a/testing/esp/esdlcmd/esdlcmd-test.py +++ b/testing/esp/esdlcmd/esdlcmd-test.py @@ -52,6 +52,7 @@ def __init__(self, stats, exe_path, output_base, test_path): self.test_path = test_path self.stats = stats +# This class is base for commands related to services: wsdl, xsd, cpp and java. class TestCaseBase: """Settings for a specific test case.""" @@ -103,6 +104,110 @@ def validate_results(self): logging.debug('TestCaseBase implementation called, no comparison run') return False +# This class is the base for the 'transform' commands: ecl and xml. +# In a future update, investigate refactoring the base classes so there is a single +# parent with any shared capabilities. +# +# When writing to stdout, the key directory should contain a file named 'from-stdout.ecl' +# that contains the expected output. +class TestCaseTransformBase: + def __init__(self, run_settings, name, command, esdl_file, xsl_path, use_stdout, expected_err=None, options=None): + self.run_settings = run_settings + self.name = name + self.command = command + self.esdl_path = (self.run_settings.test_path / 'inputs' / esdl_file) + self.xsl_path = xsl_path + self.options = options + self.output_path = Path(self.run_settings.output_base) / name + self.stdout = use_stdout + self.expected_err = expected_err + if self.stdout: + self.args = [ + str(run_settings.exe_path), + self.command, + self.esdl_path, + '-cde', + self.xsl_path, + ] + else: + self.args = [ + str(run_settings.exe_path), + self.command, + self.esdl_path, + self.output_path, + '-cde', + self.xsl_path, + ] + + if options: + self.args.extend(options) + + self.result = None + + def run_test(self): + safe_mkdir(self.output_path) + logging.debug("Test %s args: %s", self.name, str(self.args)) + self.result = subprocess.run(self.args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + if self.expected_err != None and self.expected_err == self.result.stderr: + success = True + elif self.result.returncode != 0: + logging.error('Error running "esdl %s" for test "%s": %s', self.command, self.name, self.result.stderr) + success = False + else: + success = self.validate_results() + + self.run_settings.stats.add_count(success) + + def is_same(self, dir1, dir2): + """ + Compare two directory trees content. + Return False if they differ, True is they are the same. + """ + compared = DirectoryCompare(dir1, dir2) + if (compared.left_only or compared.right_only or compared.diff_files + or compared.funny_files): + return False + for subdir in compared.common_dirs: + if not self.is_same(os.path.join(dir1, subdir), os.path.join(dir2, subdir)): + return False + return True + + def validate_results(self): + """Compare test case results to the known key. + + Return True if the two are identical or False otherwise. + """ + outName = self.output_path + key = (self.run_settings.test_path / 'key' / self.name) + + # When output was stdout, write the captured stdout text to a file named 'from-stdout.ecl' + # in the output directory. This allows us to use the same method of comparing the key + # and result. + if self.stdout: + if self.result.stdout != None and len(self.result.stdout) > 0: + with open((outName / 'from-stdout.ecl'), 'w', encoding='utf-8') as f: + f.write(self.result.stdout) + else: + logging.error('Missing stdout output for test %s', self.name) + return False + + if (not key.exists()): + logging.error('Missing key file %s', str(key)) + return False + + if (not outName.exists()): + logging.error('Missing output for test %s', self.name) + return False + + if (not self.is_same(str(key), str(outName))): + logging.debug('Comparing key %s to output %s', str(key), str(outName)) + logging.error('Test failed: %s', self.name) + return False + else: + logging.debug('Passed: %s', self.name) + return True + class TestCaseXSD(TestCaseBase): """Test case for the wsdl or xsd commands. @@ -212,6 +317,8 @@ def parse_options(): the parsed options and arguments. """ + command_values = ['all', 'cpp', 'ecl', 'java', 'wsdl', 'xsd'] + parser = argparse.ArgumentParser(description=DESC) parser.add_argument('testroot', help='Path of the root folder of the esdlcmd testing project') @@ -231,7 +338,15 @@ def parse_options(): help='Enable debug logging of test cases', action='store_true', default=False) + parser.add_argument('-c', '--commands', + help='esdl commands to run tests for, use once for each command or pass "all" to test all commands. Defaults to "all".', + action="append", choices=command_values) + args = parser.parse_args() + + if args.commands == None: + args.commands = ['all'] + return args @@ -278,6 +393,11 @@ def main(): stats = Statistics() run_settings = TestRun(stats, exe_path, args.outdir, test_path) + esdl_includes_path = str(test_path / 'inputs') + + expected_err_multi_file_incl = '\nOutput to stdout is not supported for multiple files. Either add the Rollup\noption or specify an output directory.\n' + expected_err_multi_file_expanded = '\nOutput to stdout is not supported for multiple files. Remove the Output expanded\n XML option or specify an output directory.\n' + test_cases = [ # wsdl TestCaseXSD(run_settings, 'wstest-wsdl-default', 'wsdl', 'ws_test.ecm', 'WsTest', @@ -390,10 +510,27 @@ def main(): # A single element is created for each request structure defined. This is default behavior. TestCaseXSD(run_settings, 'use-request-name', 'wsdl', 'ws_userequestname.ecm', 'WsUseRequestName', xsl_base_path), + + # ecl + TestCaseTransformBase(run_settings, 'ecl-stdout-single', 'ecl', 'ws_test.ecm', xsl_base_path, use_stdout=True), + + TestCaseTransformBase(run_settings, 'ecl-stdout-incl-err', 'ecl', 'ws_test.ecm', xsl_base_path, use_stdout=True, + expected_err=expected_err_multi_file_incl, options=['-I', esdl_includes_path, '--includes']), + + TestCaseTransformBase(run_settings, 'ecl-stdout-expanded-err', 'ecl', 'ws_test.ecm', xsl_base_path, use_stdout=True, + expected_err=expected_err_multi_file_expanded, options=['-x']), + + TestCaseTransformBase(run_settings, 'ecl-stdout-incl-rollup', 'ecl', 'ws_test.ecm', xsl_base_path, use_stdout=True, + options=['-I', esdl_includes_path, '--includes', '--rollup']), + + TestCaseTransformBase(run_settings, 'ecl-incl', 'ecl', 'ws_test.ecm', xsl_base_path, use_stdout=False, + options=['-I', esdl_includes_path, '--includes']) + ] for case in test_cases: - case.run_test() + if 'all' in args.commands or case.command in args.commands: + case.run_test() logging.info('Success count: %d', stats.successCount) logging.info('Failure count: %d', stats.failureCount) diff --git a/testing/esp/esdlcmd/key/ecl-incl/allversionreport.ecl b/testing/esp/esdlcmd/key/ecl-incl/allversionreport.ecl new file mode 100644 index 00000000000..009048a0674 --- /dev/null +++ b/testing/esp/esdlcmd/key/ecl-incl/allversionreport.ecl @@ -0,0 +1,37 @@ +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from allversionreport.xml. ***/ +/*===================================================*/ + + +EXPORT allversionreport := MODULE + +EXPORT t_FooBar := RECORD + UTF8 Foo {XPATH('Foo')}; + UTF8 Bar {XPATH('Bar')}; +END; + +EXPORT t_AllVersionArrays := RECORD + SET OF UTF8 StringArray {XPATH('StringArray/Item'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! + DATASET(t_FooBar) FooBarArray {XPATH('FooBarArray/FooBar'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! + DATASET(t_FooBar) NamedItemFooBarArray {XPATH('NamedItemFooBarArray/NamedItem'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! +END; + +EXPORT t_AllVersionReportRequest := RECORD + UTF8 OptionalDeveloperStringVal {XPATH('OptionalDeveloperStringVal')};//hidden[developer] + INTEGER Annotate20ColsIntVal {XPATH('Annotate20ColsIntVal')}; + t_AllVersionArrays Arrays {XPATH('Arrays')}; + UTF8 UnrelentingForce {XPATH('UnrelentingForce')}; //values['1','2','3',''] +END; + +EXPORT t_AllVersionReportResponse := RECORD + UTF8 ResultVal {XPATH('ResultVal')}; + t_AllVersionArrays ResultArrays {XPATH('ResultArrays')}; +END; + + +END; + +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from allversionreport.xml. ***/ +/*===================================================*/ + diff --git a/testing/esp/esdlcmd/key/ecl-incl/ws_test.ecl b/testing/esp/esdlcmd/key/ecl-incl/ws_test.ecl new file mode 100644 index 00000000000..89894f075b8 --- /dev/null +++ b/testing/esp/esdlcmd/key/ecl-incl/ws_test.ecl @@ -0,0 +1,40 @@ +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + + +EXPORT ws_test := MODULE + +EXPORT t_MinVersionReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlRequest +EXPORT t_WsTestPingRequest := RECORD +END; +*/ + +EXPORT t_MinVersionReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlResponse +EXPORT t_WsTestPingResponse := RECORD +END; +*/ + + +END; + +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + diff --git a/testing/esp/esdlcmd/key/ecl-stdout-incl-rollup/from-stdout.ecl b/testing/esp/esdlcmd/key/ecl-stdout-incl-rollup/from-stdout.ecl new file mode 100644 index 00000000000..f558558fa0d --- /dev/null +++ b/testing/esp/esdlcmd/key/ecl-stdout-incl-rollup/from-stdout.ecl @@ -0,0 +1,63 @@ +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + + +EXPORT ws_test := MODULE + +EXPORT t_FooBar := RECORD + UTF8 Foo {XPATH('Foo')}; + UTF8 Bar {XPATH('Bar')}; +END; + +EXPORT t_AllVersionArrays := RECORD + SET OF UTF8 StringArray {XPATH('StringArray/Item'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! + DATASET(t_FooBar) FooBarArray {XPATH('FooBarArray/FooBar'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! + DATASET(t_FooBar) NamedItemFooBarArray {XPATH('NamedItemFooBarArray/NamedItem'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! +END; + +EXPORT t_AllVersionReportRequest := RECORD + UTF8 OptionalDeveloperStringVal {XPATH('OptionalDeveloperStringVal')};//hidden[developer] + INTEGER Annotate20ColsIntVal {XPATH('Annotate20ColsIntVal')}; + t_AllVersionArrays Arrays {XPATH('Arrays')}; + UTF8 UnrelentingForce {XPATH('UnrelentingForce')}; //values['1','2','3',''] +END; + +EXPORT t_MinVersionReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlRequest +EXPORT t_WsTestPingRequest := RECORD +END; +*/ + +EXPORT t_AllVersionReportResponse := RECORD + UTF8 ResultVal {XPATH('ResultVal')}; + t_AllVersionArrays ResultArrays {XPATH('ResultArrays')}; +END; + +EXPORT t_MinVersionReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlResponse +EXPORT t_WsTestPingResponse := RECORD +END; +*/ + + +END; + +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + diff --git a/testing/esp/esdlcmd/key/ecl-stdout-single/from-stdout.ecl b/testing/esp/esdlcmd/key/ecl-stdout-single/from-stdout.ecl new file mode 100644 index 00000000000..89894f075b8 --- /dev/null +++ b/testing/esp/esdlcmd/key/ecl-stdout-single/from-stdout.ecl @@ -0,0 +1,40 @@ +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + + +EXPORT ws_test := MODULE + +EXPORT t_MinVersionReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlRequest +EXPORT t_WsTestPingRequest := RECORD +END; +*/ + +EXPORT t_MinVersionReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlResponse +EXPORT t_WsTestPingResponse := RECORD +END; +*/ + + +END; + +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + diff --git a/tools/esdlcmd/esdl2ecl.cpp b/tools/esdlcmd/esdl2ecl.cpp index fa5ff0b7e80..321dbe524a2 100644 --- a/tools/esdlcmd/esdl2ecl.cpp +++ b/tools/esdlcmd/esdl2ecl.cpp @@ -19,6 +19,7 @@ #include "xslprocessor.hpp" #include "esdlcmd_core.hpp" +#include "esdlcmdutils.hpp" typedef IPropertyTree * IPTreePtr; @@ -103,7 +104,7 @@ class EsdlIndexedPropertyTrees { fileName.append(srcext); StringBuffer esxml; - EsdlCmdHelper::convertECMtoESXDL(fileName.str(), srcfile, esxml, loadincludes && rollUp, true, true, isIncludedESDL, includePath); + EsdlCmdHelper::convertECMtoESXDL(fileName.str(), srcfile, esxml, loadincludes && rollUp, false, true, isIncludedESDL, includePath); src = createPTreeFromXMLString(esxml, 0); } else if (!srcext || !*srcext || stricmp(srcext, XML_FILE_EXTENSION)==0) @@ -198,35 +199,32 @@ class Esdl2EclCmd : public EsdlConvertCmd { if (iter.done()) { + fprintf(stderr, "\nRequired sourcePath parameter missing.\n"); usage(); return false; } - //First two parameters' order is fixed. - for (int par = 0; par < 2 && !iter.done(); par++) + //First parameter's order is fixed. + const char *arg = iter.query(); + if (*arg != '-') + optSource.set(arg); + else + { + fprintf(stderr, "\nOption detected before required sourcePath parameter: %s\n", arg); + usage(); + return false; + } + + + if (iter.next()) { - const char *arg = iter.query(); + arg = iter.query(); if (*arg != '-') { - if (optSource.isEmpty()) - optSource.set(arg); - else if (optOutDirPath.isEmpty()) - optOutDirPath.set(arg); - else - { - fprintf(stderr, "\nunrecognized argument detected before required parameters: %s\n", arg); - usage(); - return false; - } + optOutDirPath.set(arg); + optStdout = false; + iter.next(); } - else - { - fprintf(stderr, "\noption detected before required parameters: %s\n", arg); - usage(); - return false; - } - - iter.next(); } for (; !iter.done(); iter.next()) @@ -281,7 +279,27 @@ class Esdl2EclCmd : public EsdlConvertCmd virtual bool finalizeOptions(IProperties *globals) { - return EsdlConvertCmd::finalizeOptions(globals); + if (optStdout) + { + // We can't call EsdlConvertCmd::finalizeOptions when using stdout because it + // requires outOutDirPath, so duplicate the other work it does here instead + extractEsdlCmdOption(optIncludePath, globals, ESDLOPT_INCLUDE_PATH_ENV, ESDLOPT_INCLUDE_PATH_INI, NULL, NULL); + if (optOutputExpandedXML) + { + fprintf(stderr, "\nOutput to stdout is not supported for multiple files. Remove the Output expanded\n XML option or specify an output directory.\n"); + usage(); + return false; + } + else if (optProcessIncludes && !optRollUpEclToSingleFile) + { + fprintf(stderr, "\nOutput to stdout is not supported for multiple files. Either add the Rollup\noption or specify an output directory.\n"); + usage(); + return false; + } + return EsdlCmdCommon::finalizeOptions(globals); + } + else + return EsdlConvertCmd::finalizeOptions(globals); } virtual int processCMD() @@ -378,10 +396,16 @@ class Esdl2EclCmd : public EsdlConvertCmd virtual void usage() { fputs("\nUsage:\n\n" - "esdl ecl sourcePath outputPath [options]\n" - "\nsourcePath must be absolute path to the ESDL Definition file containing the" + "esdl ecl sourcePath [outputPath] [options]\n" + "\n" + "sourcePath must be absolute path to the ESDL Definition file containing the\n" "EsdlService definition for the service you want to work with.\n" - "outputPath must be the absolute path where the ECL output with be created.\n" + "\n" + "outputPath, if supplied, must be the absolute path where the ECL output will be\n" + "created. When outputPath is omitted options must generate a single file which is\n" + "written to stdout. This means to write to stdout, you must not use -x/--expandedxml,\n" + "nor can you use --includes without also using --rollup.\n" + "\n" " Options:\n" " -x, --expandedxml Output expanded XML files\n" " --includes Process all included files\n" @@ -390,7 +414,7 @@ class Esdl2EclCmd : public EsdlConvertCmd " --ecl-imports Comma-delimited import list to be attached to output ECL\n" " each entry generates a corresponding import *.\n" " --ecl-header Text included in target header (must be valid ECL) \n" - " --utf8- Don't use UTF8 strings" + " --utf8- Don't use UTF8 strings\n" " " ESDLOPT_INCLUDE_PATH_USAGE ,stdout); } @@ -398,10 +422,11 @@ class Esdl2EclCmd : public EsdlConvertCmd void outputEcl(const char *srcpath, const char *file, const char *path, const char *types, const char * xml, const char * eclimports, const char * eclheader) { DBGLOG("Generating ECL file for %s", file); - + StringBuffer outfile; StringBuffer filePath; StringBuffer fileName; StringBuffer fileExt; + StringBuffer fullSrcFile; splitFilename(file, NULL, &filePath, &fileName, &fileExt); @@ -409,30 +434,30 @@ class Esdl2EclCmd : public EsdlConvertCmd if (!strnicmp(finger, "wsm_", 4)) finger+=4; - StringBuffer outfile; - if (path && *path) - { - outfile.append(path); - if (outfile.length() && !strchr("/\\", outfile.charAt(outfile.length()-1))) - outfile.append('/'); - } - outfile.append(finger).append(".ecl"); + fullSrcFile.append(srcpath); + addPathSepChar(fullSrcFile, PATHSEPCHAR).append(file); + if (!optStdout) { - //If the target output file cannot be accessed, this operation will - //throw, and will be caught and reported at the shell level. - Owned ofile = createIFile(outfile.str()); - if (ofile) + if (path && *path) { - Owned fileIO = ofile->open(IFOcreate); - fileIO.clear(); + outfile.append(path); + if (outfile.length() && !strchr("/\\", outfile.charAt(outfile.length()-1))) + outfile.append('/'); } - } + outfile.append(finger).append(".ecl"); - StringBuffer fullname(srcpath); - if (fullname.length() && !strchr("/\\", fullname.charAt(fullname.length()-1))) - fullname.append('/'); - fullname.append(fileName.str()).append(".xml"); + { + //If the target output file cannot be accessed, this operation will + //throw, and will be caught and reported at the shell level. + Owned ofile = createIFile(outfile.str()); + if (ofile) + { + Owned fileIO = ofile->open(IFOcreate); + fileIO.clear(); + } + } + } StringBuffer expstr; expstr.append(""); @@ -468,10 +493,10 @@ class Esdl2EclCmd : public EsdlConvertCmd params->setProp("utf8strings", optUseUtf8Strings ? "yes" : "no"); StringBuffer esdl2eclxslt (optHPCCCompFilesDir.get()); esdl2eclxslt.append("/xslt/esdl2ecl.xslt"); - esdl2eclxsltTransform(expstr.str(), esdl2eclxslt.str(), params, outfile.str()); + esdl2eclxsltTransform(expstr.str(), esdl2eclxslt.str(), params, outfile.str(), fullSrcFile.str()); } - void esdl2eclxsltTransform(const char* xml, const char* sheet, IProperties *params, const char *filename) + void esdl2eclxsltTransform(const char* xml, const char* sheet, IProperties *params, const char *outputFilename, const char *srcFilename) { StringBuffer xsl; xsl.loadFile(sheet); @@ -494,15 +519,23 @@ class Esdl2EclCmd : public EsdlConvertCmd } } - trans->setResultTarget(filename); - try { - trans->transform(); + if (optStdout) + { + StringBuffer output; + trans->transform(output); + fputs(output.str(), stdout); + } + else + { + trans->setResultTarget(outputFilename); + trans->transform(); + } } catch(...) { - fprintf(stderr, "Error transforming Esdl to ECL file %s", filename); + fprintf(stderr, "Error transforming Esdl file %s to ECL file", srcFilename); } } @@ -511,6 +544,7 @@ class Esdl2EclCmd : public EsdlConvertCmd bool optProcessIncludes; bool optOutputExpandedXML; bool optUseUtf8Strings = true; + bool optStdout = true; StringAttr optHPCCCompFilesDir; StringAttr optECLIncludesList; StringAttr optECLHeaderBlock;