From 87f679de78e23e6010528be9094c72868068c5f3 Mon Sep 17 00:00:00 2001 From: Joe George Date: Mon, 22 Jan 2024 10:04:24 -0500 Subject: [PATCH] More Python cleanup (#1708) --- .github/workflows/ruff.yml | 2 +- certs/makecerts.py | 49 +- config/makeprops.py | 260 ++- cpp/test/Ice/invoke/test.py | 2 +- csharp/allTests.py | 1 + csharp/test/Ice/invoke/test.py | 15 +- csharp/test/IceSSL/certs/makecerts.py | 31 +- java/allTests.py | 1 + .../src/main/java/test/Ice/invoke/test.py | 15 +- .../main/java/test/IceSSL/certs/makecerts.py | 31 +- js/allTests.py | 1 + php/allTests.py | 1 + php/test/Ice/ini/test.py | 51 +- ruby/allTests.py | 1 + scripts/Component.py | 195 +- scripts/Controller.py | 112 +- scripts/Expect.py | 228 ++- scripts/Glacier2Util.py | 53 +- scripts/IceBoxUtil.py | 26 +- scripts/IceBridgeUtil.py | 37 +- scripts/IceGridUtil.py | 213 ++- scripts/IcePatch2Util.py | 27 +- scripts/IceStormUtil.py | 179 +- scripts/LocalDriver.py | 267 ++- scripts/NetworkProxy.py | 24 +- scripts/Util.py | 1670 ++++++++++++----- scripts/icehashpassword.py | 27 +- swift/allTests.py | 1 + 28 files changed, 2471 insertions(+), 1049 deletions(-) diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 9adc5711a1c..35f88c1011c 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -20,4 +20,4 @@ jobs: run: pip install ruff - name: Run Ruff - run: ruff check 'cpp' --ignore E402 + run: ruff check . --exclude "scripts" --exclude "python" --ignore E402 diff --git a/certs/makecerts.py b/certs/makecerts.py index 38f8ce72554..7d88a8f9dde 100755 --- a/certs/makecerts.py +++ b/certs/makecerts.py @@ -11,8 +11,10 @@ try: import IceCertUtils except Exception as ex: - print("couldn't load IceCertUtils, did you install the `zeroc-icecertutils'\n" - "package from the Python package repository?\nerror: " + str(ex)) + print( + "couldn't load IceCertUtils, did you install the `zeroc-icecertutils'\n" + "package from the Python package repository?\nerror: " + str(ex) + ) sys.exit(1) @@ -38,13 +40,15 @@ def usage(): usedns = False impl = "" try: - opts, args = getopt.getopt(sys.argv[1:], "hd", ["help", "debug", "ip=", "dns=", "use-dns", "impl="]) + opts, args = getopt.getopt( + sys.argv[1:], "hd", ["help", "debug", "ip=", "dns=", "use-dns", "impl="] + ) except getopt.GetoptError as e: print("Error %s " % e) usage() sys.exit(1) -for (o, a) in opts: +for o, a in opts: if o == "-h" or o == "--help": usage() sys.exit(0) @@ -65,7 +69,7 @@ def request(question, newvalue, value): sys.stdout.write(question) sys.stdout.flush() input = sys.stdin.readline().strip() - if input == 'n': + if input == "n": sys.stdout.write(newvalue) sys.stdout.flush() return sys.stdin.readline().strip() @@ -81,15 +85,23 @@ def request(question, newvalue, value): if not ip: try: ip = socket.gethostbyname(socket.gethostname()) - except: + except Exception: ip = "127.0.0.1" - ip = request("The IP address used for the server certificate will be: " + ip + "\n" - "Do you want to keep this IP address? (y/n) [y]", "IP : ", ip) + ip = request( + "The IP address used for the server certificate will be: " + ip + "\n" + "Do you want to keep this IP address? (y/n) [y]", + "IP : ", + ip, + ) if not dns: dns = "localhost" - dns = request("The DNS name used for the server certificate will be: " + dns + "\n" - "Do you want to keep this DNS name? (y/n) [y]", "DNS : ", dns) + dns = request( + "The DNS name used for the server certificate will be: " + dns + "\n" + "Do you want to keep this DNS name? (y/n) [y]", + "DNS : ", + dns, + ) CertificateFactory = vars(IceCertUtils)[impl + "CertificateFactory"] factory = CertificateFactory(debug=debug, cn="Ice Tests CA") @@ -110,7 +122,13 @@ def request(question, newvalue, value): # # NOTE: server.pem is used by scripts/TestController.py # -server = factory.create("server", cn=(dns if usedns else ip), ip=ip, dns=dns, extendedKeyUsage="serverAuth,clientAuth") +server = factory.create( + "server", + cn=(dns if usedns else ip), + ip=ip, + dns=dns, + extendedKeyUsage="serverAuth,clientAuth", +) server.save("server.p12").save("server.pem") try: @@ -125,14 +143,19 @@ def request(question, newvalue, value): for f in ["server.bks", "client.bks"]: if os.path.exists(f): os.remove(f) - print("warning: couldn't generate BKS certificates for Android applications:\n" + str(ex)) + print( + "warning: couldn't generate BKS certificates for Android applications:\n" + + str(ex) + ) print("Please fix this issue if you want to run the Android tests.") except Exception as ex: for f in ["server.jks", "client.jks"]: if os.path.exists(f): os.remove(f) - print("warning: couldn't generate JKS certificates for Java applications:\n" + str(ex)) + print( + "warning: couldn't generate JKS certificates for Java applications:\n" + str(ex) + ) print("Please fix this issue if you want to run the Java tests.") factory.destroy() diff --git a/config/makeprops.py b/config/makeprops.py index 45f797c8cf9..938d30dba8b 100755 --- a/config/makeprops.py +++ b/config/makeprops.py @@ -6,15 +6,12 @@ import os import sys import shutil -import re import signal import time -import pprint from xml.sax import make_parser from xml.sax.handler import feature_namespaces from xml.sax.handler import ContentHandler -from xml.sax import saxutils from xml.sax import SAXException from xml.dom.minidom import parse @@ -28,13 +25,24 @@ // """ -commonPreamble = commonPreamble + "// Generated by " + progname + " from file %(inputfile)s, " + time.ctime() -commonPreamble = commonPreamble + """ +commonPreamble = ( + commonPreamble + + "// Generated by " + + progname + + " from file %(inputfile)s, " + + time.ctime() +) +commonPreamble = ( + commonPreamble + + """ // IMPORTANT: Do not edit this file -- any edits made here will be lost! """ +) -cppHeaderPreamble = commonPreamble + """ +cppHeaderPreamble = ( + commonPreamble + + """ #ifndef ICE_INTERNAL_%(classname)s_H #define ICE_INTERNAL_%(classname)s_H @@ -82,6 +90,7 @@ class %(classname)s public: """ +) cppHeaderPostamble = """ static const PropertyArray validProps[]; @@ -93,26 +102,37 @@ class %(classname)s #endif """ -cppSrcPreamble = commonPreamble + """ +cppSrcPreamble = ( + commonPreamble + + """ #include """ +) -javaPreamble = commonPreamble + """ +javaPreamble = ( + commonPreamble + + """ package com.zeroc.IceInternal; public final class %(classname)s { """ +) -csPreamble = commonPreamble + """ +csPreamble = ( + commonPreamble + + """ namespace IceInternal { public sealed class %(classname)s { """ +) -jsPreamble = commonPreamble + """ +jsPreamble = ( + commonPreamble + + """ /* eslint comma-dangle: "off" */ /* eslint array-bracket-newline: "off" */ /* eslint no-useless-escape: "off" */ @@ -121,9 +141,9 @@ class %(classname)s const %(classname)s = {}; const Property = Ice.Property; """ +) -jsEpilogue = \ - """ +jsEpilogue = """ Ice.%(classname)s = %(classname)s; module.exports.Ice = Ice; """ @@ -131,12 +151,13 @@ class %(classname)s def usage(): global progname - print >> sys.stderr, "Usage: " + progname + " [--{cpp|java|csharp|js} file]" + print(sys.stderr, "Usage: " + progname + " [--{cpp|java|csharp|js} file]") def progError(msg): global progname - print >> sys.stderr, progname + ": " + msg + print(sys.stderr, progname + ": " + msg) + # # Currently the processing of PropertyNames.xml is going to take place @@ -178,7 +199,10 @@ def initPropertyClasses(filename): attmap[a.attributes.item(i).name] = a.attributes.item(i).value properties.append(attmap) - propertyClasses[className] = PropertyClass(classType.lower() == "true", properties) + propertyClasses[className] = PropertyClass( + classType.lower() == "true", properties + ) + # # SAX part. @@ -202,7 +226,6 @@ def __str__(self): class PropertyHandler(ContentHandler): - def __init__(self, inputfile, className): self.start = False self.properties = {} @@ -283,7 +306,7 @@ def startElement(self, name, attrs): if "class" in attrs: c = propertyClasses[attrs["class"]] for p in c.getChildren(): - if propertyName == None: + if propertyName is None: self.startElement(name, p) else: t = dict(p) @@ -293,17 +316,21 @@ def startElement(self, name, attrs): # usually absolute or 'raw', but in the case of # a property class, they need to be expanded. if "deprecatedBy" in t: - t["deprecatedBy"] = "%s.%s.%s" % (self.currentSection, propertyName, t["deprecatedBy"]) - t['name'] = "%s.%s" % (propertyName, p['name']) + t["deprecatedBy"] = "%s.%s.%s" % ( + self.currentSection, + propertyName, + t["deprecatedBy"], + ) + t["name"] = "%s.%s" % (propertyName, p["name"]) self.startElement(name, t) if c.isPrefixOnly(): return # - # != None implies deprecated == true + # 'is not None' implies deprecated == true # deprecatedBy = attrs.get("deprecatedBy", None) - if deprecatedBy != None: + if deprecatedBy is not None: self.handleDeprecatedWithReplacement(propertyName, deprecatedBy) elif attrs.get("deprecated", "false").lower() == "true": self.handleDeprecated(propertyName) @@ -318,18 +345,17 @@ def endElement(self, name): class CppPropertyHandler(PropertyHandler): - def __init__(self, inputfile, c): PropertyHandler.__init__(self, inputfile, c) self.hFile = None self.cppFile = None def cleanup(self): - if self.hFile != None: + if self.hFile is not None: self.hFile.close() if os.path.exists(self.className + ".h"): os.remove(self.className + ".h") - if self.cppFile != None: + if self.cppFile is not None: self.cppFile.close() if os.path.exists(self.className + ".cpp"): os.remove(self.className + ".cpp") @@ -337,14 +363,21 @@ def cleanup(self): def startFiles(self): self.hFile = open(self.className + ".h", "w") self.cppFile = open(self.className + ".cpp", "w") - self.hFile.write(cppHeaderPreamble % {'inputfile': self.inputfile, 'classname': self.className}) - self.cppFile.write(cppSrcPreamble % {'inputfile': self.inputfile, 'classname': self.className}) + self.hFile.write( + cppHeaderPreamble + % {"inputfile": self.inputfile, "classname": self.className} + ) + self.cppFile.write( + cppSrcPreamble % {"inputfile": self.inputfile, "classname": self.className} + ) def closeFiles(self): - self.hFile.write(cppHeaderPostamble % {'classname': self.className}) - self.cppFile.write("const IceInternal::PropertyArray " - "IceInternal::%(classname)s::validProps[] =\n" % - {'classname': self.className}) + self.hFile.write(cppHeaderPostamble % {"classname": self.className}) + self.cppFile.write( + "const IceInternal::PropertyArray " + "IceInternal::%(classname)s::validProps[] =\n" + % {"classname": self.className} + ) self.cppFile.write("{\n") for s in self.sections: @@ -352,11 +385,13 @@ def closeFiles(self): self.cppFile.write(" IceInternal::PropertyArray(0,0)\n") self.cppFile.write("};\n\n") - self.cppFile.write("const char* IceInternal::%(classname)s::clPropNames[] =\n" % - {'classname': self.className}) + self.cppFile.write( + "const char* IceInternal::%(classname)s::clPropNames[] =\n" + % {"classname": self.className} + ) self.cppFile.write("{\n") for s in self.cmdLineOptions: - self.cppFile.write(" \"%s\",\n" % s) + self.cppFile.write(' "%s",\n' % s) self.cppFile.write(" 0\n") self.cppFile.write("};\n") self.hFile.close() @@ -366,30 +401,43 @@ def fix(self, propertyName): return propertyName.replace("[any]", "*") def deprecatedImpl(self, propertyName): - self.cppFile.write(" IceInternal::Property(\"%s.%s\", true, 0),\n" % (self.currentSection, - self.fix(propertyName))) + self.cppFile.write( + ' IceInternal::Property("%s.%s", true, 0),\n' + % (self.currentSection, self.fix(propertyName)) + ) def deprecatedImplWithReplacementImpl(self, propertyName, deprecatedBy): - self.cppFile.write(" IceInternal::Property(\"%s.%s\", true, \"%s\"),\n" % (self.currentSection, - self.fix(propertyName), deprecatedBy)) + self.cppFile.write( + ' IceInternal::Property("%s.%s", true, "%s"),\n' + % (self.currentSection, self.fix(propertyName), deprecatedBy) + ) def propertyImpl(self, propertyName): - self.cppFile.write(" IceInternal::Property(\"%s.%s\", false, 0),\n" % - (self.currentSection, self.fix(propertyName))) + self.cppFile.write( + ' IceInternal::Property("%s.%s", false, 0),\n' + % (self.currentSection, self.fix(propertyName)) + ) def newSection(self): - self.hFile.write(" static const PropertyArray %sProps;\n" % self.currentSection) - self.cppFile.write("const IceInternal::Property %sPropsData[] =\n" % self.currentSection) + self.hFile.write( + " static const PropertyArray %sProps;\n" % self.currentSection + ) + self.cppFile.write( + "const IceInternal::Property %sPropsData[] =\n" % self.currentSection + ) self.cppFile.write("{\n") def closeSection(self): self.cppFile.write("};\n") - self.cppFile.write(""" + self.cppFile.write( + """ const IceInternal::PropertyArray IceInternal::%(className)s::%(section)sProps(%(section)sPropsData, sizeof(%(section)sPropsData)/sizeof(%(section)sPropsData[0])); -""" % {'className': self.className, 'section': self.currentSection}) +""" + % {"className": self.className, "section": self.currentSection} + ) def moveFiles(self, location): dest = os.path.join(location, "cpp", "src", "Ice") @@ -407,14 +455,16 @@ def __init__(self, inputfile, c): self.srcFile = None def cleanup(self): - if self.srcFile != None: + if self.srcFile is not None: self.srcFile.close() if os.path.exists(self.className + ".java"): os.remove(self.className + ".java") def startFiles(self): self.srcFile = open(self.className + ".java", "w") - self.srcFile.write(javaPreamble % {'inputfile': self.inputfile, 'classname': self.className}) + self.srcFile.write( + javaPreamble % {"inputfile": self.inputfile, "classname": self.className} + ) def closeFiles(self): self.srcFile.write(" public static final Property[] validProps[] =\n") @@ -428,7 +478,7 @@ def closeFiles(self): self.srcFile.write("\n public static final String clPropNames[] =\n") self.srcFile.write(" {\n") for s in self.cmdLineOptions: - self.srcFile.write(" \"%s\",\n" % s) + self.srcFile.write(' "%s",\n' % s) self.srcFile.write(" null\n") self.srcFile.write(" };\n") self.srcFile.write("}\n") @@ -441,23 +491,34 @@ def fix(self, propertyName): return propertyName.replace(".", "\\\\.").replace("[any]", "[^\\\\s]+") def deprecatedImpl(self, propertyName): - self.srcFile.write(" new Property(\"%(section)s\\\\.%(pattern)s\", " - "true, null),\n" % - {"section": self.currentSection, "pattern": self.fix(propertyName)}) + self.srcFile.write( + ' new Property("%(section)s\\\\.%(pattern)s", ' + "true, null),\n" + % {"section": self.currentSection, "pattern": self.fix(propertyName)} + ) def deprecatedImplWithReplacementImpl(self, propertyName, deprecatedBy): - self.srcFile.write(" new Property(\"%(section)s\\\\.%(pattern)s\", " - "true, \"%(deprecatedBy)s\"),\n" % - {"section": self.currentSection, "pattern": self.fix(propertyName), - "deprecatedBy": deprecatedBy}) + self.srcFile.write( + ' new Property("%(section)s\\\\.%(pattern)s", ' + 'true, "%(deprecatedBy)s"),\n' + % { + "section": self.currentSection, + "pattern": self.fix(propertyName), + "deprecatedBy": deprecatedBy, + } + ) def propertyImpl(self, propertyName): - self.srcFile.write(" new Property(\"%(section)s\\\\.%(pattern)s\", " - "false, null),\n" % - {"section": self.currentSection, "pattern": self.fix(propertyName)}) + self.srcFile.write( + ' new Property("%(section)s\\\\.%(pattern)s", ' + "false, null),\n" + % {"section": self.currentSection, "pattern": self.fix(propertyName)} + ) def newSection(self): - self.srcFile.write(" public static final Property %sProps[] =\n" % self.currentSection) + self.srcFile.write( + " public static final Property %sProps[] =\n" % self.currentSection + ) self.srcFile.write(" {\n") def closeSection(self): @@ -465,7 +526,18 @@ def closeSection(self): self.srcFile.write(" };\n\n") def moveFiles(self, location): - dest = os.path.join(location, "java", "src", "Ice", "src", "main", "java", "com", "zeroc", "IceInternal") + dest = os.path.join( + location, + "java", + "src", + "Ice", + "src", + "main", + "java", + "com", + "zeroc", + "IceInternal", + ) if os.path.exists(os.path.join(dest, self.className + ".java")): os.remove(os.path.join(dest, self.className + ".java")) shutil.move(self.className + ".java", dest) @@ -477,14 +549,16 @@ def __init__(self, inputfile, c): self.srcFile = None def cleanup(self): - if self.srcFile != None: + if self.srcFile is not None: self.srcFile.close() if os.path.exists(self.className + ".cs"): os.remove(self.className + ".cs") def startFiles(self): self.srcFile = open(self.className + ".cs", "w") - self.srcFile.write(csPreamble % {'inputfile': self.inputfile, 'classname': self.className}) + self.srcFile.write( + csPreamble % {"inputfile": self.inputfile, "classname": self.className} + ) def closeFiles(self): self.srcFile.write(" public static Property[][] validProps =\n") @@ -497,7 +571,7 @@ def closeFiles(self): self.srcFile.write(" public static string[] clPropNames =\n") self.srcFile.write(" {\n") for s in self.cmdLineOptions: - self.srcFile.write(" \"%s\",\n" % s) + self.srcFile.write(' "%s",\n' % s) self.srcFile.write(" };\n") self.srcFile.write(" }\n") self.srcFile.write("}\n") @@ -507,19 +581,27 @@ def fix(self, propertyName): return propertyName.replace(".", "\\.").replace("[any]", "[^\\s]+") def deprecatedImpl(self, propertyName): - self.srcFile.write(" new Property(@\"^%s\.%s$\", true, null),\n" % (self.currentSection, - self.fix(propertyName))) + self.srcFile.write( + ' new Property(@"^%s\.%s$", true, null),\n' + % (self.currentSection, self.fix(propertyName)) + ) def deprecatedImplWithReplacementImpl(self, propertyName, deprecatedBy): - self.srcFile.write(" new Property(@\"^%s\.%s$\", true, @\"%s\"),\n" % - (self.currentSection, self.fix(propertyName), deprecatedBy)) + self.srcFile.write( + ' new Property(@"^%s\.%s$", true, @"%s"),\n' + % (self.currentSection, self.fix(propertyName), deprecatedBy) + ) def propertyImpl(self, propertyName): - self.srcFile.write(" new Property(@\"^%s\.%s$\", false, null),\n" % (self.currentSection, - self.fix(propertyName))) + self.srcFile.write( + ' new Property(@"^%s\.%s$", false, null),\n' + % (self.currentSection, self.fix(propertyName)) + ) def newSection(self): - self.srcFile.write(" public static Property[] %sProps =\n" % self.currentSection) + self.srcFile.write( + " public static Property[] %sProps =\n" % self.currentSection + ) self.srcFile.write(" {\n") def closeSection(self): @@ -540,14 +622,16 @@ def __init__(self, inputfile, c): self.validSections = ["Ice"] def cleanup(self): - if self.srcFile != None: + if self.srcFile is not None: self.srcFile.close() if os.path.exists(self.className + ".js"): os.remove(self.className + ".js") def startFiles(self): self.srcFile = open(self.className + ".js", "w") - self.srcFile.write(jsPreamble % {'inputfile': self.inputfile, 'classname': self.className}) + self.srcFile.write( + jsPreamble % {"inputfile": self.inputfile, "classname": self.className} + ) def closeFiles(self): self.srcFile.write("%s.validProps =\n" % (self.className)) @@ -561,10 +645,10 @@ def closeFiles(self): self.srcFile.write("[\n") for s in self.cmdLineOptions: if s in self.validSections: - self.srcFile.write(" \"%s\",\n" % s) + self.srcFile.write(' "%s",\n' % s) self.srcFile.write("];\n") - self.srcFile.write(jsEpilogue % {'classname': self.className}) + self.srcFile.write(jsEpilogue % {"classname": self.className}) self.srcFile.close() def fix(self, propertyName): @@ -572,18 +656,24 @@ def fix(self, propertyName): def deprecatedImpl(self, propertyName): if self.currentSection in self.validSections: - self.srcFile.write(" new Property(\"/^%s\.%s/\", true, null),\n" % (self.currentSection, - self.fix(propertyName))) + self.srcFile.write( + ' new Property("/^%s\.%s/", true, null),\n' + % (self.currentSection, self.fix(propertyName)) + ) def deprecatedImplWithReplacementImpl(self, propertyName, deprecatedBy): if self.currentSection in self.validSections: - self.srcFile.write(" new Property(\"/^%s\.%s/\", true, \"%s\"),\n" % - (self.currentSection, self.fix(propertyName), deprecatedBy)) + self.srcFile.write( + ' new Property("/^%s\.%s/", true, "%s"),\n' + % (self.currentSection, self.fix(propertyName), deprecatedBy) + ) def propertyImpl(self, propertyName): if self.currentSection in self.validSections: - self.srcFile.write(" new Property(\"/^%s\.%s/\", false, null),\n" % (self.currentSection, - self.fix(propertyName))) + self.srcFile.write( + ' new Property("/^%s\.%s/", false, null),\n' + % (self.currentSection, self.fix(propertyName)) + ) def newSection(self): if self.currentSection in self.validSections: @@ -697,12 +787,16 @@ def main(): className, ext = os.path.splitext(os.path.basename(infile)) global contentHandler - if lang == None: + if lang is None: contentHandler = MultiHandler(infile, "") - contentHandler.addHandlers([CppPropertyHandler(infile, className), - JavaPropertyHandler(infile, className), - CSPropertyHandler(infile, className), - JSPropertyHandler(infile, className)]) + contentHandler.addHandlers( + [ + CppPropertyHandler(infile, className), + JavaPropertyHandler(infile, className), + CSPropertyHandler(infile, className), + JSPropertyHandler(infile, className), + ] + ) else: if lang == "cpp": contentHandler = CppPropertyHandler(infile, className) diff --git a/cpp/test/Ice/invoke/test.py b/cpp/test/Ice/invoke/test.py index c2ab0da604a..730405c7ea3 100644 --- a/cpp/test/Ice/invoke/test.py +++ b/cpp/test/Ice/invoke/test.py @@ -2,7 +2,7 @@ # Copyright (c) ZeroC, Inc. All rights reserved. # -from Util import ClientServerTestCase, TestSuite, Server +from Util import ClientServerTestCase, Server, TestSuite TestSuite( diff --git a/csharp/allTests.py b/csharp/allTests.py index 94a9466c3db..58b0a12520c 100755 --- a/csharp/allTests.py +++ b/csharp/allTests.py @@ -5,6 +5,7 @@ import os import sys + sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts")) from Util import runTestsWithPath diff --git a/csharp/test/Ice/invoke/test.py b/csharp/test/Ice/invoke/test.py index da809b21a8c..117c0ec6b5d 100644 --- a/csharp/test/Ice/invoke/test.py +++ b/csharp/test/Ice/invoke/test.py @@ -2,6 +2,15 @@ # Copyright (c) ZeroC, Inc. All rights reserved. # -TestSuite(__file__, - [ClientServerTestCase(), - ClientServerTestCase(name="client/server async", servers=[Server(args=["--async"])])]) +from Util import ClientServerTestCase, Server, TestSuite + + +TestSuite( + __file__, + [ + ClientServerTestCase(), + ClientServerTestCase( + name="client/server async", servers=[Server(args=["--async"])] + ), + ], +) diff --git a/csharp/test/IceSSL/certs/makecerts.py b/csharp/test/IceSSL/certs/makecerts.py index a782e065f5a..93e6816dc21 100755 --- a/csharp/test/IceSSL/certs/makecerts.py +++ b/csharp/test/IceSSL/certs/makecerts.py @@ -5,18 +5,19 @@ import os import sys -import socket import getopt try: import IceCertUtils -except: - print("error: couldn't find IceCertUtils, install `zeroc-icecertutils' package " - "from Python package repository") +except ImportError: + print( + "error: couldn't find IceCertUtils, install `zeroc-icecertutils' package " + "from Python package repository" + ) sys.exit(1) toplevel = "." -while (os.path.abspath(toplevel) != "/"): +while os.path.abspath(toplevel) != "/": toplevel = os.path.normpath(os.path.join("..", toplevel)) if os.path.exists(os.path.join(toplevel, "scripts", "Util.py")): break @@ -25,8 +26,12 @@ cppcerts = os.path.join(toplevel, "cpp", "test", "IceSSL", "certs") if not os.path.exists(os.path.join(cppcerts, "db", "ca1", "ca.pem")): - print("error: CA database is not initialized in `" + os.path.join(cppcerts, "db") + "'," - " run makecerts.py in `" + cppcerts + "' first") + print( + "error: CA database is not initialized in `" + + os.path.join(cppcerts, "db") + + "'," + " run makecerts.py in `" + cppcerts + "' first" + ) sys.exit(1) @@ -52,7 +57,7 @@ def usage(): usage() sys.exit(1) -for (o, a) in opts: +for o, a in opts: if o == "-h" or o == "--help": usage() sys.exit(0) @@ -61,8 +66,12 @@ def usage(): elif o == "--force": force = True -ca1 = IceCertUtils.CertificateFactory(home=os.path.join(cppcerts, "db", "ca1"), debug=debug) -ca2 = IceCertUtils.CertificateFactory(home=os.path.join(cppcerts, "db", "ca2"), debug=debug) +ca1 = IceCertUtils.CertificateFactory( + home=os.path.join(cppcerts, "db", "ca1"), debug=debug +) +ca2 = IceCertUtils.CertificateFactory( + home=os.path.join(cppcerts, "db", "ca2"), debug=debug +) cai1 = ca1.getIntermediateFactory("intermediate1") cai2 = cai1.getIntermediateFactory("intermediate1") @@ -105,7 +114,7 @@ def usage(): # # Save the certificate PKCS12 files. # -for (ca, alias, path, args) in certs: +for ca, alias, path, args in certs: if not path: path = alias cert = ca.get(alias) diff --git a/java/allTests.py b/java/allTests.py index 94a9466c3db..58b0a12520c 100755 --- a/java/allTests.py +++ b/java/allTests.py @@ -5,6 +5,7 @@ import os import sys + sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts")) from Util import runTestsWithPath diff --git a/java/test/src/main/java/test/Ice/invoke/test.py b/java/test/src/main/java/test/Ice/invoke/test.py index da809b21a8c..117c0ec6b5d 100644 --- a/java/test/src/main/java/test/Ice/invoke/test.py +++ b/java/test/src/main/java/test/Ice/invoke/test.py @@ -2,6 +2,15 @@ # Copyright (c) ZeroC, Inc. All rights reserved. # -TestSuite(__file__, - [ClientServerTestCase(), - ClientServerTestCase(name="client/server async", servers=[Server(args=["--async"])])]) +from Util import ClientServerTestCase, Server, TestSuite + + +TestSuite( + __file__, + [ + ClientServerTestCase(), + ClientServerTestCase( + name="client/server async", servers=[Server(args=["--async"])] + ), + ], +) diff --git a/java/test/src/main/java/test/IceSSL/certs/makecerts.py b/java/test/src/main/java/test/IceSSL/certs/makecerts.py index 81c3ff4acc2..3d325cb73a0 100755 --- a/java/test/src/main/java/test/IceSSL/certs/makecerts.py +++ b/java/test/src/main/java/test/IceSSL/certs/makecerts.py @@ -5,18 +5,19 @@ import os import sys -import socket import getopt try: import IceCertUtils -except: - print("error: couldn't find IceCertUtils, install `zeroc-icecertutils' package " - "from Python package repository") +except ImportError: + print( + "error: couldn't find IceCertUtils, install `zeroc-icecertutils' package " + "from Python package repository" + ) sys.exit(1) toplevel = "." -while (os.path.abspath(toplevel) != "/"): +while os.path.abspath(toplevel) != "/": toplevel = os.path.normpath(os.path.join("..", toplevel)) if os.path.exists(os.path.join(toplevel, "scripts", "Util.py")): break @@ -25,8 +26,12 @@ cppcerts = os.path.join(toplevel, "cpp", "test", "IceSSL", "certs") if not os.path.exists(os.path.join(cppcerts, "db", "ca1", "ca.pem")): - print("error: CA database is not initialized in `" + os.path.join(cppcerts, "db") + "'," - " run makecerts.py in `" + cppcerts + "' first") + print( + "error: CA database is not initialized in `" + + os.path.join(cppcerts, "db") + + "'," + " run makecerts.py in `" + cppcerts + "' first" + ) sys.exit(1) @@ -52,7 +57,7 @@ def usage(): usage() sys.exit(1) -for (o, a) in opts: +for o, a in opts: if o == "-h" or o == "--help": usage() sys.exit(0) @@ -61,8 +66,12 @@ def usage(): elif o == "--force": force = True -ca1 = IceCertUtils.CertificateFactory(home=os.path.join(cppcerts, "db", "ca1"), debug=debug) -ca2 = IceCertUtils.CertificateFactory(home=os.path.join(cppcerts, "db", "ca2"), debug=debug) +ca1 = IceCertUtils.CertificateFactory( + home=os.path.join(cppcerts, "db", "ca1"), debug=debug +) +ca2 = IceCertUtils.CertificateFactory( + home=os.path.join(cppcerts, "db", "ca2"), debug=debug +) cai1 = ca1.getIntermediateFactory("intermediate1") cai2 = cai1.getIntermediateFactory("intermediate1") @@ -97,7 +106,7 @@ def usage(): # # Save the certificate JKS files. # -for (ca, alias, path, args) in certs: +for ca, alias, path, args in certs: if not path: path = alias cert = ca.get(alias) diff --git a/js/allTests.py b/js/allTests.py index 94a9466c3db..58b0a12520c 100755 --- a/js/allTests.py +++ b/js/allTests.py @@ -5,6 +5,7 @@ import os import sys + sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts")) from Util import runTestsWithPath diff --git a/php/allTests.py b/php/allTests.py index 94a9466c3db..58b0a12520c 100755 --- a/php/allTests.py +++ b/php/allTests.py @@ -5,6 +5,7 @@ import os import sys + sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts")) from Util import runTestsWithPath diff --git a/php/test/Ice/ini/test.py b/php/test/Ice/ini/test.py index eef8114afec..ed2c828458c 100644 --- a/php/test/Ice/ini/test.py +++ b/php/test/Ice/ini/test.py @@ -2,8 +2,10 @@ # Copyright (c) ZeroC, Inc. All rights reserved. # -class IniClient(Client): +from Util import Client, ClientTestCase, TestSuite + +class IniClient(Client): def __init__(self, iceOptions, iceProfile=None, *args, **kargs): Client.__init__(self, *args, **kargs) self.iceOptions = iceOptions @@ -11,11 +13,14 @@ def __init__(self, iceOptions, iceProfile=None, *args, **kargs): def setup(self, current): if self.iceProfile: - current.createFile("ice.profiles", [ - "[%s]" % self.iceProfile, - "ice.config=\"config.client\"", - "ice.options=\"%s\"" % self.iceOptions, - ]) + current.createFile( + "ice.profiles", + [ + "[%s]" % self.iceProfile, + 'ice.config="config.client"', + 'ice.options="%s"' % self.iceOptions, + ], + ) current.write("testing... ") def teardown(self, current, success): @@ -24,16 +29,30 @@ def teardown(self, current, success): def getPhpArgs(self, current): if self.iceProfile: - return ["-d", "ice.profiles=\"ice.profiles\""] + return ["-d", 'ice.profiles="ice.profiles"'] else: - return ["-d", "ice.options=\"{0}\"".format(self.iceOptions), "-d", "ice.config=\"config.client\""] + return [ + "-d", + 'ice.options="{0}"'.format(self.iceOptions), + "-d", + 'ice.config="config.client"', + ] -TestSuite(__name__, [ - ClientTestCase("php INI settings", - client=IniClient("--Ice.Trace.Network=1 --Ice.Warn.Connections=1")), - ClientTestCase("php INI settings with profiles", - client=IniClient("--Ice.Trace.Network=1 --Ice.Warn.Connections=1", - "Test", - exe="ClientWithProfile")) -]) +TestSuite( + __name__, + [ + ClientTestCase( + "php INI settings", + client=IniClient("--Ice.Trace.Network=1 --Ice.Warn.Connections=1"), + ), + ClientTestCase( + "php INI settings with profiles", + client=IniClient( + "--Ice.Trace.Network=1 --Ice.Warn.Connections=1", + "Test", + exe="ClientWithProfile", + ), + ), + ], +) diff --git a/ruby/allTests.py b/ruby/allTests.py index 94a9466c3db..58b0a12520c 100755 --- a/ruby/allTests.py +++ b/ruby/allTests.py @@ -5,6 +5,7 @@ import os import sys + sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts")) from Util import runTestsWithPath diff --git a/scripts/Component.py b/scripts/Component.py index d1ca2568719..47f144689ef 100644 --- a/scripts/Component.py +++ b/scripts/Component.py @@ -8,7 +8,6 @@ class Ice(Component): - # Options for all transports (ran only with Ice client/server tests defined for cross testing) transportOptions = { "protocol": ["tcp", "ssl", "wss", "ws"], @@ -41,12 +40,20 @@ def useBinDist(self, mapping, current): def getInstallDir(self, mapping, current): # On Windows, the Ice MSI installation can only be used for C++ - envHomeName = None if isinstance(platform, Windows) and not isinstance(mapping, CppMapping) else "ICE_HOME" + envHomeName = ( + None + if isinstance(platform, Windows) and not isinstance(mapping, CppMapping) + else "ICE_HOME" + ) return Component._getInstallDir(self, mapping, current, envHomeName) def getPhpExtension(self, mapping, current): if isinstance(platform, Windows): - return "php_ice.dll" if current.driver.configs[mapping].buildConfig in ["Debug", "Release"] else "php_ice_nts.dll" + return ( + "php_ice.dll" + if current.driver.configs[mapping].buildConfig in ["Debug", "Release"] + else "php_ice_nts.dll" + ) else: return "ice.so" @@ -54,44 +61,74 @@ def getNugetPackageVersionFile(self, mapping): if isinstance(mapping, CSharpMapping): return os.path.join(toplevel, "csharp", "msbuild", "zeroc.ice.net.nuspec") else: - return os.path.join(toplevel, "cpp", "msbuild", "zeroc.ice.{0}.nuspec".format(platform.getPlatformToolset())) + return os.path.join( + toplevel, + "cpp", + "msbuild", + "zeroc.ice.{0}.nuspec".format(platform.getPlatformToolset()), + ) def getFilters(self, mapping, config): if "xcodesdk" in config.buildConfig: - return (["Ice/.*", "IceSSL/configuration"], - ["Ice/background", - "Ice/echo", - "Ice/faultTolerance", - "Ice/library", - "Ice/logger", - "Ice/properties", - "Ice/plugin", - "Ice/stringConverter", - "Ice/threadPoolPriority", - "Ice/udp"]) + return ( + ["Ice/.*", "IceSSL/configuration"], + [ + "Ice/background", + "Ice/echo", + "Ice/faultTolerance", + "Ice/library", + "Ice/logger", + "Ice/properties", + "Ice/plugin", + "Ice/stringConverter", + "Ice/threadPoolPriority", + "Ice/udp", + ], + ) elif "static" in config.buildConfig: - return (["Ice/.*", "IceSSL/configuration", "IceDiscovery/simple", "IceGrid/simple", "Glacier2/application"], - ["Ice/library", "Ice/plugin"]) + return ( + [ + "Ice/.*", + "IceSSL/configuration", + "IceDiscovery/simple", + "IceGrid/simple", + "Glacier2/application", + ], + ["Ice/library", "Ice/plugin"], + ) elif isinstance(mapping, JavaMapping) and config.android: - return (["Ice/.*"], - ["Ice/hash", - "Ice/faultTolerance", - "Ice/metrics", - "Ice/networkProxy", - "Ice/throughput", - "Ice/plugin", - "Ice/logger", - "Ice/properties"]) + return ( + ["Ice/.*"], + [ + "Ice/hash", + "Ice/faultTolerance", + "Ice/metrics", + "Ice/networkProxy", + "Ice/throughput", + "Ice/plugin", + "Ice/logger", + "Ice/properties", + ], + ) elif isinstance(mapping, JavaScriptMapping): return ([], ["typescript/.*"]) - elif isinstance(mapping, SwiftMapping) and config.buildPlatform in ["iphonesimulator", "iphoneos"]: - return (["Ice/.*", "IceSSL/configuration", "Slice/*"], ["Ice/properties", "Ice/udp"]) + elif isinstance(mapping, SwiftMapping) and config.buildPlatform in [ + "iphonesimulator", + "iphoneos", + ]: + return ( + ["Ice/.*", "IceSSL/configuration", "Slice/*"], + ["Ice/properties", "Ice/udp"], + ) return ([], []) def canRun(self, testId, mapping, current): - parent = re.match(r'^([\w]*).*', testId).group(1) + parent = re.match(r"^([\w]*).*", testId).group(1) if isinstance(platform, Linux): - if platform.getLinuxId() in ["centos", "rhel", "fedora"] and current.config.buildPlatform == "x86": + if ( + platform.getLinuxId() in ["centos", "rhel", "fedora"] + and current.config.buildPlatform == "x86" + ): # # Don't test Glacier2/IceStorm/IceGrid services with multilib platforms. We only # build services for the native platform. @@ -104,10 +141,15 @@ def canRun(self, testId, mapping, current): # with the Debug configurations since we don't provide binaries for them. # if self.useBinDist(mapping, current): - if parent in ["Glacier2", "IceBridge"] and current.config.buildConfig.find("Debug") >= 0: + if ( + parent in ["Glacier2", "IceBridge"] + and current.config.buildConfig.find("Debug") >= 0 + ): return False elif isinstance(platform, AIX): - if current.config.buildPlatform == "ppc" and self.useBinDist(mapping, current): + if current.config.buildPlatform == "ppc" and self.useBinDist( + mapping, current + ): # # Don't test Glacier2, IceBridge and IceGrid services on ppc with bindist. We only ship # ppc64 binaries for these services @@ -118,11 +160,11 @@ def canRun(self, testId, mapping, current): return False # No C++98 tests for Glacier2, IceGrid, IceStorm, IceBridge - if isinstance(mapping, CppMapping) and not current.config.cpp11 and parent in [ - "Glacier2", - "IceBridge", - "IceGrid", - "IceStorm"]: + if ( + isinstance(mapping, CppMapping) + and not current.config.cpp11 + and parent in ["Glacier2", "IceBridge", "IceGrid", "IceStorm"] + ): return False return True @@ -140,20 +182,33 @@ def getDefaultProcesses(self, mapping, processType, testId): return [IceGridServer()] def getOptions(self, testcase, current): - - parent = re.match(r'^([\w]*).*', testcase.getTestSuite().getId()).group(1) - if parent not in ["Ice", "IceBox", "IceGrid", "Glacier2", "IceStorm", "IceDiscovery", "IceBridge"]: + parent = re.match(r"^([\w]*).*", testcase.getTestSuite().getId()).group(1) + if parent not in [ + "Ice", + "IceBox", + "IceGrid", + "Glacier2", + "IceStorm", + "IceDiscovery", + "IceBridge", + ]: return None if isinstance(testcase, CollocatedTestCase): return None # Define here Ice tests which are slow to execute and for which it's not useful to test different options - if testcase.getTestSuite().getId() in ["Ice/binding", "Ice/faultTolerance", "Ice/location"]: + if testcase.getTestSuite().getId() in [ + "Ice/binding", + "Ice/faultTolerance", + "Ice/location", + ]: return self.serviceOptions # We only run the client/server tests defined for cross testing with all transports - if isinstance(testcase, ClientServerTestCase) and self.isCross(testcase.getTestSuite().getId()): + if isinstance(testcase, ClientServerTestCase) and self.isCross( + testcase.getTestSuite().getId() + ): return self.transportOptions elif parent in ["Ice", "IceBox"]: return self.coreOptions @@ -161,7 +216,16 @@ def getOptions(self, testcase, current): return self.serviceOptions def getRunOrder(self): - return ["Slice", "IceUtil", "Ice", "IceSSL", "IceBox", "Glacier2", "IceGrid", "IceStorm"] + return [ + "Slice", + "IceUtil", + "Ice", + "IceSSL", + "IceBox", + "Glacier2", + "IceGrid", + "IceStorm", + ] def isCross(self, testId): return testId in [ @@ -180,17 +244,21 @@ def isCross(self, testId): ] def getSoVersion(self): - with open(os.path.join(toplevel, "cpp", "include", "IceUtil", "Config.h"), "r") as config: - intVersion = int(re.search("ICE_INT_VERSION ([0-9]*)", config.read()).group(1)) + with open( + os.path.join(toplevel, "cpp", "include", "IceUtil", "Config.h"), "r" + ) as config: + intVersion = int( + re.search("ICE_INT_VERSION ([0-9]*)", config.read()).group(1) + ) majorVersion = int(intVersion / 10000) minorVersion = int(intVersion / 100) - 100 * majorVersion patchVersion = intVersion % 100 if patchVersion < 50: - return '%d' % (majorVersion * 10 + minorVersion) + return "%d" % (majorVersion * 10 + minorVersion) elif patchVersion < 60: - return '%da%d' % (majorVersion * 10 + minorVersion, patchVersion - 50) + return "%da%d" % (majorVersion * 10 + minorVersion, patchVersion - 50) else: - return '%db%d' % (majorVersion * 10 + minorVersion, patchVersion - 60) + return "%db%d" % (majorVersion * 10 + minorVersion, patchVersion - 60) component = Ice() @@ -205,7 +273,9 @@ def getSoVersion(self): # # Supported mappings # -for m in filter(lambda x: os.path.isdir(os.path.join(toplevel, x)), os.listdir(toplevel)): +for m in filter( + lambda x: os.path.isdir(os.path.join(toplevel, x)), os.listdir(toplevel) +): if m == "cpp" or re.match("cpp-.*", m): Mapping.add(m, CppMapping(), component) elif m == "java" or re.match("java-.*", m): @@ -213,17 +283,34 @@ def getSoVersion(self): elif m == "python" or re.match("python-.*", m): Mapping.add(m, PythonMapping(), component) elif m == "ruby" or re.match("ruby-.*", m): - Mapping.add(m, RubyMapping(), component, enable=not isinstance(platform, Windows)) + Mapping.add( + m, RubyMapping(), component, enable=not isinstance(platform, Windows) + ) elif m == "php" or re.match("php-.*", m): - Mapping.add(m, PhpMapping(), component, enable=not isinstance(platform, Windows)) + Mapping.add( + m, PhpMapping(), component, enable=not isinstance(platform, Windows) + ) elif m == "js" or re.match("js-.*", m): Mapping.add(m, JavaScriptMapping(), component, enable=platform.hasNodeJS()) - Mapping.add("typescript", TypeScriptMapping(), component, "js", enable=platform.hasNodeJS()) + Mapping.add( + "typescript", + TypeScriptMapping(), + component, + "js", + enable=platform.hasNodeJS(), + ) elif m == "swift" or re.match("swift-.*", m): # Swift mapping requires Swift 5.0 or greater - Mapping.add("swift", SwiftMapping(), component, enable=platform.hasSwift((5, 0))) + Mapping.add( + "swift", SwiftMapping(), component, enable=platform.hasSwift((5, 0)) + ) elif m == "csharp" or re.match("charp-.*", m): - Mapping.add("csharp", CSharpMapping(), component, enable=isinstance(platform, Windows) or platform.hasDotNet()) + Mapping.add( + "csharp", + CSharpMapping(), + component, + enable=isinstance(platform, Windows) or platform.hasDotNet(), + ) # # Check if Matlab is installed and eventually add the Matlab mapping diff --git a/scripts/Controller.py b/scripts/Controller.py index 1e4b4fd7605..61ec8aed15e 100755 --- a/scripts/Controller.py +++ b/scripts/Controller.py @@ -9,11 +9,11 @@ class ControllerDriver(Driver): - class Current(Driver.Current): - def __init__(self, driver, testsuite, testcase, cross, protocol, host, args): - Driver.Current.__init__(self, driver, testsuite, Result(testsuite, driver.debug)) + Driver.Current.__init__( + self, driver, testsuite, Result(testsuite, driver.debug) + ) self.testcase = testcase self.serverTestCase = self.testcase.getServerTestCase(cross) self.clientTestCase = self.testcase.getClientTestCase() @@ -45,7 +45,6 @@ def __init__(self, options, *args, **kargs): self.endpoints = ("tcp -h " + self.interface) if self.interface else "tcp" def run(self, mappings, testSuiteIds): - if isinstance(platform, Darwin): # # On macOS, we set the trust settings on the certificate to prevent @@ -55,27 +54,47 @@ def run(self, mappings, testSuiteIds): # serverCert = os.path.join(toplevel, "certs", "server.pem") if self.clean: - if os.system("security verify-cert -c " + serverCert + " >& /dev/null") == 0: - sys.stdout.write("removing trust settings for the HTTP server certificate... ") + if ( + os.system("security verify-cert -c " + serverCert + " >& /dev/null") + == 0 + ): + sys.stdout.write( + "removing trust settings for the HTTP server certificate... " + ) sys.stdout.flush() if os.system("security remove-trusted-cert " + serverCert) != 0: - print("\nerror: couldn't remove trust settings for the HTTP server certificate") + print( + "\nerror: couldn't remove trust settings for the HTTP server certificate" + ) else: print("ok") else: print("trust settings already removed") return else: - if os.system("security verify-cert -c " + serverCert + " >& /dev/null") != 0: - sys.stdout.write("adding trust settings for the HTTP server certificate... ") + if ( + os.system("security verify-cert -c " + serverCert + " >& /dev/null") + != 0 + ): + sys.stdout.write( + "adding trust settings for the HTTP server certificate... " + ) sys.stdout.flush() - if os.system("security add-trusted-cert -r trustAsRoot " + serverCert) != 0: - print("error: couldn't add trust settings for the HTTP server certificate") + if ( + os.system( + "security add-trusted-cert -r trustAsRoot " + serverCert + ) + != 0 + ): + print( + "error: couldn't add trust settings for the HTTP server certificate" + ) print("ok") print("run " + sys.argv[0] + " --clean to remove the trust setting") self.initCommunicator() import Ice + Ice.loadSlice(os.path.join(toplevel, "scripts", "Controller.ice")) import Test @@ -93,15 +112,20 @@ def startServerSide(self, config, c): except Exception as ex: self.serverSideRunning = False raise Test.Common.TestCaseFailedException( - self.current.result.getOutput() + "\n" + traceback.format_exc()) + self.current.result.getOutput() + "\n" + traceback.format_exc() + ) def stopServerSide(self, success, c): if self.serverSideRunning: try: - self.current.serverTestCase._stopServerSide(self.current, success) + self.current.serverTestCase._stopServerSide( + self.current, success + ) return self.current.result.getOutput() except Exception as ex: - raise Test.Common.TestCaseFailedException(self.current.result.getOutput() + "\n" + str(ex)) + raise Test.Common.TestCaseFailedException( + self.current.result.getOutput() + "\n" + str(ex) + ) def runClientSide(self, host, config, c): self.updateCurrent(config) @@ -109,7 +133,9 @@ def runClientSide(self, host, config, c): self.current.clientTestCase._runClientSide(self.current, host) return self.current.result.getOutput() except Exception as ex: - raise Test.Common.TestCaseFailedException(self.current.result.getOutput() + "\n" + str(ex)) + raise Test.Common.TestCaseFailedException( + self.current.result.getOutput() + "\n" + str(ex) + ) def destroy(self, c): if self.serverSideRunning: @@ -120,7 +146,15 @@ def destroy(self, c): c.adapter.remove(c.id) def updateCurrent(self, config): - attrs = ["protocol", "mx", "serialize", "compress", "ipv6", "cprops", "sprops"] + attrs = [ + "protocol", + "mx", + "serialize", + "compress", + "ipv6", + "cprops", + "sprops", + ] for a in attrs: v = getattr(config, a) if v is not Ice.Unset: @@ -144,43 +178,67 @@ def runTestCase(self, mapping, testsuite, testcase, cross, c): current = self.driver.getCurrent(mapping, testsuite, testcase, cross) prx = c.adapter.addWithUUID(TestCaseI(self.driver, current)) self.testcase = Test.Common.TestCasePrx.uncheckedCast( - c.adapter.createDirectProxy(prx.ice_getIdentity())) + c.adapter.createDirectProxy(prx.ice_getIdentity()) + ) return self.testcase def getTestSuites(self, mapping, c): mapping = Mapping.getByName(mapping) config = self.driver.configs[mapping] - return [str(t) for t in mapping.getTestSuites() if not mapping.filterTestSuite(t.getId(), config)] + return [ + str(t) + for t in mapping.getTestSuites() + if not mapping.filterTestSuite(t.getId(), config) + ] def getOptionOverrides(self, c): - return Test.Common.OptionOverrides(ipv6=([False] if not self.driver.hostIPv6 else [False, True])) + return Test.Common.OptionOverrides( + ipv6=([False] if not self.driver.hostIPv6 else [False, True]) + ) self.initCommunicator() - self.communicator.getProperties().setProperty("ControllerAdapter.Endpoints", self.endpoints) - self.communicator.getProperties().setProperty("ControllerAdapter.AdapterId", Ice.generateUUID()) + self.communicator.getProperties().setProperty( + "ControllerAdapter.Endpoints", self.endpoints + ) + self.communicator.getProperties().setProperty( + "ControllerAdapter.AdapterId", Ice.generateUUID() + ) adapter = self.communicator.createObjectAdapter("ControllerAdapter") adapter.add(ControllerI(self), self.communicator.stringToIdentity(self.id)) adapter.activate() self.communicator.waitForShutdown() - def getCurrent(self, mapping, testsuite, testcase, cross, protocol=None, host=None, args=[]): + def getCurrent( + self, mapping, testsuite, testcase, cross, protocol=None, host=None, args=[] + ): import Test + mapping = Mapping.getByName(mapping) if not mapping: - raise Test.Common.TestCaseNotExistException("unknown mapping {0}".format(mapping)) + raise Test.Common.TestCaseNotExistException( + "unknown mapping {0}".format(mapping) + ) if cross: cross = Mapping.getByName(cross) if not cross: - raise Test.Common.TestCaseNotExistException("unknown mapping {0} for cross testing".format(cross)) + raise Test.Common.TestCaseNotExistException( + "unknown mapping {0} for cross testing".format(cross) + ) ts = mapping.findTestSuite(testsuite) if not ts: - raise Test.Common.TestCaseNotExistException("unknown testsuite {0}".format(testsuite)) + raise Test.Common.TestCaseNotExistException( + "unknown testsuite {0}".format(testsuite) + ) - tc = ts.findTestCase("server" if ts.getId() == "Ice/echo" else (testcase or "client/server")) + tc = ts.findTestCase( + "server" if ts.getId() == "Ice/echo" else (testcase or "client/server") + ) if not tc or not tc.getServerTestCase(): - raise Test.Common.TestCaseNotExistException("unknown testcase {0}".format(testcase)) + raise Test.Common.TestCaseNotExistException( + "unknown testcase {0}".format(testcase) + ) return ControllerDriver.Current(self, ts, tc, cross, protocol, host, args) diff --git a/scripts/Expect.py b/scripts/Expect.py index 118cabadb9f..5a3c60955d1 100755 --- a/scripts/Expect.py +++ b/scripts/Expect.py @@ -16,14 +16,13 @@ __all__ = ["Expect", "EOF", "TIMEOUT"] -win32 = (sys.platform == "win32") +win32 = sys.platform == "win32" if win32: import ctypes class EOF: - """Raised when EOF is read from a child. - """ + """Raised when EOF is read from a child.""" def __init__(self, value): self.value = value @@ -33,8 +32,7 @@ def __str__(self): class TIMEOUT(Exception): - """Raised when a read time exceeds the timeout. - """ + """Raised when a read time exceeds the timeout.""" def __init__(self, value): self.value = value @@ -48,35 +46,40 @@ def escape(s, escapeNewlines=True): return "" o = io.StringIO() for c in s: - if c == '\\': - o.write('\\\\') - elif c == '\'': + if c == "\\": + o.write("\\\\") + elif c == "'": o.write("\\'") - elif c == '\"': + elif c == '"': o.write('\\"') - elif c == '\b': - o.write('\\b') - elif c == '\f': - o.write('\\f') - elif c == '\n': + elif c == "\b": + o.write("\\b") + elif c == "\f": + o.write("\\f") + elif c == "\n": if escapeNewlines: - o.write('\\n') + o.write("\\n") else: - o.write('\n') - elif c == '\r': - o.write('\\r') - elif c == '\t': - o.write('\\t') + o.write("\n") + elif c == "\r": + o.write("\\r") + elif c == "\t": + o.write("\\t") else: if c in string.printable: o.write(c) else: - o.write('\\%03o' % ord(c)) + o.write("\\%03o" % ord(c)) return o.getvalue() def taskkill(args): - p = subprocess.Popen("taskkill {0}".format(args), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + p = subprocess.Popen( + "taskkill {0}".format(args), + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) p.wait() p.stdout.close() @@ -97,7 +100,9 @@ def terminateProcess(p, hasInterruptSupport=True): # if hasInterruptSupport: try: - ctypes.windll.kernel32.GenerateConsoleCtrlEvent(1, p.pid) # 1 is CTRL_BREAK_EVENT + ctypes.windll.kernel32.GenerateConsoleCtrlEvent( + 1, p.pid + ) # 1 is CTRL_BREAK_EVENT except NameError: taskkill("/F /T /PID {0}".format(p.pid)) pass @@ -138,7 +143,7 @@ def run(self): self.cv.notify() self.cv.release() break - if c == '\r': + if c == "\r": continue self.cv.acquire() @@ -162,11 +167,13 @@ def trace(self, c): if self._tracesuppress: if c is not None: self._tbuf.write(c) - if c == '\n' or c is None: + if c == "\n" or c is None: content = self._tbuf.getvalue() suppress = False for p in self._tracesuppress: - if isinstance(p, types.LambdaType) or isinstance(p, types.FunctionType): + if isinstance(p, types.LambdaType) or isinstance( + p, types.FunctionType + ): content = p(content) elif p.search(content): suppress = True @@ -216,13 +223,15 @@ def match(self, pattern, timeout, matchall=False): if len(p) == 1: pdesc.write(escape(p[0])) else: - pdesc.write('[') + pdesc.write("[") for pat in p: if pat != p[0]: - pdesc.write(',') + pdesc.write(",") pdesc.write(escape(pat)) - pdesc.write(']') - self.logfile.write('%s: expect: "%s" timeout: %s\n' % (self.desc, pdesc.getvalue(), tdesc)) + pdesc.write("]") + self.logfile.write( + '%s: expect: "%s" timeout: %s\n' % (self.desc, pdesc.getvalue(), tdesc) + ) self.logfile.flush() maxend = None @@ -245,9 +254,9 @@ def match(self, pattern, timeout, matchall=False): break m = regexp.search(buf) if m is not None: - before = buf[:m.start()] - matched = buf[m.start():m.end()] - after = buf[m.end():] + before = buf[: m.start()] + matched = buf[m.start() : m.end()] + after = buf[m.end() :] if maxend is None or m.end() > maxend: maxend = m.end() @@ -256,13 +265,27 @@ def match(self, pattern, timeout, matchall=False): if self.logfile: if len(pattern) > 1: self.logfile.write( - '%s: match found in %.2fs.\npattern: "%s"\nbuffer: "%s||%s||%s"\n' % - (self.desc, time.time() - start, escape(s), escape(before), - escape(matched), escape(after))) + '%s: match found in %.2fs.\npattern: "%s"\nbuffer: "%s||%s||%s"\n' + % ( + self.desc, + time.time() - start, + escape(s), + escape(before), + escape(matched), + escape(after), + ) + ) else: - self.logfile.write('%s: match found in %.2fs.\nbuffer: "%s||%s||%s"\n' % - (self.desc, time.time() - start, escape(before), escape(matched), - escape(after))) + self.logfile.write( + '%s: match found in %.2fs.\nbuffer: "%s||%s||%s"\n' + % ( + self.desc, + time.time() - start, + escape(before), + escape(matched), + escape(after), + ) + ) if matchall: del pattern[index] @@ -289,8 +312,10 @@ def match(self, pattern, timeout, matchall=False): # If no match and we have finished processing output raise a TIMEOUT if self._finish: - raise TIMEOUT('timeout exceeded in match\npattern: "%s"\nbuffer: "%s"\n' % - (escape(s), escape(buf, False))) + raise TIMEOUT( + 'timeout exceeded in match\npattern: "%s"\nbuffer: "%s"\n' + % (escape(s), escape(buf, False)) + ) if timeout is None: self.cv.wait() @@ -299,11 +324,15 @@ def match(self, pattern, timeout, matchall=False): if time.time() >= end: # Log the failure if self.logfile: - self.logfile.write('%s: match failed.\npattern: "%s"\nbuffer: "%s"\n"' % - (self.desc, escape(s), escape(buf))) + self.logfile.write( + '%s: match failed.\npattern: "%s"\nbuffer: "%s"\n"' + % (self.desc, escape(s), escape(buf)) + ) self.logfile.flush() - raise TIMEOUT('timeout exceeded in match\npattern: "%s"\nbuffer: "%s"\n' % - (escape(s), escape(buf, False))) + raise TIMEOUT( + 'timeout exceeded in match\npattern: "%s"\nbuffer: "%s"\n' + % (escape(s), escape(buf, False)) + ) except TIMEOUT as e: if (TIMEOUT, None) in pattern: return buf, buf, TIMEOUT, None, pattern.index((TIMEOUT, None)) @@ -314,7 +343,7 @@ def match(self, pattern, timeout, matchall=False): def splitCommand(command_line): arg_list = [] - arg = '' + arg = "" state_basic = 0 state_esc = 1 @@ -325,7 +354,7 @@ def splitCommand(command_line): pre_esc_state = state_basic for c in command_line: - if state != state_esc and c == '\\': + if state != state_esc and c == "\\": pre_esc_state = state state = state_esc elif state == state_basic or state == state_whitespace: @@ -338,7 +367,7 @@ def splitCommand(command_line): None else: arg_list.append(arg) - arg = '' + arg = "" state = state_whitespace else: arg = arg + c @@ -357,7 +386,7 @@ def splitCommand(command_line): else: arg = arg + c - if arg != '': + if arg != "": arg_list.append(arg) return arg_list @@ -375,16 +404,28 @@ def cleanup(): processes.clear() -class Expect (object): - def __init__(self, command, startReader=True, timeout=30, logfile=None, mapping=None, desc=None, cwd=None, env=None, - preexec_fn=None): +class Expect(object): + def __init__( + self, + command, + startReader=True, + timeout=30, + logfile=None, + mapping=None, + desc=None, + cwd=None, + env=None, + preexec_fn=None, + ): self.buf = "" # The part before the match self.before = "" # The part before the match self.after = "" # The part after the match self.matchindex = 0 # the index of the matched pattern self.match = None # The last match self.mapping = mapping # The mapping of the test. - self.exitstatus = None # The exitstatus, either -signal or, if positive, the exit code. + self.exitstatus = ( + None # The exitstatus, either -signal or, if positive, the exit code. + ) self.killed = None # If killed, the signal that was sent. self.desc = desc self.logfile = logfile @@ -407,13 +448,30 @@ def __init__(self, command, startReader=True, timeout=30, logfile=None, mapping= # command. # CREATE_NEW_PROCESS_GROUP = 512 - self.p = subprocess.Popen(command, env=env, cwd=cwd, shell=False, bufsize=0, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - creationflags=CREATE_NEW_PROCESS_GROUP, universal_newlines=True) + self.p = subprocess.Popen( + command, + env=env, + cwd=cwd, + shell=False, + bufsize=0, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + creationflags=CREATE_NEW_PROCESS_GROUP, + universal_newlines=True, + ) else: - self.p = subprocess.Popen(splitCommand(command), env=env, cwd=cwd, shell=False, bufsize=0, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - preexec_fn=preexec_fn) + self.p = subprocess.Popen( + splitCommand(command), + env=env, + cwd=cwd, + shell=False, + bufsize=0, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + preexec_fn=preexec_fn, + ) global processes processes[self.p.pid] = self.p @@ -429,7 +487,9 @@ def __init__(self, command, startReader=True, timeout=30, logfile=None, mapping= self.startReader() def __str__(self): - return "{0} pid={1}".format(self.desc, "" if self.p is None else self.p.pid) + return "{0} pid={1}".format( + self.desc, "" if self.p is None else self.p.pid + ) def startReader(self, watchDog=None): if watchDog is not None: @@ -439,9 +499,9 @@ def startReader(self, watchDog=None): def expect(self, pattern, timeout=60): """pattern is either a string, or a list of string regexp patterns. - timeout == None expect can block indefinitely. + timeout == None expect can block indefinitely. - timeout == -1 then the default is used. + timeout == -1 then the default is used. """ if timeout == -1: timeout = self.timeout @@ -456,7 +516,13 @@ def compile(s): pattern = [(p, compile(p)) for p in pattern] try: - self.buf, self.before, self.after, self.match, self.matchindex = self.r.match(pattern, timeout) + ( + self.buf, + self.before, + self.after, + self.match, + self.matchindex, + ) = self.r.match(pattern, timeout) except TIMEOUT as e: self.buf = "" self.before = "" @@ -469,9 +535,9 @@ def compile(s): def expectall(self, pattern, timeout=60): """pattern is a list of string regexp patterns. - timeout == None expect can block indefinitely. + timeout == None expect can block indefinitely. - timeout == -1 then the default is used. + timeout == -1 then the default is used. """ if timeout == -1: timeout = self.timeout @@ -492,8 +558,7 @@ def expectall(self, pattern, timeout=60): raise e def sendline(self, data): - """send data to the application. - """ + """send data to the application.""" if self.logfile: self.logfile.write('%s: sendline: "%s"\n' % (self.desc, escape(data))) self.logfile.flush() @@ -505,11 +570,11 @@ def sendline(self, data): def wait(self, timeout=None): """Wait for the application to terminate for up to timeout seconds, or - raises a TIMEOUT exception. If timeout is None, the wait is - indefinite. + raises a TIMEOUT exception. If timeout is None, the wait is + indefinite. - The exit status is returned. A negative exit status means - the application was killed by a signal. + The exit status is returned. A negative exit status means + the application was killed by a signal. """ if self.p is None: return self.exitstatus @@ -521,7 +586,7 @@ def wait(self, timeout=None): while time.time() < end and self.p and self.p.poll() is None: time.sleep(0.1) if self.p and self.p.poll() is None: - raise TIMEOUT('timed wait exceeded timeout') + raise TIMEOUT("timed wait exceeded timeout") elif win32: # We poll on Windows or otherwise KeyboardInterrupt isn't delivered while self.p.poll() is None: @@ -619,7 +684,7 @@ def trace(self, suppress=None): def waitSuccess(self, exitstatus=0, timeout=None): """Wait for the process to terminate for up to timeout seconds, and - validate the exit status is as expected.""" + validate the exit status is as expected.""" self.wait(timeout) self.testExitStatus(exitstatus) @@ -633,16 +698,25 @@ def hasInterruptSupport(self): return True def testExitStatus(self, exitstatus): - def test(result, expected): - if not win32 and result == -2: # Interrupted by Ctrl-C, simulate KeyboardInterrupt + if ( + not win32 and result == -2 + ): # Interrupted by Ctrl-C, simulate KeyboardInterrupt raise KeyboardInterrupt() if isinstance(expected, list): if result not in expected: - raise RuntimeError("unexpected exit status: expected either: {0}, got {1}".format(expected, result)) + raise RuntimeError( + "unexpected exit status: expected either: {0}, got {1}".format( + expected, result + ) + ) else: if expected != result: - raise RuntimeError("unexpected exit status: expected: {0}, got {1}\n".format(expected, result)) + raise RuntimeError( + "unexpected exit status: expected: {0}, got {1}\n".format( + expected, result + ) + ) if self.killed is not None: # diff --git a/scripts/Glacier2Util.py b/scripts/Glacier2Util.py index e3dadd45618..d74e9bb0ea0 100644 --- a/scripts/Glacier2Util.py +++ b/scripts/Glacier2Util.py @@ -8,10 +8,16 @@ class Glacier2Router(ProcessFromBinDir, ProcessIsReleaseOnly, Server): - def __init__(self, portnum=50, passwords={"userid": "abc123"}, *args, **kargs): - Server.__init__(self, "glacier2router++11", mapping=Mapping.getByName("cpp"), desc="Glacier2 router", - readyCount=2, *args, **kargs) + Server.__init__( + self, + "glacier2router++11", + mapping=Mapping.getByName("cpp"), + desc="Glacier2 router", + readyCount=2, + *args, + **kargs, + ) self.portnum = portnum self.passwords = passwords @@ -22,8 +28,12 @@ def setup(self, current): if self.passwords: path = os.path.join(current.testsuite.getPath(), "passwords") with open(path, "w") as file: - command = "\"%s\" %s" % (sys.executable, - os.path.abspath(os.path.join(toplevel, "scripts", "icehashpassword.py"))) + command = '"%s" %s' % ( + sys.executable, + os.path.abspath( + os.path.join(toplevel, "scripts", "icehashpassword.py") + ), + ) # # For Linux ARM default rounds makes test slower (Usually runs on embedded boards) @@ -32,19 +42,29 @@ def setup(self, current): command += " --rounds 100000" for user, password in self.passwords.items(): - file.write("%s %s\n" % (user, run(command, stdin=(password + '\r\n').encode('UTF-8')))) + file.write( + "%s %s\n" + % ( + user, + run(command, stdin=(password + "\r\n").encode("UTF-8")), + ) + ) current.files.append(path) def getProps(self, current): props = Server.getProps(self, current) - props.update({ - "Glacier2.Client.Endpoints": current.getTestEndpoint(self.portnum), - "Glacier2.Server.Endpoints": "tcp", - "Ice.Admin.Endpoints": current.getTestEndpoint(self.portnum + 1), - "Ice.Admin.InstanceName": "Glacier2", - }) + props.update( + { + "Glacier2.Client.Endpoints": current.getTestEndpoint(self.portnum), + "Glacier2.Server.Endpoints": "tcp", + "Ice.Admin.Endpoints": current.getTestEndpoint(self.portnum + 1), + "Ice.Admin.InstanceName": "Glacier2", + } + ) if self.passwords: - props["Glacier2.CryptPasswords"] = os.path.join(current.testsuite.getPath(), "passwords") + props["Glacier2.CryptPasswords"] = os.path.join( + current.testsuite.getPath(), "passwords" + ) if isinstance(current.testcase.getTestSuite(), Glacier2TestSuite): # Add the properties provided by the Glacier2TestSuite routerProps parameter. props.update(current.testcase.getTestSuite().getRouterProps(self, current)) @@ -55,7 +75,6 @@ def getClientProxy(self, current): class Glacier2TestSuite(TestSuite): - def __init__(self, path, routerProps={}, testcases=None, *args, **kargs): if testcases is None: testcases = [ClientServerTestCase(servers=[Glacier2Router(), Server()])] @@ -63,4 +82,8 @@ def __init__(self, path, routerProps={}, testcases=None, *args, **kargs): self.routerProps = routerProps def getRouterProps(self, process, current): - return self.routerProps(process, current) if callable(self.routerProps) else self.routerProps.copy() + return ( + self.routerProps(process, current) + if callable(self.routerProps) + else self.routerProps.copy() + ) diff --git a/scripts/IceBoxUtil.py b/scripts/IceBoxUtil.py index 2512d3b6ade..e9f7a36f321 100644 --- a/scripts/IceBoxUtil.py +++ b/scripts/IceBoxUtil.py @@ -7,7 +7,6 @@ class IceBox(ProcessFromBinDir, Server): - def __init__(self, configFile=None, *args, **kargs): Server.__init__(self, *args, **kargs) self.configFile = configFile @@ -20,12 +19,13 @@ def getExe(self, current): return "iceboxnet" else: name = "icebox" - if isinstance(platform, Linux) and \ - platform.getLinuxId() in ["centos", "rhel", "fedora"] and \ - current.config.buildPlatform == "x86": + if ( + isinstance(platform, Linux) + and platform.getLinuxId() in ["centos", "rhel", "fedora"] + and current.config.buildPlatform == "x86" + ): name += "32" # Multilib platform - if isinstance(platform, AIX) and \ - current.config.buildPlatform == "ppc": + if isinstance(platform, AIX) and current.config.buildPlatform == "ppc": name += "_32" if current.config.cpp11: name += "++11" @@ -36,14 +36,17 @@ def getEffectiveArgs(self, current, args): if self.configFile: mapping = self.getMapping(current) if isinstance(mapping, CSharpMapping) and current.config.dotnet: - args.append("--Ice.Config={0}.{1}".format(self.configFile, mapping.getTargetFramework(current))) + args.append( + "--Ice.Config={0}.{1}".format( + self.configFile, mapping.getTargetFramework(current) + ) + ) else: args.append("--Ice.Config={0}".format(self.configFile)) return args class IceBoxAdmin(ProcessFromBinDir, ProcessIsReleaseOnly, Client): - def getMapping(self, current): # IceBox admin is only provided with the C++/Java, not C# mapping = Client.getMapping(self, current) @@ -56,8 +59,11 @@ def getExe(self, current): mapping = self.getMapping(current) if isinstance(mapping, JavaMapping): return "com.zeroc.IceBox.Admin" - elif isinstance(platform, AIX) and \ - current.config.buildPlatform == "ppc" and not component.useBinDist(mapping, current): + elif ( + isinstance(platform, AIX) + and current.config.buildPlatform == "ppc" + and not component.useBinDist(mapping, current) + ): return "iceboxadmin_32" else: return "iceboxadmin" diff --git a/scripts/IceBridgeUtil.py b/scripts/IceBridgeUtil.py index e616707234b..15de3277b7e 100644 --- a/scripts/IceBridgeUtil.py +++ b/scripts/IceBridgeUtil.py @@ -6,24 +6,35 @@ class IceBridge(ProcessFromBinDir, ProcessIsReleaseOnly, Server): - def __init__(self, *args, **kargs): - Server.__init__(self, "icebridge++11", mapping=Mapping.getByName("cpp"), desc="IceBridge", *args, **kargs) + Server.__init__( + self, + "icebridge++11", + mapping=Mapping.getByName("cpp"), + desc="IceBridge", + *args, + **kargs, + ) def getExe(self, current): return self.exe + "_32" if current.config.buildPlatform == "ppc" else self.exe def getProps(self, current): props = Server.getProps(self, current) - props.update({ - 'Ice.Warn.Dispatch': 0, - 'IceBridge.Target.Endpoints': current.getTestEndpoint(0) + ":" + current.getTestEndpoint(0, "udp"), - 'IceBridge.Source.Endpoints': current.getTestEndpoint(1) + ":" + current.getTestEndpoint(1, "udp"), - 'Ice.Admin.Endpoints': current.getTestEndpoint(2, "tcp"), - 'Ice.Admin.InstanceName': "IceBridge", - - }) - del props['Ice.ThreadPool.Server.Size'] - del props['Ice.ThreadPool.Server.SizeMax'] - del props['Ice.ThreadPool.Server.SizeWarn'] + props.update( + { + "Ice.Warn.Dispatch": 0, + "IceBridge.Target.Endpoints": current.getTestEndpoint(0) + + ":" + + current.getTestEndpoint(0, "udp"), + "IceBridge.Source.Endpoints": current.getTestEndpoint(1) + + ":" + + current.getTestEndpoint(1, "udp"), + "Ice.Admin.Endpoints": current.getTestEndpoint(2, "tcp"), + "Ice.Admin.InstanceName": "IceBridge", + } + ) + del props["Ice.ThreadPool.Server.Size"] + del props["Ice.ThreadPool.Server.SizeMax"] + del props["Ice.ThreadPool.Server.SizeWarn"] return props diff --git a/scripts/IceGridUtil.py b/scripts/IceGridUtil.py index 722ac2d9154..a544222d1d1 100644 --- a/scripts/IceGridUtil.py +++ b/scripts/IceGridUtil.py @@ -11,7 +11,6 @@ class IceGridProcess: - def __init__(self, replica): self.replica = replica @@ -32,28 +31,37 @@ def getProps(self, current): class IceGridServer(IceGridProcess, Server): - def __init__(self, replica=None, *args, **kargs): Server.__init__(self, *args, **kargs) IceGridProcess.__init__(self, replica) - getParentProps = Server.getProps # Used by IceGridProcess to get the server properties + getParentProps = ( + Server.getProps + ) # Used by IceGridProcess to get the server properties class IceGridClient(IceGridProcess, Client): - def __init__(self, replica=None, *args, **kargs): Client.__init__(self, *args, **kargs) IceGridProcess.__init__(self, replica) - getParentProps = Client.getProps # Used by IceGridProcess to get the client properties + getParentProps = ( + Client.getProps + ) # Used by IceGridProcess to get the client properties class IceGridAdmin(ProcessFromBinDir, ProcessIsReleaseOnly, IceGridClient): - - def __init__(self, replica=None, username="admin1", password="test1", *args, **kargs): - IceGridClient.__init__(self, replica=replica, exe="icegridadmin++11", mapping=Mapping.getByName("cpp"), - *args, **kargs) + def __init__( + self, replica=None, username="admin1", password="test1", *args, **kargs + ): + IceGridClient.__init__( + self, + replica=replica, + exe="icegridadmin++11", + mapping=Mapping.getByName("cpp"), + *args, + **kargs, + ) self.username = username self.password = password @@ -68,11 +76,16 @@ def getProps(self, current): class IceGridNode(ProcessFromBinDir, Server): - def __init__(self, name="localnode", *args, **kargs): - - Server.__init__(self, "icegridnode++11", mapping=Mapping.getByName("cpp"), desc="IceGrid node " + name, - ready="node", *args, **kargs) + Server.__init__( + self, + "icegridnode++11", + mapping=Mapping.getByName("cpp"), + desc="IceGrid node " + name, + ready="node", + *args, + **kargs, + ) self.name = name def getExe(self, current): @@ -80,7 +93,9 @@ def getExe(self, current): def setup(self, current): # Create the database directory - self.dbdir = os.path.join(current.testsuite.getPath(), "node-{0}".format(self.name)) + self.dbdir = os.path.join( + current.testsuite.getPath(), "node-{0}".format(self.name) + ) if os.path.exists(self.dbdir): shutil.rmtree(self.dbdir) os.mkdir(self.dbdir) @@ -96,22 +111,22 @@ def teardown(self, current, success): def getProps(self, current): props = { - 'IceGrid.InstanceName': 'TestIceGrid', - 'IceGrid.Node.Endpoints': 'default', - 'IceGrid.Node.WaitTime': 240, - 'Ice.ProgramName': 'icegridnode', - 'IceGrid.Node.Trace.Replica': 0, - 'IceGrid.Node.Trace.Activator': 0, - 'IceGrid.Node.Trace.Adapter': 0, - 'IceGrid.Node.Trace.Server': 0, - 'IceGrid.Node.Trace.Patch': 0, - 'IceGrid.Node.ThreadPool.SizeWarn': 0, - 'IceGrid.Node.PrintServersReady': 'node', - 'IceGrid.Node.Name': self.name, - 'IceGrid.Node.Data': '{testdir}/node-{process.name}', - 'IceGrid.Node.PropertiesOverride': self.getPropertiesOverride(current), - 'Ice.Default.Locator': current.testcase.getLocator(current), - 'Ice.NullHandleAbort': 1, + "IceGrid.InstanceName": "TestIceGrid", + "IceGrid.Node.Endpoints": "default", + "IceGrid.Node.WaitTime": 240, + "Ice.ProgramName": "icegridnode", + "IceGrid.Node.Trace.Replica": 0, + "IceGrid.Node.Trace.Activator": 0, + "IceGrid.Node.Trace.Adapter": 0, + "IceGrid.Node.Trace.Server": 0, + "IceGrid.Node.Trace.Patch": 0, + "IceGrid.Node.ThreadPool.SizeWarn": 0, + "IceGrid.Node.PrintServersReady": "node", + "IceGrid.Node.Name": self.name, + "IceGrid.Node.Data": "{testdir}/node-{process.name}", + "IceGrid.Node.PropertiesOverride": self.getPropertiesOverride(current), + "Ice.Default.Locator": current.testcase.getLocator(current), + "Ice.NullHandleAbort": 1, } return props @@ -122,17 +137,23 @@ def getEnv(self, current): def getPropertiesOverride(self, current): # Add properties for servers based on the test case mapping. props = Server().getEffectiveProps(current, {}) - return ' '.join(["{0}={1}".format(k, val(v)) for k, v in props.items()]) + return " ".join(["{0}={1}".format(k, val(v)) for k, v in props.items()]) def shutdown(self, current): current.testcase.runadmin(current, "node shutdown {0}".format(self.name)) class IceGridRegistry(ProcessFromBinDir, Server): - def __init__(self, name, portnum=20, ready="AdminSessionManager", *args, **kargs): - Server.__init__(self, "icegridregistry++11", mapping=Mapping.getByName("cpp"), desc="IceGrid registry " + name, - ready=ready, *args, **kargs) + Server.__init__( + self, + "icegridregistry++11", + mapping=Mapping.getByName("cpp"), + desc="IceGrid registry " + name, + ready=ready, + *args, + **kargs, + ) self.portnum = portnum self.readyCount = -1 self.name = name @@ -142,7 +163,9 @@ def getExe(self, current): def setup(self, current): # Create the database directory - self.dbdir = os.path.join(current.testsuite.getPath(), "registry-{0}".format(self.name)) + self.dbdir = os.path.join( + current.testsuite.getPath(), "registry-{0}".format(self.name) + ) if os.path.exists(self.dbdir): shutil.rmtree(self.dbdir) os.mkdir(self.dbdir) @@ -158,31 +181,34 @@ def teardown(self, current, success): def getProps(self, current): props = { - 'IceGrid.InstanceName': 'TestIceGrid', - 'IceGrid.Registry.PermissionsVerifier': 'TestIceGrid/NullPermissionsVerifier', - 'IceGrid.Registry.AdminPermissionsVerifier': 'TestIceGrid/NullPermissionsVerifier', - 'IceGrid.Registry.SSLPermissionsVerifier': 'TestIceGrid/NullSSLPermissionsVerifier', - 'IceGrid.Registry.AdminSSLPermissionsVerifier': 'TestIceGrid/NullSSLPermissionsVerifier', - 'IceGrid.Registry.Server.Endpoints': 'default', - 'IceGrid.Registry.Internal.Endpoints': 'default', - 'IceGrid.Registry.Client.Endpoints': self.getEndpoints(current), - 'IceGrid.Registry.Discovery.Port': current.driver.getTestPort(99), - 'IceGrid.Registry.SessionManager.Endpoints': 'default', - 'IceGrid.Registry.AdminSessionManager.Endpoints': 'default', - 'IceGrid.Registry.SessionTimeout': 60, - 'IceGrid.Registry.ReplicaName': self.name, - 'Ice.ProgramName': self.name, - 'Ice.PrintAdapterReady': 1, - 'Ice.Warn.Connections': 0, - 'Ice.ThreadPool.Client.SizeWarn': 0, - 'IceGrid.Registry.LMDB.MapSize': 1, - 'IceGrid.Registry.LMDB.Path': '{testdir}/registry-{process.name}', - 'IceGrid.Registry.Client.ThreadPool.SizeWarn': 0, - 'IceGrid.Registry.DefaultTemplates': - '"' + os.path.abspath(os.path.join(toplevel, "cpp", "config", "templates.xml")) + '"' + "IceGrid.InstanceName": "TestIceGrid", + "IceGrid.Registry.PermissionsVerifier": "TestIceGrid/NullPermissionsVerifier", + "IceGrid.Registry.AdminPermissionsVerifier": "TestIceGrid/NullPermissionsVerifier", + "IceGrid.Registry.SSLPermissionsVerifier": "TestIceGrid/NullSSLPermissionsVerifier", + "IceGrid.Registry.AdminSSLPermissionsVerifier": "TestIceGrid/NullSSLPermissionsVerifier", + "IceGrid.Registry.Server.Endpoints": "default", + "IceGrid.Registry.Internal.Endpoints": "default", + "IceGrid.Registry.Client.Endpoints": self.getEndpoints(current), + "IceGrid.Registry.Discovery.Port": current.driver.getTestPort(99), + "IceGrid.Registry.SessionManager.Endpoints": "default", + "IceGrid.Registry.AdminSessionManager.Endpoints": "default", + "IceGrid.Registry.SessionTimeout": 60, + "IceGrid.Registry.ReplicaName": self.name, + "Ice.ProgramName": self.name, + "Ice.PrintAdapterReady": 1, + "Ice.Warn.Connections": 0, + "Ice.ThreadPool.Client.SizeWarn": 0, + "IceGrid.Registry.LMDB.MapSize": 1, + "IceGrid.Registry.LMDB.Path": "{testdir}/registry-{process.name}", + "IceGrid.Registry.Client.ThreadPool.SizeWarn": 0, + "IceGrid.Registry.DefaultTemplates": '"' + + os.path.abspath(os.path.join(toplevel, "cpp", "config", "templates.xml")) + + '"', } if not isinstance(platform, Linux): - props["IceGrid.Registry.Discovery.Interface"] = "::1" if current.config.ipv6 else "127.0.0.1" + props["IceGrid.Registry.Discovery.Interface"] = ( + "::1" if current.config.ipv6 else "127.0.0.1" + ) return props def getEndpoints(self, current): @@ -192,20 +218,25 @@ def getLocator(self, current): return "TestIceGrid/Locator:{0}".format(self.getEndpoints(current)) def shutdown(self, current): - current.testcase.runadmin(current, "registry shutdown {0}".format(self.name), replica=self.name) + current.testcase.runadmin( + current, "registry shutdown {0}".format(self.name), replica=self.name + ) class IceGridRegistryMaster(IceGridRegistry): - def __init__(self, portnum=20, *args, **kargs): IceGridRegistry.__init__(self, "Master", portnum, *args, **kargs) class IceGridRegistrySlave(IceGridRegistry): - def __init__(self, replica=1, portnum=None, *args, **kargs): - IceGridRegistry.__init__(self, "Slave{0}".format(replica), (20 + replica) if portnum is None else portnum, - *args, **kargs) + IceGridRegistry.__init__( + self, + "Slave{0}".format(replica), + (20 + replica) if portnum is None else portnum, + *args, + **kargs, + ) def getProps(self, current): props = IceGridRegistry.getProps(self, current) @@ -214,17 +245,32 @@ def getProps(self, current): class IceGridTestCase(TestCase): - - def __init__(self, name="IceGrid", icegridregistry=None, icegridnode=None, application="application.xml", - variables={}, targets=[], exevars={}, *args, **kargs): + def __init__( + self, + name="IceGrid", + icegridregistry=None, + icegridnode=None, + application="application.xml", + variables={}, + targets=[], + exevars={}, + *args, + **kargs, + ): TestCase.__init__(self, name, *args, **kargs) if icegridnode: - self.icegridnode = icegridnode if isinstance(icegridnode, list) else [icegridnode] + self.icegridnode = ( + icegridnode if isinstance(icegridnode, list) else [icegridnode] + ) else: self.icegridnode = [IceGridNode()] if icegridregistry: - self.icegridregistry = icegridregistry if isinstance(icegridregistry, list) else [icegridregistry] + self.icegridregistry = ( + icegridregistry + if isinstance(icegridregistry, list) + else [icegridregistry] + ) else: self.icegridregistry = [IceGridRegistryMaster(), IceGridRegistrySlave(1)] @@ -256,29 +302,38 @@ def setupClientSide(self, current): serverProps = Server().getProps(current) variables = { "test.dir": self.getPath(current), - "java.exe": os.path.join(javaHome, "bin", "java") if javaHome else "java", + "java.exe": os.path.join(javaHome, "bin", "java") + if javaHome + else "java", "icebox.exe": IceBox().getCommandLine(current), "icegridnode.exe": IceGridNode().getCommandLine(current), "glacier2router.exe": Glacier2Router().getCommandLine(current), "icepatch2server.exe": IcePatch2Server().getCommandLine(current), "icegridregistry.exe": IceGridRegistryMaster().getCommandLine(current), - "properties-override": self.icegridnode[0].getPropertiesOverride(current), + "properties-override": self.icegridnode[0].getPropertiesOverride( + current + ), } if platform.getDotNetExe(): variables["dotnet.exe"] = platform.getDotNetExe() # Add variables that point to the directories containing the built executables - for (k, v) in self.exevars.items(): + for k, v in self.exevars.items(): variables[k] = current.getBuildDir(v) variables.update(self.variables) - varStr = " ".join(["{0}={1}".format(k, val(v)) for k, v in variables.items()]) + varStr = " ".join( + ["{0}={1}".format(k, val(v)) for k, v in variables.items()] + ) targets = " ".join(self.targets) application = self.application if isinstance(self.mapping, CSharpMapping) and current.config.dotnet: application = application.replace(".xml", ".dotnet.xml") - self.runadmin(current, "application add -n {0} {1} {2}".format(application, varStr, targets)) + self.runadmin( + current, + "application add -n {0} {1} {2}".format(application, varStr, targets), + ) def teardownClientSide(self, current, success): if (success or current.driver.isInterrupted()) and self.application: @@ -288,7 +343,13 @@ def teardownClientSide(self, current, success): p.shutdown(current) def getLocator(self, current): - endpoints = ":".join([s.getEndpoints(current) for s in self.servers if isinstance(s, IceGridRegistry)]) + endpoints = ":".join( + [ + s.getEndpoints(current) + for s in self.servers + if isinstance(s, IceGridRegistry) + ] + ) return "TestIceGrid/Locator:{0}".format(endpoints) def getMasterLocator(self, current): @@ -297,7 +358,9 @@ def getMasterLocator(self, current): return "TestIceGrid/Locator:{0}".format(s.getEndpoints(current)) def runadmin(self, current, cmd, replica="Master", exitstatus=0, quiet=False): - admin = IceGridAdmin(args=["-r", replica, "-e", cmd], replica=replica, quiet=quiet) + admin = IceGridAdmin( + args=["-r", replica, "-e", cmd], replica=replica, quiet=quiet + ) admin.run(current, exitstatus=exitstatus) return admin.getOutput(current) diff --git a/scripts/IcePatch2Util.py b/scripts/IcePatch2Util.py index 2f2beb17d0a..436502895d3 100644 --- a/scripts/IcePatch2Util.py +++ b/scripts/IcePatch2Util.py @@ -6,27 +6,42 @@ class IcePatch2Calc(ProcessFromBinDir, ProcessIsReleaseOnly, Process): - def __init__(self, *args, **kargs): - Process.__init__(self, exe="icepatch2calc++11", mapping=Mapping.getByName("cpp"), *args, **kargs) + Process.__init__( + self, + exe="icepatch2calc++11", + mapping=Mapping.getByName("cpp"), + *args, + **kargs, + ) def getExe(self, current): return self.exe + "_32" if current.config.buildPlatform == "ppc" else self.exe class IcePatch2Client(ProcessFromBinDir, ProcessIsReleaseOnly, Process): - def __init__(self, *args, **kargs): - Process.__init__(self, exe="icepatch2client++11", mapping=Mapping.getByName("cpp"), *args, **kargs) + Process.__init__( + self, + exe="icepatch2client++11", + mapping=Mapping.getByName("cpp"), + *args, + **kargs, + ) def getExe(self, current): return self.exe + "_32" if current.config.buildPlatform == "ppc" else self.exe class IcePatch2Server(ProcessFromBinDir, ProcessIsReleaseOnly, Process): - def __init__(self, *args, **kargs): - Process.__init__(self, exe="icepatch2server++11", mapping=Mapping.getByName("cpp"), *args, **kargs) + Process.__init__( + self, + exe="icepatch2server++11", + mapping=Mapping.getByName("cpp"), + *args, + **kargs, + ) def getExe(self, current): return self.exe + "_32" if current.config.buildPlatform == "ppc" else self.exe diff --git a/scripts/IceStormUtil.py b/scripts/IceStormUtil.py index 16e2558cbe2..036c7435ed6 100644 --- a/scripts/IceStormUtil.py +++ b/scripts/IceStormUtil.py @@ -9,10 +9,26 @@ class IceStorm(ProcessFromBinDir, Server): - - def __init__(self, instanceName="IceStorm", replica=0, nreplicas=0, transient=False, portnum=0, - createDb=True, cleanDb=True, *args, **kargs): - Server.__init__(self, exe="icebox++11", ready="IceStorm", mapping=Mapping.getByName("cpp"), *args, **kargs) + def __init__( + self, + instanceName="IceStorm", + replica=0, + nreplicas=0, + transient=False, + portnum=0, + createDb=True, + cleanDb=True, + *args, + **kargs, + ): + Server.__init__( + self, + exe="icebox++11", + ready="IceStorm", + mapping=Mapping.getByName("cpp"), + *args, + **kargs, + ) self.portnum = portnum self.replica = replica self.nreplicas = nreplicas @@ -20,15 +36,21 @@ def __init__(self, instanceName="IceStorm", replica=0, nreplicas=0, transient=Fa self.instanceName = instanceName self.createDb = createDb self.cleanDb = cleanDb - self.desc = self.instanceName if self.nreplicas == 0 else "{0} replica #{1}".format(self.instanceName, - self.replica) + self.desc = ( + self.instanceName + if self.nreplicas == 0 + else "{0} replica #{1}".format(self.instanceName, self.replica) + ) def getExe(self, current): return self.exe + "_32" if current.config.buildPlatform == "ppc" else self.exe def setup(self, current): # Create the database directory - self.dbdir = os.path.join(current.testsuite.getPath(), "{0}-{1}.db".format(self.instanceName, self.replica)) + self.dbdir = os.path.join( + current.testsuite.getPath(), + "{0}-{1}.db".format(self.instanceName, self.replica), + ) if self.createDb: if os.path.exists(self.dbdir): shutil.rmtree(self.dbdir) @@ -46,20 +68,24 @@ def getProps(self, current): props = Server.getProps(self, current) # Default properties - props.update({ - 'IceBox.Service.IceStorm': 'IceStormService,' + component.getSoVersion() + ':createIceStorm', - 'IceBox.PrintServicesReady': 'IceStorm', - 'IceBox.InheritProperties': 1, - 'IceStorm.InstanceName': self.instanceName, - 'Ice.Admin.InstanceName': 'IceBox', - 'Ice.Warn.Dispatch': 0, - 'Ice.Warn.Connections': 0, - 'IceStorm.LMDB.MapSize': 1, - 'IceStorm.LMDB.Path': '{testdir}/{process.instanceName}-{process.replica}.db', - }) + props.update( + { + "IceBox.Service.IceStorm": "IceStormService," + + component.getSoVersion() + + ":createIceStorm", + "IceBox.PrintServicesReady": "IceStorm", + "IceBox.InheritProperties": 1, + "IceStorm.InstanceName": self.instanceName, + "Ice.Admin.InstanceName": "IceBox", + "Ice.Warn.Dispatch": 0, + "Ice.Warn.Connections": 0, + "IceStorm.LMDB.MapSize": 1, + "IceStorm.LMDB.Path": "{testdir}/{process.instanceName}-{process.replica}.db", + } + ) if self.nreplicas > 0: - props['IceStorm.NodeId'] = self.replica + props["IceStorm.NodeId"] = self.replica if self.transient: props["IceStorm.Transient"] = 1 @@ -73,27 +99,43 @@ def getProps(self, current): # # Manager, publish, node and admin endpoints for given replica number - def manager(replica): return current.getTestEndpoint(self.portnum + replica * 4 + 0) - def publish(replica): return "{0}:{1}".format(current.getTestEndpoint(self.portnum + replica * 4 + 1), - current.getTestEndpoint(self.portnum + replica * 4 + 1, "udp")) + def manager(replica): + return current.getTestEndpoint(self.portnum + replica * 4 + 0) + + def publish(replica): + return "{0}:{1}".format( + current.getTestEndpoint(self.portnum + replica * 4 + 1), + current.getTestEndpoint(self.portnum + replica * 4 + 1, "udp"), + ) + + def node(replica): + return current.getTestEndpoint(self.portnum + replica * 4 + 2) - def node(replica): return current.getTestEndpoint(self.portnum + replica * 4 + 2) - def admin(replica): return current.getTestEndpoint(self.portnum + replica * 4 + 3) + def admin(replica): + return current.getTestEndpoint(self.portnum + replica * 4 + 3) # The endpoints for the given replica - props.update({ - "IceStorm.TopicManager.Endpoints": manager(self.replica), - "IceStorm.Publish.Endpoints": publish(self.replica), - "Ice.Admin.Endpoints": admin(self.replica), - }) + props.update( + { + "IceStorm.TopicManager.Endpoints": manager(self.replica), + "IceStorm.Publish.Endpoints": publish(self.replica), + "Ice.Admin.Endpoints": admin(self.replica), + } + ) # Compute the node and replicated endpoints to be configured for each replica if self.nreplicas > 0: - props['IceStorm.Node.Endpoints'] = node(self.replica) + props["IceStorm.Node.Endpoints"] = node(self.replica) for i in range(0, self.nreplicas): - props["IceStorm.Nodes.{0}".format(i)] = "{2}/node{0}:{1}".format(i, node(i), self.instanceName) - props['IceStorm.ReplicatedTopicManagerEndpoints'] = ":".join([manager(i) for i in range(0, self.nreplicas)]) - props['IceStorm.ReplicatedPublishEndpoints'] = ":".join([publish(i) for i in range(0, self.nreplicas)]) + props["IceStorm.Nodes.{0}".format(i)] = "{2}/node{0}:{1}".format( + i, node(i), self.instanceName + ) + props["IceStorm.ReplicatedTopicManagerEndpoints"] = ":".join( + [manager(i) for i in range(0, self.nreplicas)] + ) + props["IceStorm.ReplicatedPublishEndpoints"] = ":".join( + [publish(i) for i in range(0, self.nreplicas)] + ) return props @@ -102,31 +144,38 @@ def getInstanceName(self): def getTopicManager(self, current): # Return the endpoint for this IceStorm replica - return "{1}/TopicManager:{0}".format(current.getTestEndpoint(self.portnum + self.replica * 4), self.instanceName) + return "{1}/TopicManager:{0}".format( + current.getTestEndpoint(self.portnum + self.replica * 4), self.instanceName + ) def getReplicatedTopicManager(self, current): # Return the replicated endpoints for IceStorm if self.nreplicas == 0: return self.getTopicManager(current) - def manager(replica): return current.getTestEndpoint(self.portnum + replica * 4) - return "{1}/TopicManager:{0}".format(":".join([manager(i) for i in range(0, self.nreplicas)]), self.instanceName) + def manager(replica): + return current.getTestEndpoint(self.portnum + replica * 4) + + return "{1}/TopicManager:{0}".format( + ":".join([manager(i) for i in range(0, self.nreplicas)]), self.instanceName + ) def shutdown(self, current): # Shutdown this replica by connecting to the IceBox service manager with iceboxadmin endpoint = current.getTestEndpoint(self.portnum + self.replica * 4 + 3) - props = {"IceBoxAdmin.ServiceManager.Proxy": "IceBox/admin -f IceBox.ServiceManager:" + endpoint} - IceBoxAdmin().run(current, props=props, args=['shutdown']) + props = { + "IceBoxAdmin.ServiceManager.Proxy": "IceBox/admin -f IceBox.ServiceManager:" + + endpoint + } + IceBoxAdmin().run(current, props=props, args=["shutdown"]) class IceStormProcess: - def __init__(self, instanceName=None, instance=None): self.instanceName = instanceName self.instance = instance def getProps(self, current): - # # An IceStorm client is provided with the IceStormAdmin.TopicManager.Default property set # to the "instance" topic manager proxy if "instance" is set. Otherwise, if a single it's @@ -141,56 +190,76 @@ def getProps(self, current): while testcase and not isinstance(testcase, IceStormTestCase): testcase = testcase.parent if self.instance: - props["IceStormAdmin.TopicManager.Default"] = self.instance.getTopicManager(current) + props["IceStormAdmin.TopicManager.Default"] = self.instance.getTopicManager( + current + ) else: - instanceNames = [self.instanceName] if self.instanceName else testcase.getInstanceNames() + instanceNames = ( + [self.instanceName] + if self.instanceName + else testcase.getInstanceNames() + ) if len(instanceNames) == 1: - props["IceStormAdmin.TopicManager.Default"] = testcase.getTopicManager(current, instanceNames[0]) + props["IceStormAdmin.TopicManager.Default"] = testcase.getTopicManager( + current, instanceNames[0] + ) else: for name in instanceNames: - props["IceStormAdmin.TopicManager.{0}".format(name)] = testcase.getTopicManager(current, name) + props[ + "IceStormAdmin.TopicManager.{0}".format(name) + ] = testcase.getTopicManager(current, name) return props class IceStormAdmin(ProcessFromBinDir, ProcessIsReleaseOnly, IceStormProcess, Client): - def __init__(self, instanceName=None, instance=None, *args, **kargs): - Client.__init__(self, exe="icestormadmin++11", mapping=Mapping.getByName("cpp"), *args, **kargs) + Client.__init__( + self, + exe="icestormadmin++11", + mapping=Mapping.getByName("cpp"), + *args, + **kargs, + ) IceStormProcess.__init__(self, instanceName, instance) def getExe(self, current): - if current.config.buildPlatform == "ppc" and not component.useBinDist(self.mapping, current): + if current.config.buildPlatform == "ppc" and not component.useBinDist( + self.mapping, current + ): return self.exe + "_32" else: return self.exe - getParentProps = Client.getProps # Used by IceStormProcess to get the client properties + getParentProps = ( + Client.getProps + ) # Used by IceStormProcess to get the client properties class Subscriber(IceStormProcess, Server): - processType = "subscriber" def __init__(self, instanceName=None, instance=None, *args, **kargs): Server.__init__(self, *args, **kargs) IceStormProcess.__init__(self, instanceName, instance) - getParentProps = Server.getProps # Used by IceStormProcess to get the server properties + getParentProps = ( + Server.getProps + ) # Used by IceStormProcess to get the server properties class Publisher(IceStormProcess, Client): - processType = "publisher" def __init__(self, instanceName=None, instance=None, *args, **kargs): Client.__init__(self, *args, **kargs) IceStormProcess.__init__(self, instanceName, instance) - getParentProps = Client.getProps # Used by IceStormProcess to get the client properties + getParentProps = ( + Client.getProps + ) # Used by IceStormProcess to get the client properties class IceStormTestCase(TestCase): - def __init__(self, name, icestorm, *args, **kargs): TestCase.__init__(self, name, *args, **kargs) self.icestorm = icestorm if isinstance(icestorm, list) else [icestorm] @@ -224,7 +293,9 @@ def shutdown(self, current): for icestorm in self.icestorm: icestorm.shutdown(current) - def runadmin(self, current, cmd, instanceName=None, instance=None, exitstatus=0, quiet=False): + def runadmin( + self, current, cmd, instanceName=None, instance=None, exitstatus=0, quiet=False + ): admin = IceStormAdmin(instanceName, instance, args=["-e", cmd], quiet=quiet) admin.run(current, exitstatus=exitstatus) return admin.getOutput(current) diff --git a/scripts/LocalDriver.py b/scripts/LocalDriver.py index 5053a2ce7c0..32de5d1bdc4 100644 --- a/scripts/LocalDriver.py +++ b/scripts/LocalDriver.py @@ -15,7 +15,6 @@ class Executor: - def __init__(self, threadlocal, workers, continueOnFailure): self.threadlocal = threadlocal self.workers = workers - 1 @@ -70,7 +69,9 @@ def runTestSuites(self, driver, total, results, mainThread=False): while True: item = self.get(total, mainThread) if not item: - results.put(None) # Notify the main thread that there are not more tests to run + results.put( + None + ) # Notify the main thread that there are not more tests to run break (testsuite, index) = item @@ -181,13 +182,13 @@ def worker(num): return resultList + # # Runner to run the test cases locally. # class TestCaseRunner: - def getTestSuites(self, mapping, testSuiteIds): return mapping.getTestSuites(testSuiteIds) @@ -203,18 +204,21 @@ def stopServerSide(self, testcase, current, success): def runClientSide(self, testcase, current, host): testcase._runClientSide(current, host) + # # Runner to run the test cases remotely with the controller (requires IcePy) # class RemoteTestCaseRunner(TestCaseRunner): - def __init__(self, communicator, clientPrx, serverPrx): import Test + if clientPrx: self.clientController = communicator.stringToProxy(clientPrx) - self.clientController = Test.Common.ControllerPrx.checkedCast(self.clientController) + self.clientController = Test.Common.ControllerPrx.checkedCast( + self.clientController + ) self.clientOptions = self.clientController.getOptionOverrides() else: self.clientController = None @@ -222,7 +226,9 @@ def __init__(self, communicator, clientPrx, serverPrx): if serverPrx: self.serverController = communicator.stringToProxy(serverPrx) - self.serverController = Test.Common.ControllerPrx.checkedCast(self.serverController) + self.serverController = Test.Common.ControllerPrx.checkedCast( + self.serverController + ) self.serverOptions = self.serverController.getOptionOverrides() else: self.serverController = None @@ -253,8 +259,9 @@ def filterOptions(self, options): if options is None: return None import Ice + options = options.copy() - for (key, values) in options.items(): + for key, values in options.items(): for opts in [self.serverOptions, self.clientOptions]: if hasattr(opts, key) and getattr(opts, key) is not Ice.Unset: options[key] = [v for v in values if v in getattr(opts, key)] @@ -265,10 +272,13 @@ def startServerSide(self, testcase, current): return TestCaseRunner.startServerSide(self, testcase, current) import Test - current.serverTestCase = self.serverController.runTestCase(str(testcase.getMapping()), - testcase.getTestSuite().getId(), - testcase.getName(), - str(current.driver.cross)) + + current.serverTestCase = self.serverController.runTestCase( + str(testcase.getMapping()), + testcase.getTestSuite().getId(), + testcase.getName(), + str(current.driver.cross), + ) try: try: return current.serverTestCase.startServerSide(self.getConfig(current)) @@ -286,6 +296,7 @@ def stopServerSide(self, testcase, current, success): return import Test + try: current.result.write(current.serverTestCase.stopServerSide(success)) current.host = None @@ -297,16 +308,21 @@ def stopServerSide(self, testcase, current, success): def runClientSide(self, testcase, current, host): import Test + if not self.clientController: TestCaseRunner.runClientSide(self, testcase, current, host) return - clientTestCase = self.clientController.runTestCase(str(testcase.getMapping()), - testcase.getTestSuite().getId(), - testcase.getName(), - str(current.driver.cross)) + clientTestCase = self.clientController.runTestCase( + str(testcase.getMapping()), + testcase.getTestSuite().getId(), + testcase.getName(), + str(current.driver.cross), + ) try: - current.result.write(clientTestCase.runClientSide(host, self.getConfig(current))) + current.result.write( + clientTestCase.runClientSide(host, self.getConfig(current)) + ) except Test.Common.TestCaseFailedException as ex: current.result.writeln(ex.output) raise RuntimeError("test failed") @@ -315,17 +331,19 @@ def runClientSide(self, testcase, current, host): def getConfig(self, current): import Test - return Test.Common.Config(current.config.protocol, - current.config.mx, - current.config.serialize, - current.config.compress, - current.config.ipv6, - current.config.cprops, - current.config.sprops) + return Test.Common.Config( + current.config.protocol, + current.config.mx, + current.config.serialize, + current.config.compress, + current.config.ipv6, + current.config.cprops, + current.config.sprops, + ) -class XmlExporter: +class XmlExporter: def __init__(self, results, duration, failures): self.results = results self.duration = duration @@ -334,18 +352,18 @@ def __init__(self, results, duration, failures): def save(self, filename, hostname): with open(filename, "w", encoding="utf-8") as out: out.write('\n') - out.write('\n'.format(len(self.results), - self.duration, - len(self.failures))) + out.write( + '\n'.format( + len(self.results), self.duration, len(self.failures) + ) + ) for r in self.results: r.writeAsXml(out, hostname) - out.write('\n') + out.write("\n") class LocalDriver(Driver): - class Current(Driver.Current): - def __init__(self, driver, testsuite, result, index, total): Driver.Current.__init__(self, driver, testsuite, result) self.index = index @@ -353,8 +371,23 @@ def __init__(self, driver, testsuite, result, index, total): @classmethod def getSupportedArgs(self): - return ("", ["cross=", "workers=", "continue", "loop", "start=", "all", "all-cross", "host=", - "client=", "server=", "show-durations", "export-xml="]) + return ( + "", + [ + "cross=", + "workers=", + "continue", + "loop", + "start=", + "all", + "all-cross", + "host=", + "client=", + "server=", + "show-durations", + "export-xml=", + ], + ) @classmethod def usage(self): @@ -366,9 +399,15 @@ def usage(self): print("--loop Run the tests in a loop.") print("--continue Don't stop on failures.") print("--all Run all sensible permutations of the tests.") - print("--all-cross Run all sensible permutations of cross language tests.") - print("--client= The endpoint of the controller to run the client side.") - print("--server= The endpoint of the controller to run the server side.") + print( + "--all-cross Run all sensible permutations of cross language tests." + ) + print( + "--client= The endpoint of the controller to run the client side." + ) + print( + "--server= The endpoint of the controller to run the server side." + ) print("--show-durations Print out the duration of each tests.") print("--export-xml= Export JUnit XML test report.") @@ -388,18 +427,26 @@ def __init__(self, options, *args, **kargs): self.clientCtlPrx = "" self.serverCtlPrx = "" - parseOptions(self, options, {"continue": "continueOnFailure", - "l": "loop", - "all-cross": "allCross", - "client": "clientCtlPrx", - "server": "serverCtlPrx", - "show-durations": "showDurations", - "export-xml": "exportToXml"}) + parseOptions( + self, + options, + { + "continue": "continueOnFailure", + "l": "loop", + "all-cross": "allCross", + "client": "clientCtlPrx", + "server": "serverCtlPrx", + "show-durations": "showDurations", + "export-xml": "exportToXml", + }, + ) if self.cross: self.cross = Mapping.getByName(self.cross) if not self.cross: - raise RuntimeError("unknown mapping `{0}' for --cross option".format(self.cross)) + raise RuntimeError( + "unknown mapping `{0}' for --cross option".format(self.cross) + ) self.results = [] self.threadlocal = threading.local() @@ -407,10 +454,11 @@ def __init__(self, options, *args, **kargs): self.executor = Executor(self.threadlocal, self.workers, self.continueOnFailure) def run(self, mappings, testSuiteIds): - if self.clientCtlPrx or self.serverCtlPrx: self.initCommunicator() - self.runner = RemoteTestCaseRunner(self.communicator, self.clientCtlPrx, self.serverCtlPrx) + self.runner = RemoteTestCaseRunner( + self.communicator, self.clientCtlPrx, self.serverCtlPrx + ) else: self.runner = TestCaseRunner() @@ -426,21 +474,38 @@ def run(self, mappings, testSuiteIds): def testsuiteKey(testsuite): for k in runOrder: - if testsuite.getId().startswith(k + '/'): - return testsuite.getId().replace(k, str(runOrder.index(k))) + if testsuite.getId().startswith(k + "/"): + return testsuite.getId().replace( + k, str(runOrder.index(k)) + ) return testsuite.getId() + testsuites = sorted(testsuites, key=testsuiteKey) for testsuite in testsuites: - if mapping.filterTestSuite(testsuite.getId(), self.configs[mapping], self.filters, self.rfilters): + if mapping.filterTestSuite( + testsuite.getId(), + self.configs[mapping], + self.filters, + self.rfilters, + ): continue if testsuite.getId() == "Ice/echo": continue - elif (self.cross or self.allCross) and not self.component.isCross(testsuite.getId()): + elif ( + self.cross or self.allCross + ) and not self.component.isCross(testsuite.getId()): continue - elif isinstance(self.runner, RemoteTestCaseRunner) and not testsuite.isMultiHost(): + elif ( + isinstance(self.runner, RemoteTestCaseRunner) + and not testsuite.isMultiHost() + ): continue - self.executor.submit(testsuite, Mapping.getAll(self) if self.allCross else [self.cross], self) + self.executor.submit( + testsuite, + Mapping.getAll(self) if self.allCross else [self.cross], + self, + ) # # Run all the tests and wait for the executor to complete. @@ -453,27 +518,41 @@ def testsuiteKey(testsuite): duration = time.time() - now if self.exportToXml: - XmlExporter(results, duration, failures).save(self.exportToXml, os.getenv("NODE_NAME", "")) + XmlExporter(results, duration, failures).save( + self.exportToXml, os.getenv("NODE_NAME", "") + ) m, s = divmod(duration, 60) print("") if m > 0: - print("Ran {0} tests in {1} minutes {2:02.2f} seconds".format(len(results), m, s)) + print( + "Ran {0} tests in {1} minutes {2:02.2f} seconds".format( + len(results), m, s + ) + ) else: print("Ran {0} tests in {1:02.2f} seconds".format(len(results), s)) if self.showDurations: for r in sorted(results, key=lambda r: r.getDuration()): - print("- {0} took {1:02.2f} seconds".format(r.testsuite, r.getDuration())) + print( + "- {0} took {1:02.2f} seconds".format( + r.testsuite, r.getDuration() + ) + ) self.loopCount += 1 if len(failures) > 0: - print("{0} succeeded and {1} failed:".format(len(results) - len(failures), len(failures))) + print( + "{0} succeeded and {1} failed:".format( + len(results) - len(failures), len(failures) + ) + ) for r in failures: print("- {0}".format(r.testsuite)) - for (c, ex) in r.getFailed().items(): - lines = r.getOutput(c).strip().split('\n') + for c, ex in r.getFailed().items(): + lines = r.getOutput(c).strip().split("\n") for i in range(0, min(4, len(lines))): print(" " + lines[i]) if len(lines) > 4: @@ -490,10 +569,18 @@ def testsuiteKey(testsuite): def runTestSuite(self, current): if self.loop: - current.result.write("*** [{0}/{1} loop={2}] ".format(current.index, current.total, self.loopCount)) + current.result.write( + "*** [{0}/{1} loop={2}] ".format( + current.index, current.total, self.loopCount + ) + ) else: current.result.write("*** [{0}/{1}] ".format(current.index, current.total)) - current.result.writeln("Running {0}/{1} tests ***".format(current.testsuite.getMapping(), current.testsuite)) + current.result.writeln( + "Running {0}/{1} tests ***".format( + current.testsuite.getMapping(), current.testsuite + ) + ) success = False try: @@ -508,7 +595,11 @@ def runTestSuite(self, current): for testcase in current.testsuite.getTestCases(): config = current.config try: - for conf in current.config.getAll(current, testcase) if self.all else [current.config]: + for conf in ( + current.config.getAll(current, testcase) + if self.all + else [current.config] + ): current.config = conf testcase.run(current) except: @@ -528,7 +619,6 @@ def runTestSuite(self, current): raise def runClientServerTestCase(self, current): - if current.testcase.getParent(): success = False host = current.testcase._startServerSide(current) @@ -540,8 +630,7 @@ def runClientServerTestCase(self, current): return client = current.testcase.getClientTestCase() - for cross in (Mapping.getAll(self) if self.allCross else [self.cross]): - + for cross in Mapping.getAll(self) if self.allCross else [self.cross]: # Only run cross tests with allCross if self.allCross and cross == current.testcase.getMapping(): continue @@ -555,7 +644,11 @@ def runClientServerTestCase(self, current): if not server: continue - current.writeln("[ running {0} test - {1} ]".format(current.testcase, time.strftime("%x %X"))) + current.writeln( + "[ running {0} test - {1} ]".format( + current.testcase, time.strftime("%x %X") + ) + ) if not self.all: current.config = current.config.cloneRunnable(current) confStr = str(current.config) @@ -565,9 +658,17 @@ def runClientServerTestCase(self, current): else: current.desc = "" if cross: - current.writeln("- Mappings: {0},{1}".format(client.getMapping(), server.getMapping())) - current.desc += (" " if current.desc else "") + "cross={0}".format(server.getMapping()) - if not current.config.canRun(current.testsuite.getId(), current) or not current.testcase.canRun(current): + current.writeln( + "- Mappings: {0},{1}".format( + client.getMapping(), server.getMapping() + ) + ) + current.desc += (" " if current.desc else "") + "cross={0}".format( + server.getMapping() + ) + if not current.config.canRun( + current.testsuite.getId(), current + ) or not current.testcase.canRun(current): current.result.skipped(current, "not supported with this configuration") return @@ -590,7 +691,9 @@ def stopServerSide(): self.runner.stopServerSide(server, current, success) except Exception as ex: failure.append(ex) - except KeyboardInterrupt: # Potentially raised by Except.py if Ctrl-C + except ( + KeyboardInterrupt + ): # Potentially raised by Except.py if Ctrl-C pass sem.release() @@ -612,18 +715,24 @@ def stopServerSide(): def runTestCase(self, current): if self.cross or self.allCross: - #current.result.skipped(current, "only client/server tests are ran with cross tests") + # current.result.skipped(current, "only client/server tests are ran with cross tests") return if not current.testcase.getParent(): - current.writeln("[ running {0} test - {1} ]".format(current.testcase, time.strftime("%x %X"))) + current.writeln( + "[ running {0} test - {1} ]".format( + current.testcase, time.strftime("%x %X") + ) + ) if not self.all: current.config = current.config.cloneRunnable(current) confStr = str(current.config) if confStr: current.writeln("- Config: {0}".format(confStr)) current.desc = confStr - if not current.config.canRun(current.testsuite.getId(), current) or not current.testcase.canRun(current): + if not current.config.canRun( + current.testsuite.getId(), current + ) or not current.testcase.canRun(current): current.result.skipped(current, "not supported with this configuration") return @@ -647,8 +756,12 @@ def setInterrupt(self, value): def getTestPort(self, portnum): # Return a port number in the range 14100-14199 for the first thread, 14200-14299 for the # second thread, etc. - assert (portnum < 100) - baseport = 14000 + self.threadlocal.num * 100 if hasattr(self.threadlocal, "num") else 12010 + assert portnum < 100 + baseport = ( + 14000 + self.threadlocal.num * 100 + if hasattr(self.threadlocal, "num") + else 12010 + ) return baseport + portnum def getProps(self, process, current): @@ -662,7 +775,13 @@ def getProps(self, process, current): return props def getMappings(self): - return Mapping.getAll(self) if self.allCross else [self.cross] if self.cross else [] + return ( + Mapping.getAll(self) + if self.allCross + else [self.cross] + if self.cross + else [] + ) def filterOptions(self, options): return self.runner.filterOptions(options) diff --git a/scripts/NetworkProxy.py b/scripts/NetworkProxy.py index 5c8188d0005..24c1b18a821 100644 --- a/scripts/NetworkProxy.py +++ b/scripts/NetworkProxy.py @@ -53,12 +53,14 @@ def run(self): return try: - while (not self.closed): - readables, writeables, exceptions = select.select([self.socket, self.remoteSocket], [], []) + while not self.closed: + readables, writeables, exceptions = select.select( + [self.socket, self.remoteSocket], [], [] + ) for r in readables: w = self.remoteSocket if r == self.socket else self.socket data = r.recv(4096) - if (len(data) == 0): + if len(data) == 0: self.closed = True break w.send(data) @@ -101,7 +103,9 @@ def run(self): self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, "SO_REUSEPORT"): try: - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + self.socket.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT, 1 + ) except: # Ignore, this can throw on some platforms if not supported (e.g: ARMHF/Qemu) pass @@ -152,7 +156,6 @@ def terminate(self): class SocksConnection(BaseConnection): - def request(self, s): def decode(c): return ord(c) if sys.version_info[0] == 2 else c @@ -186,19 +189,17 @@ def encode(c): class SocksProxy(BaseProxy): - def createConnection(self, socket, peer): return SocksConnection(socket, peer) class HttpConnection(BaseConnection): - def request(self, s): def decode(c): return c[0] if sys.version_info[0] == 2 else chr(c[0]) data = "" - while (len(data) < 4 or data[len(data) - 4:] != "\r\n\r\n"): + while len(data) < 4 or data[len(data) - 4 :] != "\r\n\r\n": data += decode(s.recv(1)) if data.find("CONNECT ") != 0: @@ -208,16 +209,16 @@ def decode(c): if sep < len("CONNECT ") + 1: raise InvalidRequest - host = data[len("CONNECT "):sep] + host = data[len("CONNECT ") : sep] space = data.find(" ", sep) if space < sep + 1: raise InvalidRequest - port = int(data[sep + 1:space]) + port = int(data[sep + 1 : space]) return (host, port) def response(self, success): - if (success): + if success: s = "HTTP/1.1 200 OK\r\nServer: CERN/3.0 libwww/2.17\r\n\r\n" else: s = "HTTP/1.1 404\r\n\r\n" @@ -225,6 +226,5 @@ def response(self, success): class HttpProxy(BaseProxy): - def createConnection(self, socket, peer): return HttpConnection(socket, peer) diff --git a/scripts/Util.py b/scripts/Util.py index f0e46d6d33b..60ba5ef694b 100644 --- a/scripts/Util.py +++ b/scripts/Util.py @@ -28,8 +28,14 @@ def run(cmd, cwd=None, err=False, stdout=False, stdin=None, stdinRepeat=True): - p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=None if stdout else subprocess.PIPE, - stderr=subprocess.STDOUT, cwd=cwd) + p = subprocess.Popen( + cmd, + shell=True, + stdin=subprocess.PIPE, + stdout=None if stdout else subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=cwd, + ) try: if stdin: try: @@ -42,7 +48,7 @@ def run(cmd, cwd=None, err=False, stdout=False, stdin=None, stdinRepeat=True): except: pass - out = (p.stderr if stdout else p.stdout).read().decode('UTF-8').strip() + out = (p.stderr if stdout else p.stdout).read().decode("UTF-8").strip() if (not err and p.wait() != 0) or (err and p.wait() == 0): raise RuntimeError(cmd + " failed:\n" + out) finally: @@ -65,18 +71,18 @@ def val(v, quoteValue=True): elif type(v) == str: if not quoteValue or v.find(" ") < 0: return v - v = v.replace("\\", "\\\\").replace("\"", "\\\"") - return "\"{0}\"".format(v) + v = v.replace("\\", "\\\\").replace('"', '\\"') + return '"{0}"'.format(v) else: return str(v) -illegalXMLChars = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]') +illegalXMLChars = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]") def escapeXml(s, attribute=False): # Remove backspace characters from the output (they aren't accepted by Jenkins XML parser) - s = "".join(ch for ch in s if ch != u"\u0008") + s = "".join(ch for ch in s if ch != "\u0008") s = illegalXMLChars.sub("?", s) # Strip invalid XML characters return xml.sax.saxutils.quoteattr(s) if attribute else xml.sax.saxutils.escape(s) @@ -88,7 +94,6 @@ class to provide component specific information. class Component(object): - def __init__(self): self.nugetVersion = {} @@ -112,7 +117,9 @@ def getSourceDir(self): def getTestDir(self, mapping): if isinstance(mapping, JavaMapping): - return os.path.join(mapping.getPath(), "test", "src", "main", "java", "test") + return os.path.join( + mapping.getPath(), "test", "src", "main", "java", "test" + ) elif isinstance(mapping, TypeScriptMapping): return os.path.join(mapping.getPath(), "test", "typescript") return os.path.join(mapping.getPath(), "test") @@ -124,8 +131,12 @@ def getPhpExtension(self, mapping, current): raise RuntimeError("must be overridden if component provides php mapping") def getNugetPackage(self, mapping): - return "zeroc.{0}.{1}".format(self.__class__.__name__.lower(), - "net" if isinstance(mapping, CSharpMapping) else platform.getPlatformToolset()) + return "zeroc.{0}.{1}".format( + self.__class__.__name__.lower(), + "net" + if isinstance(mapping, CSharpMapping) + else platform.getPlatformToolset(), + ) def getNugetPackageVersion(self, mapping): if not mapping in self.nugetVersion: @@ -133,18 +144,24 @@ def getNugetPackageVersion(self, mapping): if file.endswith(".nuspec"): expr = "(.*)" elif file.endswith("packages.config"): - expr = "id=\"{0}\" version=\"(.*)\" target".format(self.getNugetPackage(mapping)) + expr = 'id="{0}" version="(.*)" target'.format( + self.getNugetPackage(mapping) + ) if expr: with open(file, "r") as config: m = re.search(expr, config.read()) if m: self.nugetVersion[mapping] = m.group(1) if not mapping in self.nugetVersion: - raise RuntimeError("couldn't figure out the nuget version from `{0}'".format(file)) + raise RuntimeError( + "couldn't figure out the nuget version from `{0}'".format(file) + ) return self.nugetVersion[mapping] def getNugetPackageVersionFile(self, mapping): - raise RuntimeError("must be overridden if component provides C++ or C# nuget packages") + raise RuntimeError( + "must be overridden if component provides C++ or C# nuget packages" + ) def getFilters(self, mapping, config): return ([], []) @@ -185,7 +202,7 @@ def isCross(self, testId): def getSliceDir(self, mapping, current): installDir = self.getInstallDir(mapping, current) if installDir.endswith(mapping.name): - installDir = installDir[0:len(installDir) - len(mapping.name) - 1] + installDir = installDir[0 : len(installDir) - len(mapping.name) - 1] if platform.getInstallDir() and installDir == platform.getInstallDir(): return os.path.join(installDir, "share", "ice", "slice") else: @@ -198,11 +215,14 @@ def getLibDir(self, process, mapping, current): return platform._getLibDir(self, process, mapping, current) def getPhpIncludePath(self, mapping, current): - return "{0}/{1}".format(self.getInstallDir(mapping, current), "php" if self.useBinDist(mapping, current) else "lib") + return "{0}/{1}".format( + self.getInstallDir(mapping, current), + "php" if self.useBinDist(mapping, current) else "lib", + ) def _useBinDist(self, mapping, current, envName): env = os.environ.get(envName, "").split() - return 'all' in env or mapping.name in env + return "all" in env or mapping.name in env def _getInstallDir(self, mapping, current, envHomeName): if self.useBinDist(mapping, current): @@ -212,7 +232,11 @@ def _getInstallDir(self, mapping, current, envHomeName): if envHomeName and not os.path.exists(packageDir): home = os.environ.get(envHomeName, "") if not home or not os.path.exists(home): - raise RuntimeError("Cannot detect a valid distribution in `" + envHomeName + "'") + raise RuntimeError( + "Cannot detect a valid distribution in `" + + envHomeName + + "'" + ) return home else: return packageDir @@ -225,11 +249,12 @@ def _getInstallDir(self, mapping, current, envHomeName): class Platform(object): - def __init__(self): try: - self.nugetPackageCache = re.search("global-packages: (.*)", - run("dotnet nuget locals --list global-packages")).groups(1)[0] + self.nugetPackageCache = re.search( + "global-packages: (.*)", + run("dotnet nuget locals --list global-packages"), + ).groups(1)[0] except: self.nugetPackageCache = None @@ -237,10 +262,13 @@ def __init__(self): self._hasSwift = None def init(self, component): - self.parseBuildVariables(component, { - "supported-platforms": ("supportedPlatforms", lambda s: s.split(" ")), - "supported-configs": ("supportedConfigs", lambda s: s.split(" ")) - }) + self.parseBuildVariables( + component, + { + "supported-platforms": ("supportedPlatforms", lambda s: s.split(" ")), + "supported-configs": ("supportedConfigs", lambda s: s.split(" ")), + }, + ) def hasDotNet(self): return self.nugetPackageCache != None @@ -257,9 +285,13 @@ def hasNodeJS(self): def hasSwift(self, version): if self._hasSwift is None: try: - m = re.search(r"Apple Swift version ([0-9]+\.[0-9]+)", run("swift --version")) + m = re.search( + r"Apple Swift version ([0-9]+\.[0-9]+)", run("swift --version") + ) if m and m.group(1): - self._hasSwift = tuple([int(n) for n in m.group(1).split(".")]) >= version + self._hasSwift = ( + tuple([int(n) for n in m.group(1).split(".")]) >= version + ) else: self.hasSwift = False except: @@ -268,14 +300,16 @@ def hasSwift(self, version): def parseBuildVariables(self, component, variables): # Run make to get the values of the given variables - if os.path.exists(os.path.join(component.getSourceDir(), "Makefile")): # Top level makefile + if os.path.exists( + os.path.join(component.getSourceDir(), "Makefile") + ): # Top level makefile cwd = component.getSourceDir() elif Mapping.getByName("cpp"): cwd = Mapping.getByName("cpp").getPath() output = run('make print V="{0}"'.format(" ".join(variables.keys())), cwd=cwd) for l in output.split("\n"): - match = re.match(r'^.*:.*: (.*) = (.*)', l) + match = re.match(r"^.*:.*: (.*) = (.*)", l) if match and match.group(1): if match.group(1) in variables: (varname, valuefn) = variables[match.group(1).strip()] @@ -292,9 +326,13 @@ def _getBinDir(self, component, process, mapping, current): installDir = component.getInstallDir(mapping, current) if isinstance(mapping, CSharpMapping): if component.useBinDist(mapping, current): - return os.path.join(installDir, "tools", mapping.getTargetFramework(current)) + return os.path.join( + installDir, "tools", mapping.getTargetFramework(current) + ) else: - return os.path.join(installDir, "bin", mapping.getTargetFramework(current)) + return os.path.join( + installDir, "bin", mapping.getTargetFramework(current) + ) return os.path.join(installDir, "bin") def _getLibDir(self, component, process, mapping, current): @@ -318,9 +356,11 @@ def getInstallDir(self): def getNugetPackageDir(self, component, mapping, current): if not self.nugetPackageCache: return None - return os.path.join(self.nugetPackageCache, - component.getNugetPackage(mapping), - component.getNugetPackageVersion(mapping)) + return os.path.join( + self.nugetPackageCache, + component.getNugetPackage(mapping), + component.getNugetPackageVersion(mapping), + ) def hasOpenSSL(self): # This is used by the IceSSL test suite to figure out how to setup certificates @@ -331,7 +371,6 @@ def getDotNetExe(self): class Darwin(Platform): - def getDefaultBuildPlatform(self): return "macosx" @@ -343,7 +382,6 @@ def getInstallDir(self): class AIX(Platform): - def hasOpenSSL(self): return True @@ -352,7 +390,9 @@ def _getLibDir(self, component, process, mapping, current): if component.useBinDist(mapping, current): return os.path.join(installDir, "lib") else: - return os.path.join(installDir, "lib32" if current.config.buildPlatform == "ppc" else "lib") + return os.path.join( + installDir, "lib32" if current.config.buildPlatform == "ppc" else "lib" + ) def getDefaultBuildPlatform(self): return "ppc64" @@ -362,7 +402,6 @@ def getInstallDir(self): class Linux(Platform): - def __init__(self): Platform.__init__(self) self.multiArch = {} @@ -372,14 +411,24 @@ def __init__(self): def init(self, component): Platform.init(self, component) - self.parseBuildVariables(component, { - "linux_id": ("linuxId", None), - "build-platform": ("buildPlatform", None), - "foreign-platforms": ("foreignPlatforms", lambda s: s.split(" ") if s else []), - }) + self.parseBuildVariables( + component, + { + "linux_id": ("linuxId", None), + "build-platform": ("buildPlatform", None), + "foreign-platforms": ( + "foreignPlatforms", + lambda s: s.split(" ") if s else [], + ), + }, + ) if self.linuxId in ["ubuntu", "debian"]: for p in [self.buildPlatform] + self.foreignPlatforms: - self.multiArch[p] = run("dpkg-architecture -f -a{0} -qDEB_HOST_MULTIARCH 2> /dev/null".format(p)) + self.multiArch[p] = run( + "dpkg-architecture -f -a{0} -qDEB_HOST_MULTIARCH 2> /dev/null".format( + p + ) + ) def hasOpenSSL(self): return True @@ -390,7 +439,11 @@ def _getBinDir(self, component, process, mapping, current): return Platform._getBinDir(self, component, process, mapping, current) if self.linuxId in ["ubuntu", "debian"]: - binDir = os.path.join(installDir, "bin", self.multiArch[current.driver.configs[mapping].buildPlatform]) + binDir = os.path.join( + installDir, + "bin", + self.multiArch[current.driver.configs[mapping].buildPlatform], + ) if os.path.exists(binDir): return binDir return os.path.join(installDir, "bin") @@ -403,11 +456,16 @@ def _getLibDir(self, component, process, mapping, current): buildPlatform = current.driver.configs[mapping].buildPlatform # PHP module is always installed in the lib directory for the default build platform - if isinstance(mapping, PhpMapping) and buildPlatform == self.getDefaultBuildPlatform(): + if ( + isinstance(mapping, PhpMapping) + and buildPlatform == self.getDefaultBuildPlatform() + ): return os.path.join(installDir, "lib") if self.linuxId in ["centos", "rhel", "fedora"]: - return os.path.join(installDir, "lib64" if buildPlatform == "x64" else "lib") + return os.path.join( + installDir, "lib64" if buildPlatform == "x64" else "lib" + ) elif self.linuxId in ["ubuntu", "debian"]: return os.path.join(installDir, "lib", self.multiArch[buildPlatform]) return os.path.join(installDir, "lib") @@ -425,7 +483,6 @@ def getLinuxId(self): class Windows(Platform): - def __init__(self): Platform.__init__(self) self.compiler = None @@ -473,25 +530,43 @@ def getPlatformToolset(self): def _getBinDir(self, component, process, mapping, current): installDir = component.getInstallDir(mapping, current) platform = current.driver.configs[mapping].buildPlatform - config = "Debug" if current.driver.configs[mapping].buildConfig.find("Debug") >= 0 else "Release" + config = ( + "Debug" + if current.driver.configs[mapping].buildConfig.find("Debug") >= 0 + else "Release" + ) if component.useBinDist(mapping, current): if installDir != self.getNugetPackageDir(component, mapping, current): return os.path.join(installDir, "bin") elif isinstance(process, SliceTranslator): return os.path.join(installDir, "tools") elif isinstance(mapping, CSharpMapping): - return os.path.join(installDir, "tools", mapping.getTargetFramework(current)) + return os.path.join( + installDir, "tools", mapping.getTargetFramework(current) + ) elif process.isReleaseOnly(): # Some services are only available in release mode in the Nuget package - return os.path.join(installDir, "build", "native", "bin", platform, "Release") + return os.path.join( + installDir, "build", "native", "bin", platform, "Release" + ) else: - return os.path.join(installDir, "build", "native", "bin", platform, config) + return os.path.join( + installDir, "build", "native", "bin", platform, config + ) else: if isinstance(mapping, CSharpMapping): - return os.path.join(installDir, "bin", mapping.getTargetFramework(current)) + return os.path.join( + installDir, "bin", mapping.getTargetFramework(current) + ) elif isinstance(mapping, PhpMapping): - return os.path.join(self.getNugetPackageDir(component, mapping, current), - "build", "native", "bin", platform, config) + return os.path.join( + self.getNugetPackageDir(component, mapping, current), + "build", + "native", + "bin", + platform, + config, + ) else: return os.path.join(installDir, "bin", platform, config) @@ -501,20 +576,34 @@ def _getLibDir(self, component, process, mapping, current): return os.path.join(installDir, "lib/netstandard2.0") else: platform = current.driver.configs[mapping].buildPlatform - config = "Debug" if current.driver.configs[mapping].buildConfig.find("Debug") >= 0 else "Release" + config = ( + "Debug" + if current.driver.configs[mapping].buildConfig.find("Debug") >= 0 + else "Release" + ) if isinstance(mapping, PhpMapping): - return os.path.join(installDir, "lib", "php-{0}".format(current.config.phpVersion), platform, config) + return os.path.join( + installDir, + "lib", + "php-{0}".format(current.config.phpVersion), + platform, + config, + ) if isinstance(mapping, MatlabMapping): return os.path.join(installDir, "lib", platform, config) elif component.useBinDist(mapping, current): - return os.path.join(installDir, "build", "native", "bin", platform, config) + return os.path.join( + installDir, "build", "native", "bin", platform, config + ) else: return os.path.join(installDir, "bin", platform, config) def getBuildSubDir(self, mapping, name, current): buildPlatform = current.driver.configs[mapping].buildPlatform buildConfig = current.driver.configs[mapping].buildConfig - if os.path.exists(os.path.join(current.testcase.getPath(current), "msbuild", name)): + if os.path.exists( + os.path.join(current.testcase.getPath(current), "msbuild", name) + ): return os.path.join("msbuild", name, buildPlatform, buildConfig) else: return os.path.join("msbuild", buildPlatform, buildConfig) @@ -529,7 +618,10 @@ def getNugetPackageDir(self, component, mapping, current): if isinstance(mapping, CSharpMapping) and current.config.dotnet: return Platform.getNugetPackageDir(self, component, mapping, current) else: - package = "{0}.{1}".format(component.getNugetPackage(mapping), component.getNugetPackageVersion(mapping)) + package = "{0}.{1}".format( + component.getNugetPackage(mapping), + component.getNugetPackageVersion(mapping), + ) # The package directory is either under the msbuild directory or in the mapping directory depending # on where the solution is located. if os.path.exists(os.path.join(mapping.path, "msbuild", "packages")): @@ -551,7 +643,7 @@ def parseOptions(obj, options, mapped={}): if not hasattr(obj, "parsedOptions"): obj.parsedOptions = [] remaining = [] - for (o, a) in options: + for o, a in options: if o.startswith("--"): o = o[2:] if o.startswith("-"): @@ -587,16 +679,28 @@ def parseOptions(obj, options, mapped={}): class Mapping(object): - mappings = OrderedDict() disabled = OrderedDict() class Config(object): - @classmethod def getSupportedArgs(self): - return ("", ["config=", "platform=", "protocol=", "target=", "compress", "ipv6", "no-ipv6", "serialize", - "mx", "cprops=", "sprops="]) + return ( + "", + [ + "config=", + "platform=", + "protocol=", + "target=", + "compress", + "ipv6", + "no-ipv6", + "serialize", + "mx", + "cprops=", + "sprops=", + ], + ) @classmethod def usage(self): @@ -611,8 +715,12 @@ def commonUsage(self): print("--ipv6 Use IPv6 addresses.") print("--serialize Run with connection serialization.") print("--mx Run with metrics enabled.") - print("--cprops= Specifies a list of additional client properties.") - print("--sprops= Specifies a list of additional server properties.") + print( + "--cprops= Specifies a list of additional client properties." + ) + print( + "--sprops= Specifies a list of additional server properties." + ) print("--config= Build configuration for native executables.") print("--platform= Build platform for native executables.") @@ -653,7 +761,9 @@ def __init__(self, options=[]): self.phpVersion = "7.1" self.python = sys.executable - parseOptions(self, options, {"config": "buildConfig", "platform": "buildPlatform"}) + parseOptions( + self, options, {"config": "buildConfig", "platform": "buildPlatform"} + ) def __str__(self): s = [] @@ -664,12 +774,10 @@ def __str__(self): return ",".join(s) def getAll(self, current, testcase, rand=False): - # # A generator to generate combinations of options (e.g.: tcp/compress/mx, ssl/ipv6/serialize, etc) # def gen(supportedOptions): - if not supportedOptions: yield self return @@ -692,8 +800,10 @@ def gen(supportedOptions): length = max([len(v) for v in supportedOptions.values()]) # Replace the values with a cycle iterator on the values - for (k, v) in supportedOptions.items(): - supportedOptions[k] = itertools.cycle(random.sample(v, len(v)) if rand else v) + for k, v in supportedOptions.items(): + supportedOptions[k] = itertools.cycle( + random.sample(v, len(v)) if rand else v + ) # Now, for the length of the longest array of values, we return # an array with the supported option combinations @@ -721,10 +831,19 @@ def gen(supportedOptions): yield self.__class__(options) - return [c for c in gen(current.driver.filterOptions(current.driver.getComponent().getOptions(testcase, current)))] + return [ + c + for c in gen( + current.driver.filterOptions( + current.driver.getComponent().getOptions(testcase, current) + ) + ) + ] def canRun(self, testId, current): - if not current.driver.getComponent().canRun(testId, current.testcase.getMapping(), current): + if not current.driver.getComponent().canRun( + testId, current.testcase.getMapping(), current + ): return False options = {} @@ -732,7 +851,7 @@ def canRun(self, testId, current): options.update(current.testcase.getTestSuite().getOptions(current)) options.update(current.testcase.getOptions(current)) - for (k, v) in options.items(): + for k, v in options.items(): if hasattr(self, k): if not getattr(self, k) in v: return False @@ -788,8 +907,12 @@ def getProps(self, process, current): if self.ipv6: props["Ice.PreferIPv6Address"] = True if self.mx: - props["Ice.Admin.Endpoints"] = "tcp -h \"::1\"" if self.ipv6 else "tcp -h 127.0.0.1" - props["Ice.Admin.InstanceName"] = "Server" if isinstance(process, Server) else "Client" + props["Ice.Admin.Endpoints"] = ( + 'tcp -h "::1"' if self.ipv6 else "tcp -h 127.0.0.1" + ) + props["Ice.Admin.InstanceName"] = ( + "Server" if isinstance(process, Server) else "Client" + ) props["IceMX.Metrics.Debug.GroupBy"] = "id" props["IceMX.Metrics.Parent.GroupBy"] = "parent" props["IceMX.Metrics.All.GroupBy"] = "none" @@ -828,8 +951,11 @@ def getProps(self, process, current): @classmethod def getByName(self, name): if not name in self.mappings: - raise RuntimeError("unknown mapping: `{0}', known mappings: `{1}'".format( - name, list(self.mappings))) + raise RuntimeError( + "unknown mapping: `{0}', known mappings: `{1}'".format( + name, list(self.mappings) + ) + ) return self.mappings.get(name) @classmethod @@ -870,8 +996,19 @@ def remove(self, name): @classmethod def getAll(self, driver=None, includeDisabled=False): - return [m for m in self.mappings.values() if not driver or driver.matchLanguage(str(m))] + \ - ([m for m in self.disabled.values() if not driver or driver.matchLanguage(str(m))] if includeDisabled else []) + return [ + m + for m in self.mappings.values() + if not driver or driver.matchLanguage(str(m)) + ] + ( + [ + m + for m in self.disabled.values() + if not driver or driver.matchLanguage(str(m)) + ] + if includeDisabled + else [] + ) def __init__(self, path=None): self.name = None @@ -882,7 +1019,9 @@ def init(self, name, component, path=None): self.name = name self.component = component if not self.path: - self.path = os.path.normpath(os.path.join(self.component.getSourceDir(), path or name)) + self.path = os.path.normpath( + os.path.join(self.component.getSourceDir(), path or name) + ) return self def __str__(self): @@ -918,12 +1057,17 @@ def loadTestSuites(self, tests, config, filters=[], rfilters=[]): try: origsyspath = sys.path prefix = os.path.commonprefix([toplevel, self.component.getScriptDir()]) - moduleprefix = self.component.getScriptDir()[len(prefix) + 1:].replace(os.sep, ".") + "." + moduleprefix = ( + self.component.getScriptDir()[len(prefix) + 1 :].replace(os.sep, ".") + + "." + ) sys.path = [prefix] + sys.path for test in tests or [""]: testDir = self.component.getTestDir(self) - for root, dirs, files in os.walk(os.path.join(testDir, test.replace('/', os.sep))): - testId = root[len(testDir) + 1:] + for root, dirs, files in os.walk( + os.path.join(testDir, test.replace("/", os.sep)) + ): + testId = root[len(testDir) + 1 :] if os.sep != "/": testId = testId.replace(os.sep, "/") @@ -937,7 +1081,7 @@ def loadTestSuites(self, tests, config, filters=[], rfilters=[]): # # WORKAROUND for Python issue 15230 (fixed in 3.2) where run_path doesn't work correctly. # - #runpy.run_path(os.path.join(root, "test.py")) + # runpy.run_path(os.path.join(root, "test.py")) origsyspath = sys.path sys.path = [root] + sys.path runpy.run_module("test", init_globals=globals(), run_name=root) @@ -948,8 +1092,14 @@ def loadTestSuites(self, tests, config, filters=[], rfilters=[]): # If there's no test.py file in the test directory, we check if there's a common # script for the test in scripts/tests. If there's one we use it. # - if os.path.isfile(os.path.join(self.component.getScriptDir(), testId + ".py")): - runpy.run_module(moduleprefix + testId.replace("/", "."), init_globals=globals(), run_name=root) + if os.path.isfile( + os.path.join(self.component.getScriptDir(), testId + ".py") + ): + runpy.run_module( + moduleprefix + testId.replace("/", "."), + init_globals=globals(), + run_name=root, + ) continue # @@ -966,40 +1116,66 @@ def loadTestSuites(self, tests, config, filters=[], rfilters=[]): def getTestSuites(self, ids=[]): if not ids: return self.testsuites.values() - return [self.testsuites[testSuiteId] for testSuiteId in ids if testSuiteId in self.testsuites] + return [ + self.testsuites[testSuiteId] + for testSuiteId in ids + if testSuiteId in self.testsuites + ] def addTestSuite(self, testsuite): assert len(testsuite.path) > len(self.component.getTestDir(self)) + 1 - testSuiteId = testsuite.path[len(self.component.getTestDir(self)) + 1:].replace('\\', '/') + testSuiteId = testsuite.path[ + len(self.component.getTestDir(self)) + 1 : + ].replace("\\", "/") self.testsuites[testSuiteId] = testsuite return testSuiteId def findTestSuite(self, testsuite): - return self.testsuites.get(testsuite if isinstance(testsuite, str) else testsuite.id) + return self.testsuites.get( + testsuite if isinstance(testsuite, str) else testsuite.id + ) def computeTestCases(self, testId, files): - # Instantiate a new test suite if the directory contains well-known source files. def checkFile(f, m): try: # If given mapping is same as local mapping, just check the files set, otherwise check # with the mapping - return (self.getDefaultSource(f) in files) if m == self else m.hasSource(testId, f) + return ( + (self.getDefaultSource(f) in files) + if m == self + else m.hasSource(testId, f) + ) except KeyError: # Expected if the mapping doesn't support the process type return False - def checkClient(f): return checkFile(f, self.getClientMapping(testId)) - def checkServer(f): return checkFile(f, self.getServerMapping(testId)) + def checkClient(f): + return checkFile(f, self.getClientMapping(testId)) + + def checkServer(f): + return checkFile(f, self.getServerMapping(testId)) testcases = [] if checkClient("client") and checkServer("server"): testcases.append(ClientServerTestCase()) - if checkClient("client") and checkServer("serveramd") and self.getServerMapping(testId) == self: + if ( + checkClient("client") + and checkServer("serveramd") + and self.getServerMapping(testId) == self + ): testcases.append(ClientAMDServerTestCase()) - if checkClient("client") and checkServer("servertie") and self.getServerMapping(testId) == self: + if ( + checkClient("client") + and checkServer("servertie") + and self.getServerMapping(testId) == self + ): testcases.append(ClientTieServerTestCase()) - if checkClient("client") and checkServer("serveramdtie") and self.getServerMapping(testId) == self: + if ( + checkClient("client") + and checkServer("serveramdtie") + and self.getServerMapping(testId) == self + ): testcases.append(ClientAMDTieServerTestCase()) if checkClient("client") and len(testcases) == 0: testcases.append(ClientTestCase()) @@ -1010,7 +1186,13 @@ def checkServer(f): return checkFile(f, self.getServerMapping(testId)) def hasSource(self, testId, processType): try: - return os.path.exists(os.path.join(self.component.getTestDir(self), testId, self.getDefaultSource(processType))) + return os.path.exists( + os.path.join( + self.component.getTestDir(self), + testId, + self.getDefaultSource(processType), + ) + ) except KeyError: return False @@ -1027,7 +1209,9 @@ def getDefaultSource(self, processType): return self._getDefaultSource(processType) def getDefaultProcesses(self, processType, testsuite): - default = self.component.getDefaultProcesses(self, processType, testsuite.getId()) + default = self.component.getDefaultProcesses( + self, processType, testsuite.getId() + ) if default: return default return self._getDefaultProcesses(processType) @@ -1049,7 +1233,13 @@ def _getDefaultProcesses(self, processType): name, ext = os.path.splitext(self.getDefaultSource(processType)) if name in globals(): return [globals()[name]()] - return [Server()] if processType.startswith("server") else [Client()] if processType else [] + return ( + [Server()] + if processType.startswith("server") + else [Client()] + if processType + else [] + ) def _getDefaultExe(self, processType): return os.path.splitext(self.getDefaultSource(processType))[0] @@ -1073,7 +1263,9 @@ def getCommandLine(self, current, process, exe, args): cmd = os.path.join(self.component.getBinDir(process, self, current), exe) elif current.testcase: # If it's a process from a testcase, the binary is in the test build directory. - cmd = os.path.join(current.testcase.getPath(current), current.getBuildDir(exe), exe) + cmd = os.path.join( + current.testcase.getPath(current), current.getBuildDir(exe), exe + ) else: cmd = exe @@ -1086,7 +1278,9 @@ def getProps(self, process, current): props = {} if isinstance(process, IceProcess): if current.config.protocol in ["bt", "bts"]: - props["Ice.Plugin.IceBT"] = self.getPluginEntryPoint("IceBT", process, current) + props["Ice.Plugin.IceBT"] = self.getPluginEntryPoint( + "IceBT", process, current + ) if current.config.protocol in ["ssl", "wss", "bts", "iaps"]: props.update(self.getSSLProps(process, current)) return props @@ -1095,7 +1289,9 @@ def getSSLProps(self, process, current): sslProps = { "Ice.Plugin.IceSSL": self.getPluginEntryPoint("IceSSL", process, current), "IceSSL.Password": "password", - "IceSSL.DefaultDir": "" if current.config.buildPlatform == "iphoneos" else os.path.join(self.component.getSourceDir(), "certs"), + "IceSSL.DefaultDir": "" + if current.config.buildPlatform == "iphoneos" + else os.path.join(self.component.getSourceDir(), "certs"), } # @@ -1116,6 +1312,7 @@ def getEnv(self, process, current): def getOptions(self, current): return {} + # # A Runnable can be used as a "client" for in test cases, it provides # implements run, setup and teardown methods. @@ -1123,7 +1320,6 @@ def getOptions(self, current): class Runnable(object): - def __init__(self, desc=None): self.desc = desc @@ -1138,17 +1334,28 @@ def teardown(self, current, success): def run(self, current): pass + # # A Process describes how to run an executable process. # class Process(Runnable): - processType = None - def __init__(self, exe=None, outfilters=None, quiet=False, args=None, props=None, envs=None, desc=None, - mapping=None, preexec_fn=None, traceProps=None): + def __init__( + self, + exe=None, + outfilters=None, + quiet=False, + args=None, + props=None, + envs=None, + desc=None, + mapping=None, + preexec_fn=None, + traceProps=None, + ): Runnable.__init__(self, desc) self.exe = exe self.outfilters = outfilters or [] @@ -1166,7 +1373,7 @@ def __str__(self): return self.exe + (" ({0})".format(self.desc) if self.desc else "") def getOutput(self, current, encoding="utf-8"): - assert (self in current.processes) + assert self in current.processes def d(s): return s.decode(encoding) if isinstance(s, bytes) else s @@ -1175,17 +1382,19 @@ def d(s): try: # Apply outfilters to the output if len(self.outfilters) > 0: - lines = output.split('\n') + lines = output.split("\n") newLines = [] previous = "" - for line in [line + '\n' for line in lines]: + for line in [line + "\n" for line in lines]: for f in self.outfilters: - if isinstance(f, types.LambdaType) or isinstance(f, types.FunctionType): + if isinstance(f, types.LambdaType) or isinstance( + f, types.FunctionType + ): line = f(line) elif f.search(line): break else: - if line.endswith('\n'): + if line.endswith("\n"): if previous: newLines.append(previous + line) previous = "" @@ -1195,14 +1404,13 @@ def d(s): previous += line output = "".join(newLines) output = output.strip() - return output + '\n' if output else "" + return output + "\n" if output else "" except Exception as ex: print("unexpected exception while filtering process output:\n" + str(ex)) raise def run(self, current, args=[], props={}, exitstatus=0, timeout=None): class WatchDog: - def __init__(self, timeout): self.lastProgressTime = time.time() self.lock = threading.Lock() @@ -1237,7 +1445,11 @@ def timedOut(self, timeout): raise except Expect.TIMEOUT: if watchDog and watchDog.timedOut(timeout): - print("process {0} is hanging - {1}".format(process, time.strftime("%x %X"))) + print( + "process {0} is hanging - {1}".format( + process, time.strftime("%x %X") + ) + ) if current.driver.isInterrupted(): self.stop(current, False, exitstatus) raise @@ -1253,7 +1465,9 @@ def getEffectiveArgs(self, current, args): allArgs += self.getArgs(current) allArgs += self.args(self, current) if callable(self.args) else self.args allArgs += args - allArgs = [a.encode("utf-8") if type(a) == "unicode" else str(a) for a in allArgs] + allArgs = [ + a.encode("utf-8") if type(a) == "unicode" else str(a) for a in allArgs + ] return allArgs def getEffectiveProps(self, current, props): @@ -1264,15 +1478,21 @@ def getEffectiveProps(self, current, props): allProps.update(self.getMapping(current).getProps(self, current)) allProps.update(current.testcase.getProps(self, current)) allProps.update(self.getProps(current)) - allProps.update(self.props(self, current) if callable(self.props) else self.props) + allProps.update( + self.props(self, current) if callable(self.props) else self.props + ) allProps.update(props) return allProps def getEffectiveEnv(self, current): - def merge(envs, newEnvs): - if platform.getLdPathEnvName() in newEnvs and platform.getLdPathEnvName() in envs: - newEnvs[platform.getLdPathEnvName()] += os.pathsep + envs[platform.getLdPathEnvName()] + if ( + platform.getLdPathEnvName() in newEnvs + and platform.getLdPathEnvName() in envs + ): + newEnvs[platform.getLdPathEnvName()] += ( + os.pathsep + envs[platform.getLdPathEnvName()] + ) envs.update(newEnvs) allEnvs = {} @@ -1286,7 +1506,11 @@ def merge(envs, newEnvs): def getEffectiveTraceProps(self, current): traceProps = {} traceProps.update(current.testcase.getTraceProps(self, current)) - traceProps.update(self.traceProps(self, current) if callable(self.traceProps) else self.traceProps) + traceProps.update( + self.traceProps(self, current) + if callable(self.traceProps) + else self.traceProps + ) return traceProps def start(self, current, args=[], props={}, watchDog=None): @@ -1295,7 +1519,9 @@ def start(self, current, args=[], props={}, watchDog=None): allEnvs = self.getEffectiveEnv(current) processController = current.driver.getProcessController(current, self) - current.processes[self] = processController.start(self, current, allArgs, allProps, allEnvs, watchDog) + current.processes[self] = processController.start( + self, current, allArgs, allProps, allEnvs, watchDog + ) try: self.waitForStart(current) except: @@ -1320,7 +1546,11 @@ def stop(self, current, waitSuccess=False, exitstatus=0): current.driver.setInterrupt(True) raise except Expect.TIMEOUT: - print("process {0} is hanging on shutdown - {1}".format(process, time.strftime("%x %X"))) + print( + "process {0} is hanging on shutdown - {1}".format( + process, time.strftime("%x %X") + ) + ) if current.driver.isInterrupted(): raise except RuntimeError as ex: @@ -1332,7 +1562,9 @@ def stop(self, current, waitSuccess=False, exitstatus=0): finally: if not process.isTerminated(): process.terminate() - if not self.quiet: # Write the output to the test case (but not on stdout) + if ( + not self.quiet + ): # Write the output to the test case (but not on stdout) current.write(self.getOutput(current), stdout=False) def teardown(self, current, success): @@ -1340,19 +1572,27 @@ def teardown(self, current, success): current.processes[self].teardown(current, success) def expect(self, current, pattern, timeout=60): - assert (self in current.processes and isinstance(current.processes[self], Expect.Expect)) + assert self in current.processes and isinstance( + current.processes[self], Expect.Expect + ) return current.processes[self].expect(pattern, timeout) def expectall(self, current, pattern, timeout=60): - assert (self in current.processes and isinstance(current.processes[self], Expect.Expect)) + assert self in current.processes and isinstance( + current.processes[self], Expect.Expect + ) return current.processes[self].expectall(pattern, timeout) def sendline(self, current, data): - assert (self in current.processes and isinstance(current.processes[self], Expect.Expect)) + assert self in current.processes and isinstance( + current.processes[self], Expect.Expect + ) return current.processes[self].sendline(data) def getMatch(self, current): - assert (self in current.processes and isinstance(current.processes[self], Expect.Expect)) + assert self in current.processes and isinstance( + current.processes[self], Expect.Expect + ) return current.processes[self].match def isStarted(self, current): @@ -1381,7 +1621,12 @@ def getExe(self, current): return self.exe or self.getMapping(current).getDefaultExe(processType) def getCommandLine(self, current, args=""): - return self.getMapping(current).getCommandLine(current, self, self.getExe(current), args).strip() + return ( + self.getMapping(current) + .getCommandLine(current, self, self.getExe(current), args) + .strip() + ) + # # A simple client (used to run Slice/IceUtil clients for example) @@ -1391,6 +1636,7 @@ def getCommandLine(self, current, args=""): class SimpleClient(Process): pass + # # An IceProcess specialization class. This is used by drivers to figure out if # the process accepts Ice configuration properties. @@ -1400,6 +1646,7 @@ class SimpleClient(Process): class IceProcess(Process): pass + # # An Ice server process. It's possible to configure when the server is considered # ready by setting readyCount or ready. The start method will only return once @@ -1409,8 +1656,16 @@ class IceProcess(Process): class Server(IceProcess): - - def __init__(self, exe=None, waitForShutdown=True, readyCount=1, ready=None, startTimeout=300, *args, **kargs): + def __init__( + self, + exe=None, + waitForShutdown=True, + readyCount=1, + ready=None, + startTimeout=300, + *args, + **kargs, + ): IceProcess.__init__(self, exe, *args, **kargs) self.waitForShutdown = waitForShutdown self.readyCount = readyCount @@ -1419,19 +1674,27 @@ def __init__(self, exe=None, waitForShutdown=True, readyCount=1, ready=None, sta def getProps(self, current): props = IceProcess.getProps(self, current) - props.update({ - "Ice.ThreadPool.Server.Size": 1, - "Ice.ThreadPool.Server.SizeMax": 3, - "Ice.ThreadPool.Server.SizeWarn": 0, - }) - props.update(current.driver.getProcessProps(current, self.ready, - self.readyCount + (1 if current.config.mx else 0))) + props.update( + { + "Ice.ThreadPool.Server.Size": 1, + "Ice.ThreadPool.Server.SizeMax": 3, + "Ice.ThreadPool.Server.SizeWarn": 0, + } + ) + props.update( + current.driver.getProcessProps( + current, self.ready, self.readyCount + (1 if current.config.mx else 0) + ) + ) return props def waitForStart(self, current): # Wait for the process to be ready - current.processes[self].waitReady(self.ready, self.readyCount + - (1 if current.config.mx else 0), self.startTimeout) + current.processes[self].waitReady( + self.ready, + self.readyCount + (1 if current.config.mx else 0), + self.startTimeout, + ) # Filter out remaining ready messages self.outfilters.append(re.compile("[^\n]+ ready")) @@ -1443,6 +1706,7 @@ def waitForStart(self, current): def stop(self, current, waitSuccess=False, exitstatus=0): IceProcess.stop(self, current, waitSuccess and self.waitForShutdown, exitstatus) + # # An Ice client process. # @@ -1451,6 +1715,7 @@ def stop(self, current, waitSuccess=False, exitstatus=0): class Client(IceProcess): pass + # # Executables for processes inheriting this marker class are looked up in the # Ice distribution bin directory. @@ -1458,10 +1723,10 @@ class Client(IceProcess): class ProcessFromBinDir: - def isFromBinDir(self): return True + # # Executables for processes inheriting this marker class are only provided # as a Release executable on Windows @@ -1469,22 +1734,24 @@ def isFromBinDir(self): class ProcessIsReleaseOnly: - def isReleaseOnly(self): return True class SliceTranslator(ProcessFromBinDir, ProcessIsReleaseOnly, SimpleClient): - def __init__(self, translator): - SimpleClient.__init__(self, exe=translator, quiet=True, mapping=Mapping.getByName("cpp")) + SimpleClient.__init__( + self, exe=translator, quiet=True, mapping=Mapping.getByName("cpp") + ) def getCommandLine(self, current, args=""): # # Look for slice2py installed by pip if not found in the bin directory # if self.exe == "slice2py": - translator = self.getMapping(current).getCommandLine(current, self, self.getExe(current), "") + translator = self.getMapping(current).getCommandLine( + current, self, self.getExe(current), "" + ) if not os.path.exists(translator): translator = sys.executable + " -m slice2py" return (translator + " " + args).strip() @@ -1501,13 +1768,16 @@ class Collocated(Client): class EchoServer(Server): - def __init__(self): - Server.__init__(self, mapping=Mapping.getByName("cpp"), quiet=True, waitForShutdown=False) + Server.__init__( + self, mapping=Mapping.getByName("cpp"), quiet=True, waitForShutdown=False + ) def getProps(self, current): props = Server.getProps(self, current) - props["Ice.MessageSizeMax"] = 8192 # Don't limit the amount of data to transmit between client/server + props[ + "Ice.MessageSizeMax" + ] = 8192 # Don't limit the amount of data to transmit between client/server return props def getCommandLine(self, current, args=""): @@ -1517,6 +1787,7 @@ def getCommandLine(self, current, args=""): finally: current.pop() + # # A test case is composed of servers and clients. When run, all servers are started # sequentially. When the servers are ready, the clients are also ran sequentially. @@ -1529,9 +1800,20 @@ def getCommandLine(self, current, args=""): class TestCase(Runnable): - - def __init__(self, name, client=None, clients=None, server=None, servers=None, args=None, props=None, envs=None, - options=None, desc=None, traceProps=None): + def __init__( + self, + name, + client=None, + clients=None, + server=None, + servers=None, + args=None, + props=None, + envs=None, + options=None, + desc=None, + traceProps=None, + ): Runnable.__init__(self, desc) self.name = name @@ -1578,7 +1860,8 @@ def init(self, mapping, testsuite): if not self.clients: if self.getClientType(): self.clients = self.mapping.getClientMapping( - testId).getDefaultProcesses(self.getClientType(), testsuite) + testId + ).getDefaultProcesses(self.getClientType(), testsuite) else: self.clients = [] @@ -1589,7 +1872,8 @@ def init(self, mapping, testsuite): if not self.servers: if self.getServerType(): self.servers = self.mapping.getServerMapping( - testId).getDefaultProcesses(self.getServerType(), testsuite) + testId + ).getDefaultProcesses(self.getServerType(), testsuite) else: self.servers = [] @@ -1680,11 +1964,17 @@ def getServerType(self): return None def getServerTestCase(self, cross=None): - testsuite = (cross or self.mapping).getServerMapping(self.testsuite.getId()).findTestSuite(self.testsuite) + testsuite = ( + (cross or self.mapping) + .getServerMapping(self.testsuite.getId()) + .findTestSuite(self.testsuite) + ) return testsuite.findTestCase(self) if testsuite else None def getClientTestCase(self): - testsuite = self.mapping.getClientMapping(self.testsuite.getId()).findTestSuite(self.testsuite) + testsuite = self.mapping.getClientMapping(self.testsuite.getId()).findTestSuite( + self.testsuite + ) return testsuite.findTestCase(self) if testsuite else None def _startServerSide(self, current): @@ -1764,14 +2054,15 @@ def run(self, current): except Exception as ex: self.teardown(current, False) if not self.parent: - current.result.failed(current, traceback.format_exc() if current.driver.debug else str(ex)) + current.result.failed( + current, traceback.format_exc() if current.driver.debug else str(ex) + ) raise finally: current.pop() class ClientTestCase(TestCase): - def __init__(self, name="client", *args, **kargs): TestCase.__init__(self, name, *args, **kargs) @@ -1783,7 +2074,6 @@ def getClientType(self): class ClientServerTestCase(ClientTestCase): - def __init__(self, name="client/server", *args, **kargs): TestCase.__init__(self, name, *args, **kargs) @@ -1795,7 +2085,6 @@ def getServerType(self): class CollocatedTestCase(ClientTestCase): - def __init__(self, name="collocated", *args, **kargs): TestCase.__init__(self, name, *args, **kargs) @@ -1804,7 +2093,6 @@ def getClientType(self): class ClientAMDServerTestCase(ClientServerTestCase): - def __init__(self, name="client/amd server", *args, **kargs): ClientServerTestCase.__init__(self, name, *args, **kargs) @@ -1813,7 +2101,6 @@ def getServerType(self): class ClientTieServerTestCase(ClientServerTestCase): - def __init__(self, name="client/tie server", *args, **kargs): ClientServerTestCase.__init__(self, name, *args, **kargs) @@ -1822,7 +2109,6 @@ def getServerType(self): class ClientAMDTieServerTestCase(ClientServerTestCase): - def __init__(self, name="client/amd tie server", *args, **kargs): ClientServerTestCase.__init__(self, name, *args, **kargs) @@ -1831,11 +2117,15 @@ def getServerType(self): class Result: + def getKey(self, current): + return ( + (current.testcase, current.config) + if isinstance(current, Driver.Current) + else current + ) - def getKey(self, current): return (current.testcase, current.config) if isinstance( - current, Driver.Current) else current - - def getDesc(self, current): return current.desc if isinstance(current, Driver.Current) else "" + def getDesc(self, current): + return current.desc if isinstance(current, Driver.Current) else "" def __init__(self, testsuite, writeToStdout): self.testsuite = testsuite @@ -1862,7 +2152,12 @@ def failed(self, current, exception): key = self.getKey(current) self._testCaseDuration = time.time() - self._testCaseDuration self.writeln("\ntest in {0} failed:\n{1}".format(self.testsuite, exception)) - self._testcases[key] = (self._start, self._stdout.tell(), self._testCaseDuration, self.getDesc(current)) + self._testcases[key] = ( + self._start, + self._stdout.tell(), + self._testCaseDuration, + self.getDesc(current), + ) self._failed[key] = exception # If ADDRINUSE, dump the current processes @@ -1871,14 +2166,21 @@ def failed(self, current, exception): if output.find(s) >= 0: if isinstance(platform, Windows): self.writeln(run("netstat -on")) - self.writeln(run("powershell.exe \"Get-Process | Select id,name,path\"")) + self.writeln( + run('powershell.exe "Get-Process | Select id,name,path"') + ) else: self.writeln(run("lsof -n -P -i; ps ax")) def succeeded(self, current): key = self.getKey(current) self._testCaseDuration = time.time() - self._testCaseDuration - self._testcases[key] = (self._start, self._stdout.tell(), self._testCaseDuration, self.getDesc(current)) + self._testcases[key] = ( + self._start, + self._stdout.tell(), + self._testCaseDuration, + self.getDesc(current), + ) def skipped(self, current, reason): self.writeln("skipped, " + reason) @@ -1936,15 +2238,18 @@ def writeln(self, msg, stdout=True): self._stdout.write("\n") def writeAsXml(self, out, hostname=""): - out.write(' \n' - .format(len(self._testcases) - 2, - len(self._failed), - 0, - self._duration, - self.testsuite, - self.testsuite.getMapping())) - - for (k, v) in self._testcases.items(): + out.write( + ' \n'.format( + len(self._testcases) - 2, + len(self._failed), + 0, + self._duration, + self.testsuite, + self.testsuite.getMapping(), + ) + ) + + for k, v in self._testcases.items(): if isinstance(k, str): # Don't keep track of setup/teardown steps continue @@ -1962,37 +2267,53 @@ def writeAsXml(self, out, hostname=""): name = str(tc) if hostname: name += " on " + hostname - out.write(' \n' - .format(escapeXml(name), - d, - self.testsuite.getMapping(), - self.testsuite.getId().replace("/", "."))) + out.write( + ' \n'.format( + escapeXml(name), + d, + self.testsuite.getMapping(), + self.testsuite.getId().replace("/", "."), + ) + ) if k in self._failed: - last = self._failed[k].strip().split('\n') + last = self._failed[k].strip().split("\n") if len(last) > 0: last = last[len(last) - 1] if hostname: last = "Failed on {0}\n{1}".format(hostname, last) - out.write(' {0}\n'.format(escapeXml(self._failed[k]), - escapeXml(last, True))) + out.write( + " {0}\n".format( + escapeXml(self._failed[k]), escapeXml(last, True) + ) + ) # elif k in self._skipped: # out.write(' \n'.format(escapeXml(self._skipped[k], True))) - out.write(' \n') + out.write(" \n") if hostname: - out.write('Running on {0}\n'.format(hostname)) + out.write("Running on {0}\n".format(hostname)) out.write(escapeXml(self.getOutput(k))) - out.write(' \n') - out.write(' \n') + out.write(" \n") + out.write(" \n") - out.write('\n') + out.write("\n") class TestSuite(object): - - def __init__(self, path, testcases=None, options=None, libDirs=None, runOnMainThread=False, chdir=False, - multihost=True, mapping=None): + def __init__( + self, + path, + testcases=None, + options=None, + libDirs=None, + runOnMainThread=False, + chdir=False, + multihost=True, + mapping=None, + ): global currentMapping - self.path = os.path.dirname(path) if os.path.basename(path) == "test.py" else path + self.path = ( + os.path.dirname(path) if os.path.basename(path) == "test.py" else path + ) self.mapping = currentMapping or Mapping.getByPath(self.path) self.id = self.mapping.addTestSuite(self) self.options = options or {} @@ -2005,13 +2326,19 @@ def __init__(self, path, testcases=None, options=None, libDirs=None, runOnMainTh self.runOnMainThread = True if testcases is None: - files = [f for f in os.listdir(self.path) if os.path.isfile(os.path.join(self.path, f))] + files = [ + f + for f in os.listdir(self.path) + if os.path.isfile(os.path.join(self.path, f)) + ] testcases = self.mapping.computeTestCases(self.id, files) self.testcases = OrderedDict() for testcase in testcases if testcases else []: testcase.init(self.mapping, self) if testcase.name in self.testcases: - raise RuntimeError("duplicate testcase {0} in testsuite {1}".format(testcase, self)) + raise RuntimeError( + "duplicate testcase {0} in testsuite {1}".format(testcase, self) + ) self.testcases[testcase.name] = testcase def __str__(self): @@ -2046,12 +2373,16 @@ def isMainThreadOnly(self, driver): def addTestCase(self, testcase): if testcase.name in self.testcases: - raise RuntimeError("duplicate testcase {0} in testsuite {1}".format(testcase, self)) + raise RuntimeError( + "duplicate testcase {0} in testsuite {1}".format(testcase, self) + ) testcase.init(self.mapping, self) self.testcases[testcase.name] = testcase def findTestCase(self, testcase): - return self.testcases.get(testcase if isinstance(testcase, str) else testcase.name) + return self.testcases.get( + testcase if isinstance(testcase, str) else testcase.name + ) def getTestCases(self): return self.testcases.values() @@ -2080,7 +2411,6 @@ def isMultiHost(self): class ProcessController: - def __init__(self, current): pass @@ -2092,9 +2422,7 @@ def destroy(self, driver): class LocalProcessController(ProcessController): - class LocalProcess(Expect.Expect): - def __init__(self, traceFile, *args, **kargs): Expect.Expect.__init__(self, *args, **kargs) self.traceFile = traceFile @@ -2121,7 +2449,6 @@ def getHost(self, current): return current.driver.getHost(current.config.protocol, current.config.ipv6) def start(self, process, current, args, props, envs, watchDog): - # # Props and arguments can use the format parameters set below in the kargs # dictionary. It's time to convert them to their values. @@ -2140,20 +2467,31 @@ def start(self, process, current, args, props, envs, watchDog): if "Ice.ProgramName" in props: programName = props["Ice.ProgramName"] else: - programName = process.exe or current.testcase.getProcessType(process) - traceFile = os.path.join(current.testsuite.getPath(), - "{0}-{1}.log".format(programName, time.strftime("%m%d%y-%H%M"))) + programName = process.exe or current.testcase.getProcessType( + process + ) + traceFile = os.path.join( + current.testsuite.getPath(), + "{0}-{1}.log".format(programName, time.strftime("%m%d%y-%H%M")), + ) traceProps["Ice.LogFile"] = traceFile props.update(traceProps) - args = ["--{0}={1}".format(k, val(v)) for k, v in props.items()] + [val(a) for a in args] + args = ["--{0}={1}".format(k, val(v)) for k, v in props.items()] + [ + val(a) for a in args + ] for k, v in envs.items(): envs[k] = val(v, quoteValue=False) cmd = "" if current.driver.valgrind: - cmd += "valgrind -q --child-silent-after-fork=yes --leak-check=full --suppressions=\"{0}\" ".format( - os.path.join(current.driver.getComponent().getSourceDir(), "config", "valgrind.sup")) + cmd += 'valgrind -q --child-silent-after-fork=yes --leak-check=full --suppressions="{0}" '.format( + os.path.join( + current.driver.getComponent().getSourceDir(), + "config", + "valgrind.sup", + ) + ) exe = process.getCommandLine(current, " ".join(args)) cmd += exe.format(**kargs) @@ -2167,20 +2505,21 @@ def start(self, process, current, args, props, envs, watchDog): env.update(envs) mapping = process.getMapping(current) cwd = mapping.getTestCwd(process, current) - process = LocalProcessController.LocalProcess(command=cmd, - startReader=False, - env=env, - cwd=cwd, - desc=process.desc or exe, - preexec_fn=process.preexec_fn, - mapping=str(mapping), - traceFile=traceFile) + process = LocalProcessController.LocalProcess( + command=cmd, + startReader=False, + env=env, + cwd=cwd, + desc=process.desc or exe, + preexec_fn=process.preexec_fn, + mapping=str(mapping), + traceFile=traceFile, + ) process.startReader(watchDog) return process class RemoteProcessController(ProcessController): - class RemoteProcess: def __init__(self, exe, proxy): self.exe = exe @@ -2197,6 +2536,7 @@ def waitReady(self, ready, readyCount, startTimeout): def waitSuccess(self, exitstatus=0, timeout=60): import Ice + try: result = self.proxy.waitSuccess(timeout) except Ice.UserException as ex: @@ -2208,7 +2548,10 @@ def waitSuccess(self, exitstatus=0, timeout=60): raise if exitstatus != result: - raise RuntimeError("unexpected exit status: expected: %d, got %d\n" % (exitstatus, result)) + raise RuntimeError( + "unexpected exit status: expected: %d, got %d\n" + % (exitstatus, result) + ) def getOutput(self): return self.output @@ -2238,26 +2581,34 @@ def __init__(self, current, endpoints="tcp"): import Test class ProcessControllerRegistryI(Test.Common.ProcessControllerRegistry): - def __init__(self, remoteProcessController): self.remoteProcessController = remoteProcessController def setProcessController(self, proxy, current): import Test - proxy = Test.Common.ProcessControllerPrx.uncheckedCast(current.con.createProxy(proxy.ice_getIdentity())) + + proxy = Test.Common.ProcessControllerPrx.uncheckedCast( + current.con.createProxy(proxy.ice_getIdentity()) + ) self.remoteProcessController.setProcessController(proxy) import Ice + comm.getProperties().setProperty("Adapter.AdapterId", Ice.generateUUID()) self.adapter = comm.createObjectAdapterWithEndpoints("Adapter", endpoints) - self.adapter.add(ProcessControllerRegistryI(self), comm.stringToIdentity("Util/ProcessControllerRegistry")) + self.adapter.add( + ProcessControllerRegistryI(self), + comm.stringToIdentity("Util/ProcessControllerRegistry"), + ) self.adapter.activate() def __str__(self): return "remote controller" def getHost(self, current): - return self.getController(current).getHost(current.config.protocol, current.config.ipv6) + return self.getController(current).getHost( + current.config.protocol, current.config.ipv6 + ) def getController(self, current): ident = self.getControllerIdentity(current) @@ -2282,13 +2633,17 @@ def getController(self, current): if current.driver.controllerApp: if ident in self.controllerApps: - self.restartControllerApp(current, ident) # Controller must have crashed, restart it + self.restartControllerApp( + current, ident + ) # Controller must have crashed, restart it else: self.controllerApps.append(ident) self.startControllerApp(current, ident) # Use well-known proxy and IceDiscovery to discover the process controller object from the app. - proxy = Test.Common.ProcessControllerPrx.uncheckedCast(comm.stringToProxy(comm.identityToString(ident))) + proxy = Test.Common.ProcessControllerPrx.uncheckedCast( + comm.stringToProxy(comm.identityToString(ident)) + ) # # First try to discover the process controller with IceDiscovery, if this doesn't @@ -2332,8 +2687,10 @@ def setProcessController(self, proxy): with self.cond: self.processControllerProxies[proxy.ice_getIdentity()] = proxy conn = proxy.ice_getConnection() - if (hasattr(conn, "setCloseCallback")): - proxy.ice_getConnection().setCloseCallback(lambda conn: self.clearProcessController(proxy, conn)) + if hasattr(conn, "setCloseCallback"): + proxy.ice_getConnection().setCloseCallback( + lambda conn: self.clearProcessController(proxy, conn) + ) else: import Ice @@ -2359,7 +2716,12 @@ def clearProcessController(self, proxy, conn=None): if proxy.ice_getIdentity() in self.processControllerProxies: if not conn: conn = proxy.ice_getCachedConnection() - if conn == self.processControllerProxies[proxy.ice_getIdentity()].ice_getCachedConnection(): + if ( + conn + == self.processControllerProxies[ + proxy.ice_getIdentity() + ].ice_getCachedConnection() + ): del self.processControllerProxies[proxy.ice_getIdentity()] def startControllerApp(self, current, ident): @@ -2379,16 +2741,27 @@ def start(self, process, current, args, props, envs, watchDog): # TODO: support envs? exe = process.getExe(current) - args = ["--{0}={1}".format(k, val(v, quoteValue=False)) for k, v in props.items()] + [val(a) for a in args] + args = [ + "--{0}={1}".format(k, val(v, quoteValue=False)) for k, v in props.items() + ] + [val(a) for a in args] if current.driver.debug: - current.writeln("(executing `{0}/{1}' on `{2}' args = {3})".format(current.testsuite, exe, self, args)) + current.writeln( + "(executing `{0}/{1}' on `{2}' args = {3})".format( + current.testsuite, exe, self, args + ) + ) prx = processController.start(str(current.testsuite), exe, args) # Create bi-dir proxy in case we're talking to a bi-bir process controller. if self.adapter: - prx = processController.ice_getConnection().createProxy(prx.ice_getIdentity()) + prx = processController.ice_getConnection().createProxy( + prx.ice_getIdentity() + ) import Test - return RemoteProcessController.RemoteProcess(exe, Test.Common.ProcessPrx.uncheckedCast(prx)) + + return RemoteProcessController.RemoteProcess( + exe, Test.Common.ProcessPrx.uncheckedCast(prx) + ) def destroy(self, driver): if driver.controllerApp: @@ -2400,7 +2773,6 @@ def destroy(self, driver): class AndroidProcessController(RemoteProcessController): - def __init__(self, current): endpoint = None if current.config.device: @@ -2443,15 +2815,20 @@ def startEmulator(self, avd): port = p if port == -1: - raise RuntimeError("cannot find free port in range 5554-5584, to run android emulator") + raise RuntimeError( + "cannot find free port in range 5554-5584, to run android emulator" + ) self.device = "emulator-{}".format(port) cmd = "emulator -avd {0} -port {1} -no-audio -partition-size 768 -no-snapshot -gpu auto -no-boot-anim -no-window".format( - avd, port) + avd, port + ) self.emulator = subprocess.Popen(cmd, shell=True) if self.emulator.poll(): - raise RuntimeError("failed to start the Android emulator `{}' on port {}".format(avd, port)) + raise RuntimeError( + "failed to start the Android emulator `{}' on port {}".format(avd, port) + ) self.avd = avd @@ -2470,11 +2847,12 @@ def startEmulator(self, avd): # If the emulator doesn't complete boot in 300 seconds give up # if (time.time() - t) > 300: - raise RuntimeError("couldn't start the Android emulator `{}'".format(avd)) + raise RuntimeError( + "couldn't start the Android emulator `{}'".format(avd) + ) time.sleep(2) def startControllerApp(self, current, ident): - # Stop previous controller app before starting new one for ident in self.controllerApps: self.stopControllerApp(ident) @@ -2491,7 +2869,9 @@ def startControllerApp(self, current, ident): pass # The SDK is downloaded by test VMs instead of here. # run("sdkmanager \"{0}\"".format(sdk), stdout=True, stdin="yes", stdinRepeat=True) # yes to accept licenses - run("avdmanager -v create avd -k \"{0}\" -d \"Nexus 6\" -n IceTests".format(sdk)) + run( + 'avdmanager -v create avd -k "{0}" -d "Nexus 6" -n IceTests'.format(sdk) + ) self.startEmulator("IceTests") elif current.config.device != "usb": run("adb connect {}".format(current.config.device)) @@ -2501,9 +2881,16 @@ def startControllerApp(self, current, ident): run("{} shell pm uninstall com.zeroc.testcontroller".format(self.adb())) except: pass - run("{} install -t -r {}".format(self.adb(), current.testcase.getMapping().getApk(current))) - run("{} shell am start -n \"{}\" -a android.intent.action.MAIN -c android.intent.category.LAUNCHER".format( - self.adb(), current.testcase.getMapping().getActivityName())) + run( + "{} install -t -r {}".format( + self.adb(), current.testcase.getMapping().getApk(current) + ) + ) + run( + '{} shell am start -n "{}" -a android.intent.action.MAIN -c android.intent.category.LAUNCHER'.format( + self.adb(), current.testcase.getMapping().getActivityName() + ) + ) def stopControllerApp(self, ident): try: @@ -2519,7 +2906,9 @@ def stopControllerApp(self, ident): if self.avd == "IceTests": try: - run("avdmanager -v delete avd -n IceTests") # Delete the created device + run( + "avdmanager -v delete avd -n IceTests" + ) # Delete the created device except: pass @@ -2544,7 +2933,6 @@ def stopControllerApp(self, ident): class iOSSimulatorProcessController(RemoteProcessController): - device = "iOSSimulatorProcessController" deviceID = "com.apple.CoreSimulator.SimDeviceType.iPhone-13" @@ -2554,17 +2942,21 @@ def __init__(self, current): self.runtimeID = None # Pick the last iOS simulator runtime ID in the list of iOS simulators (assumed to be the latest). try: - for r in run("xcrun simctl list runtimes").split('\n'): + for r in run("xcrun simctl list runtimes").split("\n"): m = re.search("iOS .* \\(.*\\) - (.*)", r) if m: self.runtimeID = m.group(1) except: pass if not self.runtimeID: - self.runtimeID = "com.apple.CoreSimulator.SimRuntime.iOS-15-2" # Default value + self.runtimeID = ( + "com.apple.CoreSimulator.SimRuntime.iOS-15-2" # Default value + ) def __str__(self): - return "iOS Simulator ({})".format(self.runtimeID.replace("com.apple.CoreSimulator.SimRuntime.", "").strip()) + return "iOS Simulator ({})".format( + self.runtimeID.replace("com.apple.CoreSimulator.SimRuntime.", "").strip() + ) def getControllerIdentity(self, current): return current.testcase.getMapping().getIOSControllerIdentity(current) @@ -2576,19 +2968,28 @@ def startControllerApp(self, current, ident): sys.stdout.write("launching simulator... ") sys.stdout.flush() try: - run("xcrun simctl boot \"{0}\"".format(self.device)) - run("xcrun simctl bootstatus \"{0}\"".format(self.device)) # Wait for the boot to complete + run('xcrun simctl boot "{0}"'.format(self.device)) + run( + 'xcrun simctl bootstatus "{0}"'.format(self.device) + ) # Wait for the boot to complete except Exception as ex: if str(ex).find("Booted") >= 0: pass - elif str(ex).find("Invalid device") >= 0 or str(ex).find("Assertion failure in SimDevicePair"): + elif str(ex).find("Invalid device") >= 0 or str(ex).find( + "Assertion failure in SimDevicePair" + ): # # Create the simulator device if it doesn't exist # - self.simulatorID = run("xcrun simctl create \"{0}\" {1} {2}".format( - self.device, self.deviceID, self.runtimeID)) - run("xcrun simctl boot \"{0}\"".format(self.device)) - run("xcrun simctl bootstatus \"{0}\"".format(self.device)) # Wait for the boot to complete + self.simulatorID = run( + 'xcrun simctl create "{0}" {1} {2}'.format( + self.device, self.deviceID, self.runtimeID + ) + ) + run('xcrun simctl boot "{0}"'.format(self.device)) + run( + 'xcrun simctl bootstatus "{0}"'.format(self.device) + ) # Wait for the boot to complete # # This not longer works on iOS 15 simulator, fails with: # "Could not write domain com.apple.springboard; exiting" @@ -2609,29 +3010,31 @@ def startControllerApp(self, current, ident): sys.stdout.flush() if not os.path.exists(appFullPath): - raise RuntimeError("couldn't find iOS simulator controller application, did you build it?") - run("xcrun simctl install \"{0}\" \"{1}\"".format(self.device, appFullPath)) - run("xcrun simctl launch \"{0}\" {1}".format(self.device, ident.name)) + raise RuntimeError( + "couldn't find iOS simulator controller application, did you build it?" + ) + run('xcrun simctl install "{0}" "{1}"'.format(self.device, appFullPath)) + run('xcrun simctl launch "{0}" {1}'.format(self.device, ident.name)) print("ok") def restartControllerApp(self, current, ident): # We reboot the simulator if the controller fails to start. Terminating the controller app # with simctl terminate doesn't always work, it can hang if the controller app died because # of the springboard watchdog. - run("xcrun simctl shutdown \"{0}\"".format(self.device)) + run('xcrun simctl shutdown "{0}"'.format(self.device)) nRetry = 0 while nRetry < 20: try: - run("xcrun simctl boot \"{0}\"".format(self.device)) + run('xcrun simctl boot "{0}"'.format(self.device)) break except Exception: time.sleep(1.0) nRetry += 1 - run("xcrun simctl launch \"{0}\" {1}".format(self.device, ident.name)) + run('xcrun simctl launch "{0}" {1}'.format(self.device, ident.name)) def stopControllerApp(self, ident): try: - run("xcrun simctl uninstall \"{0}\" {1}".format(self.device, ident.name)) + run('xcrun simctl uninstall "{0}" {1}'.format(self.device, ident.name)) except: pass @@ -2641,7 +3044,7 @@ def destroy(self, driver): sys.stdout.write("shutting down simulator... ") sys.stdout.flush() try: - run("xcrun simctl shutdown \"{0}\"".format(self.simulatorID)) + run('xcrun simctl shutdown "{0}"'.format(self.simulatorID)) except: pass print("ok") @@ -2650,14 +3053,13 @@ def destroy(self, driver): sys.stdout.write("destroying simulator... ") sys.stdout.flush() try: - run("xcrun simctl delete \"{0}\"".format(self.simulatorID)) + run('xcrun simctl delete "{0}"'.format(self.simulatorID)) except: pass print("ok") class iOSDeviceProcessController(RemoteProcessController): - appPath = "cpp/test/ios/controller/build" def __init__(self, current): @@ -2678,10 +3080,11 @@ def stopControllerApp(self, ident): class BrowserProcessController(RemoteProcessController): - def __init__(self, current): self.host = current.driver.host or "127.0.0.1" - RemoteProcessController.__init__(self, current, "ws -h {0} -p 15002:wss -h {0} -p 15003".format(self.host)) + RemoteProcessController.__init__( + self, current, "ws -h {0} -p 15002:wss -h {0} -p 15003".format(self.host) + ) self.httpServer = None self.url = None self.driver = None @@ -2693,13 +3096,19 @@ def __init__(self, current): if current.config.browser.startswith("Remote:"): from selenium import webdriver - from selenium.webdriver.common.desired_capabilities import DesiredCapabilities + from selenium.webdriver.common.desired_capabilities import ( + DesiredCapabilities, + ) + (driver, capabilities, port) = current.config.browser.split(":") - self.driver = webdriver.Remote("http://localhost:{0}".format(port), - desired_capabilities=getattr(DesiredCapabilities, capabilities), - keep_alive=True) + self.driver = webdriver.Remote( + "http://localhost:{0}".format(port), + desired_capabilities=getattr(DesiredCapabilities, capabilities), + keep_alive=True, + ) elif current.config.browser != "Manual": from selenium import webdriver + if current.config.browser.find(":") > 0: (driver, port) = current.config.browser.split(":") else: @@ -2709,7 +3118,11 @@ def __init__(self, current): raise RuntimeError("unknown browser `{0}'".format(driver)) if driver == "Firefox": - if isinstance(platform, Linux) and os.environ.get("DISPLAY", "") != ":1" and os.environ.get("USER", "") == "ubuntu": + if ( + isinstance(platform, Linux) + and os.environ.get("DISPLAY", "") != ":1" + and os.environ.get("USER", "") == "ubuntu" + ): current.writeln("error: DISPLAY is unset, setting it to :1") os.environ["DISPLAY"] = ":1" @@ -2718,8 +3131,12 @@ def __init__(self, current): # contains our Test CA cert. It should be possible to avoid this by setting the webdriver # acceptInsecureCerts capability but it's only supported by latest Firefox releases. # - profilepath = os.path.join(current.driver.getComponent().getSourceDir(), - "scripts", "selenium", "firefox") + profilepath = os.path.join( + current.driver.getComponent().getSourceDir(), + "scripts", + "selenium", + "firefox", + ) options = webdriver.FirefoxOptions() options.set_preference("profile", profilepath) self.driver = webdriver.Firefox(options=options) @@ -2757,15 +3174,14 @@ def getControllerIdentity(self, current): protocol = "http" port = "8080" cport = "15002" - url = "{0}://{5}:{1}/test/{2}/controller.html?port={3}&worker={4}".format(protocol, - port, - testsuite, - cport, - current.config.worker, - self.host) + url = "{0}://{5}:{1}/test/{2}/controller.html?port={3}&worker={4}".format( + protocol, port, testsuite, cport, current.config.worker, self.host + ) if url != self.url: self.url = url - ident = current.driver.getCommunicator().stringToIdentity("Browser/ProcessController") + ident = current.driver.getCommunicator().stringToIdentity( + "Browser/ProcessController" + ) if self.driver: # Clear the previous controller connection if it exists. This ensures that the reload # of the test controller will use a new connection (with Chrome the connection close @@ -2785,12 +3201,17 @@ def getControllerIdentity(self, current): if ident in self.processControllerProxies: prx = self.processControllerProxies[ident] break - print("Please load http://{0}:8080/{1}".format(self.host, "start")) + print( + "Please load http://{0}:8080/{1}".format(self.host, "start") + ) self.cond.wait(5) try: import Test - Test.Common.BrowserProcessControllerPrx.uncheckedCast(prx).redirect(url) + + Test.Common.BrowserProcessControllerPrx.uncheckedCast(prx).redirect( + url + ) except: pass finally: @@ -2806,11 +3227,17 @@ def getController(self, current): # Print out the client & server console element values for element in ["clientConsole", "serverConsole"]: try: - console = self.driver.find_element_by_id(element).get_attribute('value') + console = self.driver.find_element_by_id(element).get_attribute( + "value" + ) if len(console) > 0: print("controller {0} value:\n{1}".format(element, console)) except Exception as exc: - print("couldn't get controller {0} value:\n{1}".format(element, exc)) + print( + "couldn't get controller {0} value:\n{1}".format( + element, exc + ) + ) pass # Print out the browser log try: @@ -2832,9 +3259,7 @@ def destroy(self, driver): class Driver: - class Current: - def __init__(self, driver, testsuite, result): self.driver = driver self.testsuite = testsuite @@ -2865,17 +3290,19 @@ def writeln(self, *args, **kargs): def push(self, testcase, host=None): if not testcase.mapping: - assert (not testcase.parent and not testcase.testsuite) + assert not testcase.parent and not testcase.testsuite testcase.mapping = self.testcase.getMapping() testcase.testsuite = self.testcase.getTestSuite() testcase.parent = self.testcase self.testcases.append((self.testcase, self.config, self.host)) self.testcase = testcase - self.config = self.driver.configs[self.testcase.getMapping()].cloneAndOverrideWith(self) + self.config = self.driver.configs[ + self.testcase.getMapping() + ].cloneAndOverrideWith(self) self.host = host def pop(self): - assert (self.testcase) + assert self.testcase testcase = self.testcase (self.testcase, self.config, self.host) = self.testcases.pop() if testcase.parent and self.testcase != testcase: @@ -2885,7 +3312,9 @@ def pop(self): def createFile(self, path, lines, encoding=None): path = os.path.join(self.testsuite.getPath(), path) - with open(path, "w", encoding=encoding) if encoding else open(path, "w") as file: + with open(path, "w", encoding=encoding) if encoding else open( + path, "w" + ) as file: for l in lines: file.write("%s\n" % l) self.files.append(path) @@ -2929,8 +3358,23 @@ def create(self, options, component): @classmethod def getSupportedArgs(self): - return ("dlrR", ["debug", "driver=", "filter=", "rfilter=", "host=", "host-ipv6=", "host-bt=", "interface=", - "controller-app", "valgrind", "languages=", "rlanguages="]) + return ( + "dlrR", + [ + "debug", + "driver=", + "filter=", + "rfilter=", + "host=", + "host-ipv6=", + "host-bt=", + "interface=", + "controller-app", + "valgrind", + "languages=", + "rlanguages=", + ], + ) @classmethod def usage(self): @@ -2941,15 +3385,27 @@ def commonUsage(self): print("") print("Driver options:") print("-d | --debug Verbose information.") - print("--driver= Use the given driver (local, client, server or remote).") + print( + "--driver= Use the given driver (local, client, server or remote)." + ) print("--filter= Run all the tests that match the given regex.") - print("--rfilter= Run all the tests that do not match the given regex.") - print("--languages=l1,l2,... List of comma-separated language mappings to test.") - print("--rlanguages=l1,l2,.. List of comma-separated language mappings to not test.") + print( + "--rfilter= Run all the tests that do not match the given regex." + ) + print( + "--languages=l1,l2,... List of comma-separated language mappings to test." + ) + print( + "--rlanguages=l1,l2,.. List of comma-separated language mappings to not test." + ) print("--host= The IPv4 address to use for Ice.Default.Host.") print("--host-ipv6= The IPv6 address to use for Ice.Default.Host.") - print("--host-bt= The Bluetooth address to use for Ice.Default.Host.") - print("--interface= The multicast interface to use to discover controllers.") + print( + "--host-bt= The Bluetooth address to use for Ice.Default.Host." + ) + print( + "--interface= The multicast interface to use to discover controllers." + ) print("--controller-app Start the process controller application.") print("--valgrind Start executables with valgrind.") @@ -2968,21 +3424,36 @@ def __init__(self, options, component): self.rlanguages = [] self.failures = [] - parseOptions(self, options, {"d": "debug", - "r": "filters", - "R": "rfilters", - "filter": "filters", - "rfilter": "rfilters", - "host-ipv6": "hostIPv6", - "host-bt": "hostBT", - "controller-app": "controllerApp"}) + parseOptions( + self, + options, + { + "d": "debug", + "r": "filters", + "R": "rfilters", + "filter": "filters", + "rfilter": "rfilters", + "host-ipv6": "hostIPv6", + "host-bt": "hostBT", + "controller-app": "controllerApp", + }, + ) if self.languages: - self.languages = [i for sublist in [l.split(",") for l in self.languages] for i in sublist] + self.languages = [ + i for sublist in [l.split(",") for l in self.languages] for i in sublist + ] if self.rlanguages: - self.rlanguages = [i for sublist in [l.split(",") for l in self.rlanguages] for i in sublist] + self.rlanguages = [ + i + for sublist in [l.split(",") for l in self.rlanguages] + for i in sublist + ] - (self.filters, self.rfilters) = ([re.compile(a) for a in self.filters], [re.compile(a) for a in self.rfilters]) + (self.filters, self.rfilters) = ( + [re.compile(a) for a in self.filters], + [re.compile(a) for a in self.rfilters], + ) self.communicator = None self.interface = "" @@ -2994,7 +3465,10 @@ def setConfigs(self, configs): def getFilters(self, mapping, config): # Return the driver and component filters (filters, rfilters) = self.component.getFilters(mapping, config) - (filters, rfilters) = ([re.compile(a) for a in filters], [re.compile(a) for a in rfilters]) + (filters, rfilters) = ( + [re.compile(a) for a in filters], + [re.compile(a) for a in rfilters], + ) return (self.filters + filters, self.rfilters + rfilters) def getHost(self, protocol, ipv6): @@ -3027,7 +3501,9 @@ def getProps(self, process, current): props = {} if isinstance(process, IceProcess): if not self.host: - props["Ice.Default.Host"] = "0:0:0:0:0:0:0:1" if current.config.ipv6 else "127.0.0.1" + props["Ice.Default.Host"] = ( + "0:0:0:0:0:0:0:1" if current.config.ipv6 else "127.0.0.1" + ) else: props["Ice.Default.Host"] = self.host return props @@ -3058,27 +3534,38 @@ def initCommunicator(self): try: pythonMapping = Mapping.getByName("python") if pythonMapping: - for p in pythonMapping.getPythonDirs(pythonMapping.getPath(), self.configs[pythonMapping]): + for p in pythonMapping.getPythonDirs( + pythonMapping.getPath(), self.configs[pythonMapping] + ): sys.path.append(p) except RuntimeError: - print("couldn't find IcePy, running these tests require it to be installed or built") + print( + "couldn't find IcePy, running these tests require it to be installed or built" + ) import Ice - Ice.loadSlice(os.path.join(self.component.getSourceDir(), "scripts", "Controller.ice")) + + Ice.loadSlice( + os.path.join(self.component.getSourceDir(), "scripts", "Controller.ice") + ) initData = Ice.InitializationData() initData.properties = Ice.createProperties() # Load IceSSL, this is useful to talk with WSS for JavaScript initData.properties.setProperty("Ice.Plugin.IceSSL", "IceSSL:createIceSSL") - initData.properties.setProperty("IceSSL.DefaultDir", os.path.join(self.component.getSourceDir(), "certs")) + initData.properties.setProperty( + "IceSSL.DefaultDir", os.path.join(self.component.getSourceDir(), "certs") + ) initData.properties.setProperty("IceSSL.CertFile", "server.p12") initData.properties.setProperty("IceSSL.Password", "password") initData.properties.setProperty("IceSSL.Keychain", "test.keychain") initData.properties.setProperty("IceSSL.KeychainPassword", "password") initData.properties.setProperty("IceSSL.VerifyPeer", "0") - initData.properties.setProperty("Ice.Plugin.IceDiscovery", "IceDiscovery:createIceDiscovery") + initData.properties.setProperty( + "Ice.Plugin.IceDiscovery", "IceDiscovery:createIceDiscovery" + ) initData.properties.setProperty("IceDiscovery.DomainId", "TestController") initData.properties.setProperty("IceDiscovery.Interface", self.interface) initData.properties.setProperty("Ice.Default.Host", self.interface) @@ -3096,7 +3583,11 @@ def getProcessController(self, current, process=None): processController = iOSSimulatorProcessController elif current.config.buildPlatform == "iphoneos": processController = iOSDeviceProcessController - elif process and current.config.browser and isinstance(process.getMapping(current), JavaScriptMixin): + elif ( + process + and current.config.browser + and isinstance(process.getMapping(current), JavaScriptMixin) + ): processController = BrowserProcessController elif process and current.config.android: processController = AndroidProcessController @@ -3126,9 +3617,7 @@ def destroy(self): class CppMapping(Mapping): - class Config(Mapping.Config): - @classmethod def getSupportedArgs(self): return ("", ["cpp-config=", "cpp-platform=", "cpp-path=", "openssl"]) @@ -3137,10 +3626,18 @@ def getSupportedArgs(self): def usage(self): print("") print("C++ Mapping options:") - print("--cpp-path= Path of alternate source tree for the C++ mapping.") - print("--cpp-config= C++ build configuration for native executables (overrides --config).") - print("--cpp-platform= C++ build platform for native executables (overrides --platform).") - print("--openssl Run SSL tests with OpenSSL instead of the default platform SSL engine.") + print( + "--cpp-path= Path of alternate source tree for the C++ mapping." + ) + print( + "--cpp-config= C++ build configuration for native executables (overrides --config)." + ) + print( + "--cpp-platform= C++ build platform for native executables (overrides --platform)." + ) + print( + "--openssl Run SSL tests with OpenSSL instead of the default platform SSL engine." + ) def __init__(self, options=[]): Mapping.Config.__init__(self, options) @@ -3155,9 +3652,15 @@ def __init__(self, options=[]): # tests on the cpp11 value in the testcase options specification self.cpp11 = self.buildConfig.lower().find("cpp11") >= 0 - parseOptions(self, options, {"cpp-config": "buildConfig", - "cpp-platform": "buildPlatform", - "cpp-path": "pathOverride"}) + parseOptions( + self, + options, + { + "cpp-config": "buildConfig", + "cpp-platform": "buildPlatform", + "cpp-path": "pathOverride", + }, + ) if self.pathOverride: self.pathOverride = os.path.abspath(self.pathOverride) @@ -3173,23 +3676,31 @@ def getSSLProps(self, process, current): props = Mapping.getSSLProps(self, process, current) server = isinstance(process, Server) - props.update({ - "IceSSL.CAs": "cacert.pem", - "IceSSL.CertFile": "server.p12" if server else "client.p12" - }) + props.update( + { + "IceSSL.CAs": "cacert.pem", + "IceSSL.CertFile": "server.p12" if server else "client.p12", + } + ) if isinstance(platform, Darwin): - props.update({ - "IceSSL.KeychainPassword": "password", - "IceSSL.Keychain": "server.keychain" if server else "client.keychain" - }) + props.update( + { + "IceSSL.KeychainPassword": "password", + "IceSSL.Keychain": "server.keychain" + if server + else "client.keychain", + } + ) return props def getPluginEntryPoint(self, plugin, process, current): return { - "IceSSL": "IceSSLOpenSSL:createIceSSLOpenSSL" if current.config.openssl else "IceSSL:createIceSSL", + "IceSSL": "IceSSLOpenSSL:createIceSSLOpenSSL" + if current.config.openssl + else "IceSSL:createIceSSL", "IceBT": "IceBT:createIceBT", "IceDiscovery": "IceDiscovery:createIceDiscovery", - "IceLocatorDiscovery": "IceLocatorDiscovery:createIceLocatorDiscovery" + "IceLocatorDiscovery": "IceLocatorDiscovery:createIceLocatorDiscovery", }[plugin] def getEnv(self, process, current): @@ -3200,7 +3711,9 @@ def getEnv(self, process, current): if isinstance(platform, Windows): testcommon = os.path.join(self.path, "test", "Common") if os.path.exists(testcommon): - libPaths.append(os.path.join(testcommon, self.getBuildDir("testcommon", current))) + libPaths.append( + os.path.join(testcommon, self.getBuildDir("testcommon", current)) + ) # # On most platforms, we also need to add the library directory to the library path environment variable. @@ -3211,13 +3724,23 @@ def getEnv(self, process, current): # On AIX we also need to add the lib directory for the TestCommon library # when testing against a binary distribution if isinstance(platform, AIX) and self.component.useBinDist(self, current): - libPaths.append(os.path.join(self.path, "lib32" if current.config.buildPlatform == "ppc" else "lib")) + libPaths.append( + os.path.join( + self.path, + "lib32" if current.config.buildPlatform == "ppc" else "lib", + ) + ) # # Add the test suite library directories to the platform library path environment variable. # if current.testcase: - for d in set([current.getBuildDir(d) for d in current.testcase.getTestSuite().getLibDirs()]): + for d in set( + [ + current.getBuildDir(d) + for d in current.testcase.getTestSuite().getLibDirs() + ] + ): libPaths.append(d) env = {} @@ -3239,23 +3762,41 @@ def _getDefaultExe(self, processType): return Mapping._getDefaultExe(self, processType).lower() def getIOSControllerIdentity(self, current): - category = "iPhoneSimulator" if current.config.buildPlatform == "iphonesimulator" else "iPhoneOS" + category = ( + "iPhoneSimulator" + if current.config.buildPlatform == "iphonesimulator" + else "iPhoneOS" + ) mapping = "Cpp11" if current.config.cpp11 else "Cpp98" return "{0}/com.zeroc.{1}-Test-Controller".format(category, mapping) def getIOSAppFullPath(self, current): - appName = "C++11 Test Controller.app" if current.config.cpp11 else "C++98 Test Controller.app" + appName = ( + "C++11 Test Controller.app" + if current.config.cpp11 + else "C++98 Test Controller.app" + ) path = os.path.join(self.component.getTestDir(self), "ios", "controller") - path = os.path.join(path, "build-{0}-{1}".format(current.config.buildPlatform, current.config.buildConfig)) - build = "Debug" if os.path.exists(os.path.join( - path, "Debug-{0}".format(current.config.buildPlatform))) else "Release" - return os.path.join(path, "{0}-{1}".format(build, current.config.buildPlatform), appName) + path = os.path.join( + path, + "build-{0}-{1}".format( + current.config.buildPlatform, current.config.buildConfig + ), + ) + build = ( + "Debug" + if os.path.exists( + os.path.join(path, "Debug-{0}".format(current.config.buildPlatform)) + ) + else "Release" + ) + return os.path.join( + path, "{0}-{1}".format(build, current.config.buildPlatform), appName + ) class JavaMapping(Mapping): - class Config(Mapping.Config): - @classmethod def getSupportedArgs(self): return ("", ["device=", "avd=", "android"]) @@ -3265,7 +3806,9 @@ def usage(self): print("") print("Java Mapping options:") print("--android Run the Android tests.") - print("--device= ID of the Android emulator or device used to run the tests.") + print( + "--device= ID of the Android emulator or device used to run the tests." + ) print("--avd= Start specific Android Virtual Device.") def getCommandLine(self, current, process, exe, args): @@ -3279,33 +3822,50 @@ def getCommandLine(self, current, process, exe, args): return "{0} -ea {1} {2}".format(java, exe, args) testdir = self.component.getTestDir(self) - assert (current.testcase.getPath(current).startswith(testdir)) - package = "test." + current.testcase.getPath(current)[len(testdir) + 1:].replace(os.sep, ".") + assert current.testcase.getPath(current).startswith(testdir) + package = "test." + current.testcase.getPath(current)[ + len(testdir) + 1 : + ].replace(os.sep, ".") javaArgs = self.getJavaArgs(process, current) if javaArgs: - return "{0} -ea {1} -Dtest.class={2}.{3} test.TestDriver {4}".format(java, " ".join(javaArgs), package, exe, args) + return "{0} -ea {1} -Dtest.class={2}.{3} test.TestDriver {4}".format( + java, " ".join(javaArgs), package, exe, args + ) else: - return "{0} -ea -Dtest.class={1}.{2} test.TestDriver {3}".format(java, package, exe, args) + return "{0} -ea -Dtest.class={1}.{2} test.TestDriver {3}".format( + java, package, exe, args + ) def getJavaArgs(self, process, current): # TODO: WORKAROUND for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=911925 - if isinstance(platform, Linux) and platform.getLinuxId() in ["debian", "ubuntu"]: + if isinstance(platform, Linux) and platform.getLinuxId() in [ + "debian", + "ubuntu", + ]: return ["-Djdk.net.URLClassPath.disableClassPathURLCheck=true"] return [] def getSSLProps(self, process, current): props = Mapping.getSSLProps(self, process, current) if current.config.android: - props.update({ - "IceSSL.KeystoreType": "BKS", - "IceSSL.TruststoreType": "BKS", - "Ice.InitPlugins": "0", - "IceSSL.Keystore": "server.bks" if isinstance(process, Server) else "client.bks" - }) + props.update( + { + "IceSSL.KeystoreType": "BKS", + "IceSSL.TruststoreType": "BKS", + "Ice.InitPlugins": "0", + "IceSSL.Keystore": "server.bks" + if isinstance(process, Server) + else "client.bks", + } + ) else: - props.update({ - "IceSSL.Keystore": "server.jks" if isinstance(process, Server) else "client.jks", - }) + props.update( + { + "IceSSL.Keystore": "server.jks" + if isinstance(process, Server) + else "client.jks", + } + ) return props def getPluginEntryPoint(self, plugin, process, current): @@ -3313,7 +3873,7 @@ def getPluginEntryPoint(self, plugin, process, current): "IceSSL": "com.zeroc.IceSSL.PluginFactory", "IceBT": "com.zeroc.IceBT.PluginFactory", "IceDiscovery": "com.zeroc.IceDiscovery.PluginFactory", - "IceLocatorDiscovery": "com.zeroc.IceLocatorDiscovery.PluginFactory" + "IceLocatorDiscovery": "com.zeroc.IceLocatorDiscovery.PluginFactory", }[plugin] def getEnv(self, process, current): @@ -3329,20 +3889,28 @@ def _getDefaultSource(self, processType): def getSDKPackage(self): return "system-images;android-33;google_apis;{}".format( - "arm64-v8a" if platform_machine() == "arm64" else "x86_64") + "arm64-v8a" if platform_machine() == "arm64" else "x86_64" + ) def getApk(self, current): - return os.path.join(self.getPath(), "test", "android", "controller", "build", "outputs", "apk", "debug", - "controller-debug.apk") + return os.path.join( + self.getPath(), + "test", + "android", + "controller", + "build", + "outputs", + "apk", + "debug", + "controller-debug.apk", + ) def getActivityName(self): return "com.zeroc.testcontroller/.ControllerActivity" class CSharpMapping(Mapping): - class Config(Mapping.Config): - @classmethod def getSupportedArgs(self): return ("", ["framework="]) @@ -3351,7 +3919,9 @@ def getSupportedArgs(self): def usage(self): print("") print("C# mapping options:") - print("--framework=net48|net6.0|net8.0 Choose the framework used to run .NET tests") + print( + "--framework=net48|net6.0|net8.0 Choose the framework used to run .NET tests" + ) def __init__(self, options=[]): Mapping.Config.__init__(self, options) @@ -3367,13 +3937,19 @@ def getBuildDir(self, name, current): def getSSLProps(self, process, current): props = Mapping.getSSLProps(self, process, current) - props.update({ - "IceSSL.Password": "password", - "IceSSL.DefaultDir": os.path.join(self.component.getSourceDir(), "certs"), - "IceSSL.CAs": "cacert.pem", - "IceSSL.VerifyPeer": "0" if current.config.protocol == "wss" else "2", - "IceSSL.CertFile": "server.p12" if isinstance(process, Server) else "client.p12", - }) + props.update( + { + "IceSSL.Password": "password", + "IceSSL.DefaultDir": os.path.join( + self.component.getSourceDir(), "certs" + ), + "IceSSL.CAs": "cacert.pem", + "IceSSL.VerifyPeer": "0" if current.config.protocol == "wss" else "2", + "IceSSL.CertFile": "server.p12" + if isinstance(process, Server) + else "client.p12", + } + ) return props def getPluginEntryPoint(self, plugin, process, current): @@ -3386,7 +3962,10 @@ def getPluginEntryPoint(self, plugin, process, current): # proccessType = current.testcase.getProcessType(process) if proccessType: - testdir = os.path.join(current.testcase.getPath(current), self.getBuildDir(proccessType, current)) + testdir = os.path.join( + current.testcase.getPath(current), + self.getBuildDir(proccessType, current), + ) if os.path.isfile(os.path.join(testdir, plugin + ".dll")): plugindir = testdir plugindir += os.sep @@ -3394,18 +3973,28 @@ def getPluginEntryPoint(self, plugin, process, current): return { "IceSSL": plugindir + "IceSSL.dll:IceSSL.PluginFactory", "IceDiscovery": plugindir + "IceDiscovery.dll:IceDiscovery.PluginFactory", - "IceLocatorDiscovery": plugindir + "IceLocatorDiscovery.dll:IceLocatorDiscovery.PluginFactory" + "IceLocatorDiscovery": plugindir + + "IceLocatorDiscovery.dll:IceLocatorDiscovery.PluginFactory", }[plugin] def getEnv(self, process, current): env = {} if isinstance(platform, Windows): if self.component.useBinDist(self, current): - env['PATH'] = self.component.getBinDir(process, self, current) + env["PATH"] = self.component.getBinDir(process, self, current) else: - env['PATH'] = os.path.join(self.component.getSourceDir(), "cpp", "msbuild", "packages", - "bzip2.{0}.1.0.6.10".format(platform.getPlatformToolset()), - "build", "native", "bin", "x64", "Release") + env["PATH"] = os.path.join( + self.component.getSourceDir(), + "cpp", + "msbuild", + "packages", + "bzip2.{0}.1.0.6.10".format(platform.getPlatformToolset()), + "build", + "native", + "bin", + "x64", + "Release", + ) return env def _getDefaultSource(self, processType): @@ -3425,33 +4014,53 @@ def getCommandLine(self, current, process, exe, args): if process.isFromBinDir(): path = self.component.getBinDir(process, self, current) else: - path = os.path.join(current.testcase.getPath(current), current.getBuildDir(exe)) + path = os.path.join( + current.testcase.getPath(current), current.getBuildDir(exe) + ) return "dotnet {}.dll {}".format(os.path.join(path, exe), args) class CppBasedMapping(Mapping): - class Config(Mapping.Config): - @classmethod def getSupportedArgs(self): - return ("", [self.mappingName + "-config=", self.mappingName + "-platform=", "openssl"]) + return ( + "", + [ + self.mappingName + "-config=", + self.mappingName + "-platform=", + "openssl", + ], + ) @classmethod def usage(self): print("") print(self.mappingDesc + " mapping options:") - print("--{0}-config= {1} build configuration for native executables (overrides --config)." - .format(self.mappingName, self.mappingDesc)) - print("--{0}-platform= {1} build platform for native executables (overrides --platform)." - .format(self.mappingName, self.mappingDesc)) - print("--openssl Run SSL tests with OpenSSL instead of the default platform SSL engine.") + print( + "--{0}-config= {1} build configuration for native executables (overrides --config).".format( + self.mappingName, self.mappingDesc + ) + ) + print( + "--{0}-platform= {1} build platform for native executables (overrides --platform).".format( + self.mappingName, self.mappingDesc + ) + ) + print( + "--openssl Run SSL tests with OpenSSL instead of the default platform SSL engine." + ) def __init__(self, options=[]): Mapping.Config.__init__(self, options) - parseOptions(self, options, - {self.mappingName + "-config": "buildConfig", - self.mappingName + "-platform": "buildPlatform"}) + parseOptions( + self, + options, + { + self.mappingName + "-config": "buildConfig", + self.mappingName + "-platform": "buildPlatform", + }, + ) def getSSLProps(self, process, current): return Mapping.getByName("cpp").getSSLProps(process, current) @@ -3464,12 +4073,13 @@ def getEnv(self, process, current): if self.component.getInstallDir(self, current) != platform.getInstallDir(): # If not installed in the default platform installation directory, add # the C++ library directory to the library path - env[platform.getLdPathEnvName()] = self.component.getLibDir(process, Mapping.getByName("cpp"), current) + env[platform.getLdPathEnvName()] = self.component.getLibDir( + process, Mapping.getByName("cpp"), current + ) return env class PythonMapping(CppBasedMapping): - class Config(CppBasedMapping.Config): mappingName = "python" mappingDesc = "Python" @@ -3482,7 +4092,9 @@ def getSupportedArgs(self): def usage(self): print("") print("Python mapping options:") - print("--python= Choose the interperter used to run python tests") + print( + "--python= Choose the interperter used to run python tests" + ) def __init__(self, options=[]): Mapping.Config.__init__(self, options) @@ -3491,19 +4103,24 @@ def __init__(self, options=[]): def getPythonVersion(self): if self.pythonVersion is None: version = subprocess.check_output( - [currentConfig.python, - "-c", - "import sys; print(\"{0}.{1}\".format(sys.version_info[0], sys.version_info[1]))"]) + [ + currentConfig.python, + "-c", + 'import sys; print("{0}.{1}".format(sys.version_info[0], sys.version_info[1]))', + ] + ) if type(version) != str: version = version.decode("utf-8") self.pythonVersion = tuple(int(num) for num in version.split(".")) return self.pythonVersion def getCommandLine(self, current, process, exe, args): - return "\"{0}\" {1} {2} {3}".format(current.config.python, - os.path.join(self.path, "test", "TestHelper.py"), - exe, - args) + return '"{0}" {1} {2} {3}'.format( + current.config.python, + os.path.join(self.path, "test", "TestHelper.py"), + exe, + args, + ) def getEnv(self, process, current): env = CppBasedMapping.getEnv(self, process, current) @@ -3511,7 +4128,9 @@ def getEnv(self, process, current): if self.component.getInstallDir(self, current) != platform.getInstallDir(): # If not installed in the default platform installation directory, add # the Ice python directory to PYTHONPATH - dirs += self.getPythonDirs(self.component.getInstallDir(self, current), current.config) + dirs += self.getPythonDirs( + self.component.getInstallDir(self, current), current.config + ) dirs += [current.testcase.getPath(current)] env["PYTHONPATH"] = os.pathsep.join(dirs) return env @@ -3519,7 +4138,9 @@ def getEnv(self, process, current): def getPythonDirs(self, iceDir, config): dirs = [] if isinstance(platform, Windows): - dirs.append(os.path.join(iceDir, "python", config.buildPlatform, config.buildConfig)) + dirs.append( + os.path.join(iceDir, "python", config.buildPlatform, config.buildConfig) + ) dirs.append(os.path.join(iceDir, "python")) return dirs @@ -3533,23 +4154,25 @@ def _getDefaultSource(self, processType): class CppBasedClientMapping(CppBasedMapping): - def loadTestSuites(self, tests, config, filters, rfilters): Mapping.loadTestSuites(self, tests, config, filters, rfilters) self.getServerMapping().loadTestSuites(self.testsuites.keys(), config) def getServerMapping(self, testId=None): - return Mapping.getByName("cpp") # By default, run clients against C++ mapping executables + return Mapping.getByName( + "cpp" + ) # By default, run clients against C++ mapping executables class RubyMapping(CppBasedClientMapping): - class Config(CppBasedClientMapping.Config): mappingName = "ruby" mappingDesc = "Ruby" def getCommandLine(self, current, process, exe, args): - return "ruby {0} {1} {2}".format(os.path.join(self.path, "test", "TestHelper.rb"), exe, args) + return "ruby {0} {1} {2}".format( + os.path.join(self.path, "test", "TestHelper.rb"), exe, args + ) def getEnv(self, process, current): env = CppBasedMapping.getEnv(self, process, current) @@ -3567,7 +4190,6 @@ def _getDefaultSource(self, processType): class PhpMapping(CppBasedClientMapping): - class Config(CppBasedClientMapping.Config): mappingName = "php" mappingDesc = "PHP" @@ -3580,7 +4202,9 @@ def getSupportedArgs(self): def usage(self): print("") print("PHP Mapping options:") - print("--php-version=[7.1|7.2|7.3|8.0|8.1] PHP Version used for Windows builds") + print( + "--php-version=[7.1|7.2|7.3|8.0|8.1] PHP Version used for Windows builds" + ) def __init__(self, options=[]): CppBasedClientMapping.Config.__init__(self, options) @@ -3594,49 +4218,87 @@ def getCommandLine(self, current, process, exe, args): # On Windows, when using a source distribution use the php executable from # the Nuget PHP dependency. # - if isinstance(platform, Windows) and not self.component.useBinDist(self, current): + if isinstance(platform, Windows) and not self.component.useBinDist( + self, current + ): nugetVersions = { "7.1": "7.1.17", "7.2": "7.2.8", "7.3": "7.3.0", "8.0": "8.0.0.1", - "8.1": "8.1.0" + "8.1": "8.1.0", } nugetVersion = nugetVersions[current.config.phpVersion] - threadSafe = current.driver.configs[self].buildConfig in ["Debug", "Release"] + threadSafe = current.driver.configs[self].buildConfig in [ + "Debug", + "Release", + ] buildPlatform = current.driver.configs[self].buildPlatform - buildConfig = "Debug" if current.driver.configs[self].buildConfig.find("Debug") >= 0 else "Release" - packageName = "php-{0}-{1}.{2}".format(current.config.phpVersion, - "ts" if threadSafe else "nts", nugetVersion) - php = os.path.join(self.path, "msbuild", "packages", packageName, "build", "native", "bin", - buildPlatform, buildConfig, "php.exe") + buildConfig = ( + "Debug" + if current.driver.configs[self].buildConfig.find("Debug") >= 0 + else "Release" + ) + packageName = "php-{0}-{1}.{2}".format( + current.config.phpVersion, "ts" if threadSafe else "nts", nugetVersion + ) + php = os.path.join( + self.path, + "msbuild", + "packages", + packageName, + "build", + "native", + "bin", + buildPlatform, + buildConfig, + "php.exe", + ) # # If Ice is not installed in the system directory, specify its location with PHP # configuration arguments. # - if isinstance(platform, Windows) and not self.component.useBinDist(self, current) or \ - platform.getInstallDir() and self.component.getInstallDir(self, current) != platform.getInstallDir(): + if ( + isinstance(platform, Windows) + and not self.component.useBinDist(self, current) + or platform.getInstallDir() + and self.component.getInstallDir(self, current) != platform.getInstallDir() + ): phpArgs += ["-n"] # Do not load any php.ini files - phpArgs += ["-d", "extension_dir='{0}'".format(self.component.getLibDir(process, self, current))] - phpArgs += ["-d", "extension='{0}'".format(self.component.getPhpExtension(self, current))] - phpArgs += ["-d", "include_path='{0}'".format(self.component.getPhpIncludePath(self, current))] + phpArgs += [ + "-d", + "extension_dir='{0}'".format( + self.component.getLibDir(process, self, current) + ), + ] + phpArgs += [ + "-d", + "extension='{0}'".format(self.component.getPhpExtension(self, current)), + ] + phpArgs += [ + "-d", + "include_path='{0}'".format( + self.component.getPhpIncludePath(self, current) + ), + ] if hasattr(process, "getPhpArgs"): phpArgs += process.getPhpArgs(current) - return "{0} {1} -f {2} -- {3} {4}".format(php, - " ".join(phpArgs), - os.path.join(self.path, "test", "TestHelper.php"), - exe, - args) + return "{0} {1} -f {2} -- {3} {4}".format( + php, + " ".join(phpArgs), + os.path.join(self.path, "test", "TestHelper.php"), + exe, + args, + ) def _getDefaultSource(self, processType): return {"client": "Client.php"}[processType] class MatlabMapping(CppBasedClientMapping): - class Config(CppBasedClientMapping.Config): mappingName = "matlab" mappingDesc = "MATLAB" @@ -3644,13 +4306,22 @@ class Config(CppBasedClientMapping.Config): def getCommandLine(self, current, process, exe, args): matlabHome = os.getenv("MATLAB_HOME") # -wait and -minimize are not available on Linux, -log causes duplicate output to stdout on Linux - return "{0} -nodesktop -nosplash{1} -r \"cd '{2}', runTest {3} {4} {5}\"".format( - "matlab" if matlabHome is None else os.path.join(matlabHome, "bin", "matlab"), - " -wait -log -minimize" if isinstance(platform, Windows) else "", - os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "matlab", "test", "lib")), - self.getTestCwd(process, current), - current.driver.getComponent().getLibDir(process, self, current), - args) + return ( + "{0} -nodesktop -nosplash{1} -r \"cd '{2}', runTest {3} {4} {5}\"".format( + "matlab" + if matlabHome is None + else os.path.join(matlabHome, "bin", "matlab"), + " -wait -log -minimize" if isinstance(platform, Windows) else "", + os.path.abspath( + os.path.join( + os.path.dirname(__file__), "..", "matlab", "test", "lib" + ) + ), + self.getTestCwd(process, current), + current.driver.getComponent().getLibDir(process, self, current), + args, + ) + ) def getServerMapping(self, testId=None): return Mapping.getByName("python") # Run clients against Python mapping servers @@ -3667,21 +4338,24 @@ def getOptions(self, current): return options -class JavaScriptMixin(): - +class JavaScriptMixin: def loadTestSuites(self, tests, config, filters, rfilters): # Exclude typescript directory when the mapping is not typescript otherwise we endup with duplicate entries if self.name != "typescript": rfilters += [re.compile("typescript/*")] Mapping.loadTestSuites(self, tests, config, filters, rfilters) - self.getServerMapping().loadTestSuites(list(self.testsuites.keys()) + ["Ice/echo"], config) + self.getServerMapping().loadTestSuites( + list(self.testsuites.keys()) + ["Ice/echo"], config + ) def getServerMapping(self, testId=None): if testId and self.hasSource(testId, "server"): return self else: - return Mapping.getByName("cpp") # Run clients against C++ mapping servers if no JS server provided + return Mapping.getByName( + "cpp" + ) # Run clients against C++ mapping servers if no JS server provided def _getDefaultProcesses(self, processType): if processType.startswith("server"): @@ -3696,7 +4370,9 @@ def getCommandLine(self, current, process, exe, args): def getEnv(self, process, current): env = Mapping.getEnv(self, process, current) - env["NODE_PATH"] = os.pathsep.join([self.getCommonDir(current), self.getTestCwd(process, current)]) + env["NODE_PATH"] = os.pathsep.join( + [self.getCommonDir(current), self.getTestCwd(process, current)] + ) return env def getSSLProps(self, process, current): @@ -3714,9 +4390,7 @@ def getOptions(self, current): class JavaScriptMapping(JavaScriptMixin, Mapping): - class Config(Mapping.Config): - @classmethod def getSupportedArgs(self): return ("", ["browser=", "worker"]) @@ -3738,23 +4412,27 @@ def getCommonDir(self, current): return os.path.join(self.getPath(), "test", "Common") def _getDefaultSource(self, processType): - return {"client": "Client.js", "serveramd": "ServerAMD.js", "server": "Server.js"}[processType] + return { + "client": "Client.js", + "serveramd": "ServerAMD.js", + "server": "Server.js", + }[processType] def getTestCwd(self, process, current): return os.path.join(self.path, "test", current.testcase.getTestSuite().getId()) def getOptions(self, current): options = JavaScriptMixin.getOptions(self, current) - options.update({ - "worker": [False, True] if current.config.browser else [False], - }) + options.update( + { + "worker": [False, True] if current.config.browser else [False], + } + ) return options class TypeScriptMapping(JavaScriptMixin, Mapping): - class Config(Mapping.Config): - @classmethod def getSupportedArgs(self): return ("", ["browser=", "worker"]) @@ -3777,13 +4455,15 @@ def canRun(self, testId, current): return Mapping.Config.canRun(self, testId, current) and not self.browser def _getDefaultSource(self, processType): - return {"client": "Client.ts", "serveramd": "ServerAMD.ts", "server": "Server.ts"}[processType] + return { + "client": "Client.ts", + "serveramd": "ServerAMD.ts", + "server": "Server.ts", + }[processType] class SwiftMapping(Mapping): - class Config(CppBasedClientMapping.Config): - mappingName = "swift" mappingDesc = "Swift" @@ -3791,40 +4471,51 @@ def __init__(self, options=[]): CppBasedClientMapping.Config.__init__(self, options) if self.buildConfig == platform.getDefaultBuildConfig(): # Check the OPTIMIZE environment variable to figure out if it's Debug/Release build - self.buildConfig = "Release" if os.environ.get("OPTIMIZE", "yes") != "no" else "Debug" + self.buildConfig = ( + "Release" if os.environ.get("OPTIMIZE", "yes") != "no" else "Debug" + ) def getCommandLine(self, current, process, exe, args): testdir = self.component.getTestDir(self) - assert (current.testcase.getPath(current).startswith(testdir)) - package = current.testcase.getPath(current)[len(testdir) + 1:].replace(os.sep, ".") + assert current.testcase.getPath(current).startswith(testdir) + package = current.testcase.getPath(current)[len(testdir) + 1 :].replace( + os.sep, "." + ) cmd = "xcodebuild -project {0} -target 'TestDriver {1}' -configuration {2} -showBuildSettings".format( - self.getXcodeProject(current), - "macOS", - current.config.buildConfig) + self.getXcodeProject(current), "macOS", current.config.buildConfig + ) targetBuildDir = re.search(r"\sTARGET_BUILD_DIR = (.*)", run(cmd)).groups(1)[0] - testDriver = os.path.join(targetBuildDir, "TestDriver.app/Contents/MacOS/TestDriver") + testDriver = os.path.join( + targetBuildDir, "TestDriver.app/Contents/MacOS/TestDriver" + ) if not os.path.exists(testDriver): # Fallback location, required with Xcode 14.2 testDriver = os.path.join( current.testcase.getMapping().getPath(), "build", current.config.buildConfig, - "TestDriver.app/Contents/MacOS/TestDriver") + "TestDriver.app/Contents/MacOS/TestDriver", + ) return "{0} {1} {2} {3}".format(testDriver, package, exe, args) def _getDefaultSource(self, processType): - return {"client": "Client.swift", - "server": "Server.swift", - "serveramd": "ServerAMD.swift", - "collocated": "Collocated.swift" - }[processType] + return { + "client": "Client.swift", + "server": "Server.swift", + "serveramd": "ServerAMD.swift", + "collocated": "Collocated.swift", + }[processType] def getIOSControllerIdentity(self, current): - category = "iPhoneSimulator" if current.config.buildPlatform == "iphonesimulator" else "iPhoneOS" + category = ( + "iPhoneSimulator" + if current.config.buildPlatform == "iphonesimulator" + else "iPhoneOS" + ) return "{0}/com.zeroc.Swift-Test-Controller".format(category) def getIOSAppFullPath(self, current): @@ -3832,9 +4523,11 @@ def getIOSAppFullPath(self, current): -target 'TestDriver iOS' \ -configuration {1} \ -showBuildSettings \ - -sdk {2}".format(self.getXcodeProject(current), - current.config.buildConfig, - current.config.buildPlatform) + -sdk {2}".format( + self.getXcodeProject(current), + current.config.buildConfig, + current.config.buildPlatform, + ) targetBuildDir = re.search("\sTARGET_BUILD_DIR = (.*)", run(cmd)).groups(1)[0] testDriver = os.path.join(targetBuildDir, "TestDriver.app") @@ -3843,22 +4536,30 @@ def getIOSAppFullPath(self, current): testDriver = os.path.join( current.testcase.getMapping().getPath(), "build", - "{0}-{1}".format(current.config.buildConfig, current.config.buildPlatform), - "TestDriver.app") + "{0}-{1}".format( + current.config.buildConfig, current.config.buildPlatform + ), + "TestDriver.app", + ) return testDriver def getSSLProps(self, process, current): props = Mapping.getByName("cpp").getSSLProps(process, current) - props["IceSSL.DefaultDir"] = ("certs" if current.config.buildPlatform == "iphoneos" else - os.path.join(self.component.getSourceDir(), "certs")) + props["IceSSL.DefaultDir"] = ( + "certs" + if current.config.buildPlatform == "iphoneos" + else os.path.join(self.component.getSourceDir(), "certs") + ) return props def getPluginEntryPoint(self, plugin, process, current): return Mapping.getByName("cpp").getPluginEntryPoint(plugin, process, current) def getXcodeProject(self, current): - return "{0}/{1}".format(current.testcase.getMapping().getPath(), - "ice.xcodeproj") + return "{0}/{1}".format( + current.testcase.getMapping().getPath(), "ice.xcodeproj" + ) + # TODO ice-test.xcodeproj once Carthage supports binary XCFramework projects # "ice-test.xcodeproj" if self.component.useBinDist(self, current) else "ice.xcodeproj") @@ -3898,7 +4599,11 @@ def getXcodeProject(self, current): def runTestsWithPath(path): mappings = Mapping.getAllByPath(path) if not mappings: - print("couldn't find mapping for `{0}' (is this mapping supported on this platform?)".format(path)) + print( + "couldn't find mapping for `{0}' (is this mapping supported on this platform?)".format( + path + ) + ) sys.exit(0) runTests(mappings) @@ -3929,7 +4634,10 @@ def usage(): try: options = [Driver.getSupportedArgs(), Mapping.Config.getSupportedArgs()] options += [driver.getSupportedArgs() for driver in drivers] - options += [mapping.Config.getSupportedArgs() for mapping in Mapping.getAll(includeDisabled=True)] + options += [ + mapping.Config.getSupportedArgs() + for mapping in Mapping.getAll(includeDisabled=True) + ] shortOptions = "h" longOptions = ["help"] for so, lo in options: @@ -3937,7 +4645,7 @@ def usage(): longOptions += lo opts, args = getopt.gnu_getopt(sys.argv[1:], shortOptions, longOptions) - for (o, a) in opts: + for o, a in opts: if o in ["-h", "--help"]: usage() sys.exit(0) diff --git a/scripts/icehashpassword.py b/scripts/icehashpassword.py index 4fb95079f69..e7978fdaf28 100755 --- a/scripts/icehashpassword.py +++ b/scripts/icehashpassword.py @@ -10,7 +10,9 @@ import getpass usePBKDF2 = any(sys.platform == p for p in ["win32", "darwin", "cygwin"]) -useCryptExt = any(sys.platform.startswith(p) for p in ["linux", "freebsd", "gnukfreebsd"]) +useCryptExt = any( + sys.platform.startswith(p) for p in ["linux", "freebsd", "gnukfreebsd"] +) def usage(): @@ -21,14 +23,18 @@ def usage(): if usePBKDF2: print("") print(" -d MESSAGE_DIGEST_ALGORITHM, --digest=MESSAGE_DIGEST_ALGORITHM") - print(" The message digest algorithm to use with PBKDF2, valid values are (sha1, sha256, sha512).") + print( + " The message digest algorithm to use with PBKDF2, valid values are (sha1, sha256, sha512)." + ) print("") print(" -s SALT_SIZE, --salt=SALT_SIZE") print(" Optional number of bytes to use when generating new salts.") print("") elif useCryptExt: print(" -d MESSAGE_DIGEST_ALGORITHM, --digest=MESSAGE_DIGEST_ALGORITHM") - print(" The message digest algorithm to use with crypt function, valid values are (sha256, sha512).") + print( + " The message digest algorithm to use with crypt function, valid values are (sha256, sha512)." + ) print("") if usePBKDF2 or useCryptExt: print(" -r ROUNDS, --rounds=ROUNDS") @@ -40,7 +46,6 @@ def usage(): def main(): - digestAlgorithms = () shortArgs = "h" longArgs = ["help"] @@ -109,14 +114,18 @@ def main(): if rounds: if not passScheme.min_rounds <= rounds <= passScheme.max_rounds: - print("Invalid number rounds for the digest algorithm. Value must be an integer between %s and %s" % - (passScheme.min_rounds, passScheme.max_rounds)) + print( + "Invalid number rounds for the digest algorithm. Value must be an integer between %s and %s" + % (passScheme.min_rounds, passScheme.max_rounds) + ) usage() return 2 if salt: if not passScheme.min_salt_size <= salt <= passScheme.max_salt_size: - print("Invalid salt size for the digest algorithm. Value must be an integer between %s and %s" % - (passScheme.min_salt_size, passScheme.max_salt_size)) + print( + "Invalid salt size for the digest algorithm. Value must be an integer between %s and %s" + % (passScheme.min_salt_size, passScheme.max_salt_size) + ) usage() return 2 @@ -142,5 +151,5 @@ def main(): return 0 -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/swift/allTests.py b/swift/allTests.py index 94a9466c3db..58b0a12520c 100755 --- a/swift/allTests.py +++ b/swift/allTests.py @@ -5,6 +5,7 @@ import os import sys + sys.path.append(os.path.join(os.path.dirname(__file__), "..", "scripts")) from Util import runTestsWithPath