From fe13b90ca592791bf11a5eb786a1ccd99eae9685 Mon Sep 17 00:00:00 2001 From: Per Berglund Date: Thu, 17 Nov 2016 16:31:46 +0100 Subject: [PATCH] Added profile option (#50) * Added --profile option to deploy and verify --- README.md | 8 ++++-- src/lighter/main.py | 41 ++++++++++++++++++++++--------- src/lighter/test/deploy_test.py | 9 +++++-- src/lighter/test/maven_test.py | 3 +++ src/resources/yaml/globals.yml | 6 ----- src/resources/yaml/myprofile1.yml | 4 +++ src/resources/yaml/myprofile2.yml | 2 ++ 7 files changed, 52 insertions(+), 21 deletions(-) create mode 100644 src/resources/yaml/myprofile1.yml create mode 100644 src/resources/yaml/myprofile2.yml diff --git a/README.md b/README.md index db5b223..9109ac3 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,9 @@ optional arguments: -v, --verbose Increase logging verbosity [default: False] -t TARGETDIR, --targetdir TARGETDIR Directory to output rendered config files + -p PROFILES, --profile PROFILES + Extra profile file(s) to be merged with service + definitions. ``` ### Deploy Command @@ -58,6 +61,7 @@ Given a directory structure like ``` config-repo/ | globals.yml +| myprofile.yml └─ production/ | | globals.yml | | myfrontend.yml @@ -70,9 +74,9 @@ config-repo/ | myfrontend.yml ``` -Running `lighter deploy staging/myfrontend.yml` will +Running `lighter deploy -p myprofile1.yml -p myprofile2.yml staging/myfrontend.yml` will -* Merge *myfrontend.yml* with environment defaults from *config-repo/staging/globals.yml* and *config-repo/globals.yml* +* Merge *myfrontend.yml* with environment defaults from *config-repo/staging/globals.yml*, *config-repo/globals.yml*, *myprofile1.yml* and *myprofile2.yml* * Fetch the *json* template for this service and version from the Maven repository * Expand the *json* template with variables and overrides from the *yml* files * Post the resulting *json* configuration into Marathon diff --git a/src/lighter/main.py b/src/lighter/main.py index a96ad87..03b46ce 100755 --- a/src/lighter/main.py +++ b/src/lighter/main.py @@ -95,8 +95,9 @@ def process_env(filename, verifySecrets, env): return result -def parse_service(filename, targetdir=None, verifySecrets=False): +def parse_service(filename, targetdir=None, verifySecrets=False, profiles=[]): logging.info("Processing %s", filename) + # Start from a service section if it exists with open(filename, 'r') as fd: try: document = yaml.load(fd) @@ -108,11 +109,12 @@ def parse_service(filename, targetdir=None, verifySecrets=False): while '/' in path: candidate = os.path.join(path, 'globals.yml') if os.path.exists(candidate): - with open(candidate, 'r') as fd2: - document = util.merge(yaml.load(fd2), document) + document = merge_with_service(candidate, document) path = path[0:path.rindex('/')] - # Start from a service section if it exists + # Merge profile .yml files into document + document = merge_with_profiles(document, profiles) + variables = util.FixedVariables(document.get('variables', {})) # Environment variables has higher precedence @@ -121,7 +123,6 @@ def parse_service(filename, targetdir=None, verifySecrets=False): # Replace variables in entire document document = util.replace(document, variables, raiseError=False, escapeVar=False) - # Start from a service section if it exists config = document.get('service', {}) # Allow resolving version/uniqueVersion variables from docker registry @@ -183,9 +184,25 @@ def parse_service(filename, targetdir=None, verifySecrets=False): return Service(filename, document, config) -def parse_services(filenames, targetdir=None, verifySecrets=False): + +def merge_with_profiles(document, profiles): + for profile in profiles: + document = merge_with_service(profile, document) + return document + + +def merge_with_service(override_file, document): + if not os.path.exists(override_file): + raise RuntimeError('Could not read file %s' % override_file) + + with open(override_file, 'r') as fd2: + document = util.merge(yaml.load(fd2), document) + return document + + +def parse_services(filenames, targetdir=None, verifySecrets=False, profiles=[]): # return [parse_service(filename, targetdir) for filename in filenames] - return Parallel(n_jobs=8, backend="threading")(delayed(parse_service)(filename, targetdir, verifySecrets) for filename in filenames) + return Parallel(n_jobs=8, backend="threading")(delayed(parse_service)(filename, targetdir, verifySecrets, profiles) for filename in filenames) def get_marathon_url(url, id, force=False): return url.rstrip('/') + '/v2/apps/' + id.strip('/') + (force and '?force=true' or '') @@ -250,7 +267,7 @@ def notify(targetMarathonUrl, service): service.id, service.image, service.environment, parsedMarathonUrl.netloc), tags=tags) -def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None): +def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None, profiles=[]): services = parse_services(filenames, targetdir) for service in services: @@ -281,8 +298,9 @@ def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None): except urllib2.URLError as e: raise RuntimeError("Failed to deploy %s (%s)" % (service.filename, e)), None, sys.exc_info()[2] -def verify(filenames, targetdir=None, verifySecrets=False): - parse_services(filenames, targetdir, verifySecrets) +def verify(filenames, targetdir=None, verifySecrets=False, profiles=[]): + parse_services(filenames, targetdir, verifySecrets, profiles) + if __name__ == '__main__': parser = argparse.ArgumentParser( @@ -297,6 +315,7 @@ def verify(filenames, targetdir=None, verifySecrets=False): action="store_true", default=False) parser.add_argument('-t', '--targetdir', dest='targetdir', help='Directory to output rendered config files', default=None) + parser.add_argument('-p', '--profile', dest='profiles', default=[], action='append', help='Extra profile files to be merged with service definitions.') # Create the parser for the "deploy" command deploy_parser = subparsers.add_parser('deploy', @@ -339,7 +358,7 @@ def verify(filenames, targetdir=None, verifySecrets=False): try: if args.command == 'deploy': - deploy(args.marathon, noop=args.noop, force=args.force, filenames=args.filenames, targetdir=args.targetdir) + deploy(args.marathon, noop=args.noop, force=args.force, filenames=args.filenames, targetdir=args.targetdir, profiles=args.profiles) elif args.command == 'verify': verify(args.filenames, targetdir=args.targetdir, verifySecrets=args.verifySecrets) except RuntimeError as e: diff --git a/src/lighter/test/deploy_test.py b/src/lighter/test/deploy_test.py index 9d94826..020f401 100644 --- a/src/lighter/test/deploy_test.py +++ b/src/lighter/test/deploy_test.py @@ -5,12 +5,17 @@ import lighter.main as lighter from lighter.util import jsonRequest +PROFILE_2 = 'src/resources/yaml/myprofile2.yml' + +PROFILE_1 = 'src/resources/yaml/myprofile1.yml' + + class DeployTest(unittest.TestCase): def setUp(self): self._called = False def testParseService(self): - service = lighter.parse_service('src/resources/yaml/staging/myservice.yml') + service = lighter.parse_service('src/resources/yaml/staging/myservice.yml', profiles=[PROFILE_1, PROFILE_2]) self.assertEquals(service.document['hipchat']['token'], 'abc123') self.assertEquals(sorted(service.document['hipchat']['rooms']), ['123', '456', '456', '789']) self.assertEquals(service.environment, 'staging') @@ -172,7 +177,7 @@ def testNonStringEnvkey(self): self.fail('Expected ValueError') def testParseNoMavenService(self): - service = lighter.parse_service('src/resources/yaml/staging/myservice-nomaven.yml') + service = lighter.parse_service('src/resources/yaml/staging/myservice-nomaven.yml', profiles=[PROFILE_1, PROFILE_2]) self.assertEquals(service.document['hipchat']['token'], 'abc123') self.assertEquals(service.config['id'], '/myproduct/myservice-nomaven') self.assertEquals(service.config['instances'], 1) diff --git a/src/lighter/test/maven_test.py b/src/lighter/test/maven_test.py index 8a8b6e8..64494e5 100644 --- a/src/lighter/test/maven_test.py +++ b/src/lighter/test/maven_test.py @@ -12,6 +12,7 @@ def testResolveSpecificVersion(self): def testSelectVersionInclusive(self): resolver = maven.ArtifactResolver('file:./src/resources/repository/', 'com.meltwater', 'myservice') + def select(expression, versions): return resolver.selectVersion(expression, versions) @@ -22,6 +23,7 @@ def select(expression, versions): def testSelectVersionExclusive(self): resolver = maven.ArtifactResolver('file:./src/resources/repository/', 'com.meltwater', 'myservice') + def select(expression, versions): return resolver.selectVersion(expression, versions) @@ -39,6 +41,7 @@ def select(expression, versions): def testSelectVersionLatest(self): resolver = maven.ArtifactResolver('file:./src/resources/repository/', 'com.meltwater', 'myservice') + def select(expression, versions): return resolver.selectVersion(expression, versions) diff --git a/src/resources/yaml/globals.yml b/src/resources/yaml/globals.yml index 600fe02..da39827 100644 --- a/src/resources/yaml/globals.yml +++ b/src/resources/yaml/globals.yml @@ -1,13 +1,7 @@ maven: repository: 'file:./src/resources/repository/' -hipchat: - token: 'abc123' - rooms: - - '123' variables: rabbitmq.url: amqp://localhost:5672 -newrelic: - token: 'i_am_an_api_token' docker: registries: 'authregistrywithport:5000': diff --git a/src/resources/yaml/myprofile1.yml b/src/resources/yaml/myprofile1.yml new file mode 100644 index 0000000..f4e1dce --- /dev/null +++ b/src/resources/yaml/myprofile1.yml @@ -0,0 +1,4 @@ +hipchat: + token: 'abc123' + rooms: + - '123' \ No newline at end of file diff --git a/src/resources/yaml/myprofile2.yml b/src/resources/yaml/myprofile2.yml new file mode 100644 index 0000000..2caadbe --- /dev/null +++ b/src/resources/yaml/myprofile2.yml @@ -0,0 +1,2 @@ +newrelic: + token: 'i_am_an_api_token' \ No newline at end of file