Added profile option (#50)

* Added --profile option to deploy and verify

README.md

@@ -31,6 +31,9 @@ optional arguments:
   -v, --verbose         Increase logging verbosity [default: False]
   -t TARGETDIR, --targetdir TARGETDIR
                         Directory to output rendered config files
+  -p PROFILES, --profile PROFILES
+                        Extra profile file(s) to be merged with service
+                        definitions.
 ```
 
 ### Deploy Command
@@ -58,6 +61,7 @@ Given a directory structure like
 ```
 config-repo/
 |   globals.yml
+|   myprofile.yml 
 └─ production/
 |   |   globals.yml
 |   |   myfrontend.yml
@@ -70,9 +74,9 @@ config-repo/
     |   myfrontend.yml
 ```
 
-Running `lighter deploy staging/myfrontend.yml` will
+Running `lighter deploy -p myprofile1.yml -p myprofile2.yml staging/myfrontend.yml` will
 
-* Merge *myfrontend.yml* with environment defaults from *config-repo/staging/globals.yml* and *config-repo/globals.yml*
+* Merge *myfrontend.yml* with environment defaults from *config-repo/staging/globals.yml*, *config-repo/globals.yml*, *myprofile1.yml* and *myprofile2.yml*
 * Fetch the *json* template for this service and version from the Maven repository
 * Expand the *json* template with variables and overrides from the *yml* files
 * Post the resulting *json* configuration into Marathon

src/lighter/main.py

@@ -95,8 +95,9 @@ def process_env(filename, verifySecrets, env):
 
     return result
 
-def parse_service(filename, targetdir=None, verifySecrets=False):
+def parse_service(filename, targetdir=None, verifySecrets=False, profiles=[]):
     logging.info("Processing %s", filename)
+    # Start from a service section if it exists
     with open(filename, 'r') as fd:
         try:
             document = yaml.load(fd)
@@ -108,11 +109,12 @@ def parse_service(filename, targetdir=None, verifySecrets=False):
     while '/' in path:
         candidate = os.path.join(path, 'globals.yml')
         if os.path.exists(candidate):
-            with open(candidate, 'r') as fd2:
-                document = util.merge(yaml.load(fd2), document)
+            document = merge_with_service(candidate, document)
         path = path[0:path.rindex('/')]
 
-    # Start from a service section if it exists
+    # Merge profile .yml files into document
+    document = merge_with_profiles(document, profiles)
+
     variables = util.FixedVariables(document.get('variables', {}))
 
     # Environment variables has higher precedence
@@ -121,7 +123,6 @@ def parse_service(filename, targetdir=None, verifySecrets=False):
     # Replace variables in entire document
     document = util.replace(document, variables, raiseError=False, escapeVar=False)
 
-    # Start from a service section if it exists
     config = document.get('service', {})
 
     # Allow resolving version/uniqueVersion variables from docker registry
@@ -183,9 +184,25 @@ def parse_service(filename, targetdir=None, verifySecrets=False):
 
     return Service(filename, document, config)
 
-def parse_services(filenames, targetdir=None, verifySecrets=False):
+
+def merge_with_profiles(document, profiles):
+    for profile in profiles:
+        document = merge_with_service(profile, document)
+    return document
+
+
+def merge_with_service(override_file, document):
+    if not os.path.exists(override_file):
+        raise RuntimeError('Could not read file %s' % override_file)
+
+    with open(override_file, 'r') as fd2:
+        document = util.merge(yaml.load(fd2), document)
+    return document
+
+
+def parse_services(filenames, targetdir=None, verifySecrets=False, profiles=[]):
     # return [parse_service(filename, targetdir) for filename in filenames]
-    return Parallel(n_jobs=8, backend="threading")(delayed(parse_service)(filename, targetdir, verifySecrets) for filename in filenames)
+    return Parallel(n_jobs=8, backend="threading")(delayed(parse_service)(filename, targetdir, verifySecrets, profiles) for filename in filenames)
 
 def get_marathon_url(url, id, force=False):
     return url.rstrip('/') + '/v2/apps/' + id.strip('/') + (force and '?force=true' or '')
@@ -250,7 +267,7 @@ def notify(targetMarathonUrl, service):
             service.id, service.image, service.environment, parsedMarathonUrl.netloc),
         tags=tags)
 
-def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None):
+def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None, profiles=[]):
     services = parse_services(filenames, targetdir)
 
     for service in services:
@@ -281,8 +298,9 @@ def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None):
         except urllib2.URLError as e:
             raise RuntimeError("Failed to deploy %s (%s)" % (service.filename, e)), None, sys.exc_info()[2]
 
-def verify(filenames, targetdir=None, verifySecrets=False):
-    parse_services(filenames, targetdir, verifySecrets)
+def verify(filenames, targetdir=None, verifySecrets=False, profiles=[]):
+    parse_services(filenames, targetdir, verifySecrets, profiles)
+
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
@@ -297,6 +315,7 @@ def verify(filenames, targetdir=None, verifySecrets=False):
                         action="store_true", default=False)
     parser.add_argument('-t', '--targetdir', dest='targetdir', help='Directory to output rendered config files',
                         default=None)
+    parser.add_argument('-p', '--profile', dest='profiles', default=[], action='append', help='Extra profile files to be merged with service definitions.')
 
     # Create the parser for the "deploy" command
     deploy_parser = subparsers.add_parser('deploy',
@@ -339,7 +358,7 @@ def verify(filenames, targetdir=None, verifySecrets=False):
 
     try:
         if args.command == 'deploy':
-            deploy(args.marathon, noop=args.noop, force=args.force, filenames=args.filenames, targetdir=args.targetdir)
+            deploy(args.marathon, noop=args.noop, force=args.force, filenames=args.filenames, targetdir=args.targetdir, profiles=args.profiles)
         elif args.command == 'verify':
             verify(args.filenames, targetdir=args.targetdir, verifySecrets=args.verifySecrets)
     except RuntimeError as e:

src/lighter/test/deploy_test.py

@@ -5,12 +5,17 @@
 import lighter.main as lighter
 from lighter.util import jsonRequest
 
+PROFILE_2 = 'src/resources/yaml/myprofile2.yml'
+
+PROFILE_1 = 'src/resources/yaml/myprofile1.yml'
+
+
 class DeployTest(unittest.TestCase):
     def setUp(self):
         self._called = False
 
     def testParseService(self):
-        service = lighter.parse_service('src/resources/yaml/staging/myservice.yml')
+        service = lighter.parse_service('src/resources/yaml/staging/myservice.yml', profiles=[PROFILE_1, PROFILE_2])
         self.assertEquals(service.document['hipchat']['token'], 'abc123')
         self.assertEquals(sorted(service.document['hipchat']['rooms']), ['123', '456', '456', '789'])
         self.assertEquals(service.environment, 'staging')
@@ -172,7 +177,7 @@ def testNonStringEnvkey(self):
             self.fail('Expected ValueError')
 
     def testParseNoMavenService(self):
-        service = lighter.parse_service('src/resources/yaml/staging/myservice-nomaven.yml')
+        service = lighter.parse_service('src/resources/yaml/staging/myservice-nomaven.yml', profiles=[PROFILE_1, PROFILE_2])
         self.assertEquals(service.document['hipchat']['token'], 'abc123')
         self.assertEquals(service.config['id'], '/myproduct/myservice-nomaven')
         self.assertEquals(service.config['instances'], 1)

src/lighter/test/maven_test.py

@@ -12,6 +12,7 @@ def testResolveSpecificVersion(self):
 
     def testSelectVersionInclusive(self):
         resolver = maven.ArtifactResolver('file:./src/resources/repository/', 'com.meltwater', 'myservice')
+
         def select(expression, versions):
             return resolver.selectVersion(expression, versions)
 
@@ -22,6 +23,7 @@ def select(expression, versions):
 
     def testSelectVersionExclusive(self):
         resolver = maven.ArtifactResolver('file:./src/resources/repository/', 'com.meltwater', 'myservice')
+
         def select(expression, versions):
             return resolver.selectVersion(expression, versions)
 
@@ -39,6 +41,7 @@ def select(expression, versions):
 
     def testSelectVersionLatest(self):
         resolver = maven.ArtifactResolver('file:./src/resources/repository/', 'com.meltwater', 'myservice')
+
         def select(expression, versions):
             return resolver.selectVersion(expression, versions)
 

src/resources/yaml/globals.yml

@@ -1,13 +1,7 @@
 maven:
   repository: 'file:./src/resources/repository/'
-hipchat:
-  token: 'abc123'
-  rooms:
-    - '123'
 variables:
   rabbitmq.url: amqp://localhost:5672
-newrelic:
-  token: 'i_am_an_api_token'
 docker:
   registries:
     'authregistrywithport:5000':

src/resources/yaml/myprofile1.yml

@@ -0,0 +1,4 @@
+hipchat:
+  token: 'abc123'
+  rooms:
+    - '123'

src/resources/yaml/myprofile2.yml

@@ -0,0 +1,2 @@
+newrelic:
+  token: 'i_am_an_api_token'