Merge pull request #12 from meltwater/default-marathon-url

Added support for setting a default Marathon URL for an environment

README.md

@@ -20,14 +20,17 @@ usage: lighter COMMAND [OPTIONS]...
 Marathon deployment tool
 
 positional arguments:
-  {deploy,verify}  Available commands
-    deploy         Deploy services to Marathon
-    verify         Verify and generate Marathon configuration files
+  {deploy,verify}       Available commands
+    deploy              Deploy services to Marathon
+    verify              Verify and generate Marathon configuration files
 
 optional arguments:
-  -h, --help       show this help message and exit
-  -n, --noop       Execute dry-run without modifying Marathon [default: False]
-  -v, --verbose    Increase logging verbosity [default: False]
+  -h, --help            show this help message and exit
+  -n, --noop            Execute dry-run without modifying Marathon [default:
+                        False]
+  -v, --verbose         Increase logging verbosity [default: False]
+  -t TARGETDIR, --targetdir TARGETDIR
+                        Directory to output rendered config files
 ```
 
 ### Deploy Command
@@ -42,7 +45,9 @@ positional arguments:
 optional arguments:
   -h, --help            show this help message and exit
   -m MARATHON, --marathon MARATHON
-                        Marathon url, e.g. "http://marathon-host:8080/"
+                        Marathon URL like "http://marathon-host:8080/".
+                        Overrides default Marathon URL's provided in config
+                        files
   -f, --force           Force deployment even if the service is already
                         affected by a running deployment [default: False]
 ```
@@ -64,13 +69,23 @@ my-config-repo/
            myservice.yml
 ```
 
-Running `lighter deploy -m http://marathon-host:8080 staging/services/myservice.yml` will
+Running `lighter deploy staging/services/myservice.yml` will
 
 * Merge *myservice.yml* with environment defaults from *my-config-repo/staging/globals.yml* and *my-config-repo/globals.yml*
 * Fetch the *json* template for this service and version from the Maven repository
 * Expand the *json* template with variables and overrides from the *yml* files
 * Post the resulting *json* configuration into Marathon
 
+## Marathon
+Yaml files may contain a `marathon:` section with a default URL to reach Marathon at. The `-m/--marathon`
+parameter will override this setting when given on the command-line.
+
+*globals.yml*
+```
+marathon:
+  url: 'http://marathon-host:8080/'
+```
+
 ## Maven
 The `maven:` section specifies where to fetch *json* templates from which are 
 merged into the configuration. For example
@@ -249,10 +264,10 @@ Use the script like
 cd my-config-repo
 
 # Deploy/sync all services (from Jenkins or other CI/CD server)
-./lighter deploy -f -m http://marathon-host:8080 $(find staging -name \*.yml -not -name globals.yml)
+./lighter deploy $(find staging -name \*.yml -not -name globals.yml)
 
 # Deploy single services
-./lighter deploy -m http://marathon-host:8080 staging/myservice.yml staging/myservice2.yml
+./lighter deploy staging/myservice.yml staging/myservice2.yml
 ```
 
 ## Integrations

src/lighter/main.py

@@ -164,12 +164,16 @@ def get_marathon_app(url):
         return {}
 
 def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None):
-    parsedMarathonUrl = urlparse(marathonurl)
     services = parse_services(filenames, targetdir)
 
     for service in services:
         try:
-            appurl = get_marathon_url(marathonurl, service.config['id'], force)
+            targetMarathonUrl = marathonurl or util.rget(service.document, 'marathon', 'url')
+            if not targetMarathonUrl:
+                raise RuntimeError("No Marathon URL defined for service %s" % service.filename)
+
+            parsedMarathonUrl = urlparse(targetMarathonUrl)
+            appurl = get_marathon_url(targetMarathonUrl, service.config['id'], force)
 
             # See if service config has changed
             prevConfig = get_marathon_app(appurl)
@@ -203,14 +207,11 @@ def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None):
             # Send Datadog deployment notification
             datadog = Datadog(util.rget(service.document, 'datadog', 'token'))
             datadog.notify(
-                title="Deployed %s to the %s environment" % (service.id, service.environment),
-                message="%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%" % (service.id, service.image, service.environment, parsedMarathonUrl.netloc),
                 id=service.id,
-                tags=[
-                    "environment:%s" % service.environment,
-                    "service:%s" % service.id
-                ]
-            )
+                title="Deployed %s to the %s environment" % (service.id, service.environment),
+                message="%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%" % (
+                    service.id, service.image, service.environment, parsedMarathonUrl.netloc),
+                tags=["environment:%s" % service.environment, "service:%s" % service.id])
 
         except urllib2.HTTPError, e:
             raise RuntimeError("Failed to deploy %s HTTP %d (%s)" % (service.filename, e.code, e)), None, sys.exc_info()[2]
@@ -241,24 +242,24 @@ def verify(filenames, targetdir=None, verifySecrets=False):
         help='Deploy services to Marathon',
         description='Deploy services to Marathon')
 
-    deploy_parser.add_argument('-m', '--marathon', required=True, dest='marathon', help='Marathon url, e.g. "http://marathon-host:8080/"',
+    deploy_parser.add_argument('-m', '--marathon', required=True, dest='marathon', help='Marathon URL like "http://marathon-host:8080/". Overrides default Marathon URL\'s provided in config files',
                       default=os.environ.get('MARATHON_URL', ''))
     deploy_parser.add_argument('-f', '--force', dest='force', help='Force deployment even if the service is already affected by a running deployment [default: %(default)s]',
                       action='store_true', default=False)
     deploy_parser.add_argument('filenames', metavar='YMLFILE', nargs='+',
                        help='Service files to expand and deploy')
 
     # Create the parser for the "verify" command
-    deploy_parser = subparsers.add_parser('verify', 
+    verify_parser = subparsers.add_parser('verify', 
         prog='lighter',
         usage='%(prog)s verify YMLFILE...',
         help='Verify and generate Marathon configuration files',
         description='Verify and generate Marathon configuration files')
 
-    deploy_parser.add_argument('filenames', metavar='YMLFILE', nargs='+',
+    verify_parser.add_argument('filenames', metavar='YMLFILE', nargs='+',
                        help='Service files to expand and deploy')
 
-    deploy_parser.add_argument('--verify-secrets', dest='verifySecrets', help='Fail verification if unencrypted secrets are found [default: %(default)s]',
+    verify_parser.add_argument('--verify-secrets', dest='verifySecrets', help='Fail verification if unencrypted secrets are found [default: %(default)s]',
                       action='store_true', default=False)
 
     args = parser.parse_args()

src/lighter/test/deploy_test.py

@@ -4,6 +4,9 @@
 from lighter.util import jsonRequest
 
 class DeployTest(unittest.TestCase):
+    def setUp(self):
+        self._called = False
+
     def testParseService(self):
         service = lighter.parse_service('src/resources/yaml/staging/myservice.yml')
         self.assertEquals(service.document['hipchat']['token'], 'abc123')
@@ -53,20 +56,37 @@ def testParseError(self):
             with self.assertRaises(RuntimeError):
                 lighter.deploy('http://localhost:1/', filenames=['src/resources/yaml/staging/myservice.yml', 'src/resources/yaml/staging/myservice-broken.yml'])
 
+    def _createJsonRequestWrapper(self, marathonurl='http://localhost:1'):
+        appurl = '%s/v2/apps/myproduct/myservice' % marathonurl
 
-    def _resolvePost(self, url, data=None, *args, **kwargs):
-        if url.startswith('file:'):
-            return jsonRequest(url, data, *args, **kwargs)
-        if '/v2/apps' in url and data:
-            self.assertEquals(data['container']['docker']['image'], 'meltwater/myservice:1.0.0')
-            self._resolvePostCalled = True
-        return {'app': {}}
-
-    def testResolve(self):
-        with patch('lighter.util.jsonRequest', wraps=self._resolvePost) as mock_jsonRequest:
+        def wrapper(url, method='GET', data=None, *args, **kwargs):
+            if url.startswith('file:'):
+                return jsonRequest(url, data, *args, **kwargs)
+            if url == appurl and method == 'PUT' and data:
+                self.assertEquals(data['container']['docker']['image'], 'meltwater/myservice:1.0.0')
+                self._called = True
+                return {}
+            if url == appurl and method == 'GET':
+                return {'app': {}}
+            return None
+        return wrapper
+
+    def testResolveMavenJson(self):
+        with patch('lighter.util.jsonRequest', wraps=self._createJsonRequestWrapper()) as mock_jsonRequest:
             lighter.deploy('http://localhost:1/', filenames=['src/resources/yaml/integration/myservice.yml'])
-            self.assertTrue(self._resolvePostCalled)
+            self.assertTrue(self._called)
+
+    def testDefaultMarathonUrl(self):
+        with patch('lighter.util.jsonRequest', wraps=self._createJsonRequestWrapper('http://defaultmarathon:2')) as mock_jsonRequest:
+            lighter.deploy(marathonurl=None, filenames=['src/resources/yaml/integration/myservice.yml'])
+            self.assertTrue(self._called)
 
+    def testNoMarathonUrlDefined(self):
+        with patch('lighter.util.jsonRequest', wraps=self._createJsonRequestWrapper()) as mock_jsonRequest:
+            with self.assertRaises(RuntimeError) as cm:
+                lighter.deploy(marathonurl=None, filenames=['src/resources/yaml/staging/myservice.yml'])
+            self.assertEqual("No Marathon URL defined for service src/resources/yaml/staging/myservice.yml", cm.exception.message)
+
     def testUnresolvedVariable(self):
         service_yaml = 'src/resources/yaml/integration/myservice-unresolved-variable.yml'
         try:

src/resources/yaml/integration/globals.yml

@@ -0,0 +1,2 @@
+marathon:
+  url: 'http://defaultmarathon:2/'