From 68e1afd299cbe3a897b297583120ecf6f5276725 Mon Sep 17 00:00:00 2001 From: Mikael Johansson Date: Thu, 10 Dec 2015 21:38:27 +0100 Subject: [PATCH 1/2] Added support for setting a default Marathon URL for an environment --- README.md | 9 +++++ src/lighter/main.py | 19 +++++----- src/lighter/test/deploy_test.py | 42 ++++++++++++++++------ src/resources/yaml/integration/globals.yml | 2 ++ 4 files changed, 52 insertions(+), 20 deletions(-) create mode 100644 src/resources/yaml/integration/globals.yml diff --git a/README.md b/README.md index 5616099..a80ccca 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,15 @@ facts: environment: 'staging' ``` +## Marathon Defaults +Yaml files may contain a `marathon:` section with a default URL to reach Marathon at + +*globals.yml* +``` +marathon: + url: 'http://marathon-host:8080/' +``` + ## Installation Place a `lighter` script in the root of your configuration repo. Replace the LIGHTER_VERSION with a version from the [releases page](https://github.com/meltwater/lighter/releases). diff --git a/src/lighter/main.py b/src/lighter/main.py index 23e9720..6275b52 100755 --- a/src/lighter/main.py +++ b/src/lighter/main.py @@ -164,12 +164,16 @@ def get_marathon_app(url): return {} def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None): - parsedMarathonUrl = urlparse(marathonurl) services = parse_services(filenames, targetdir) for service in services: try: - appurl = get_marathon_url(marathonurl, service.config['id'], force) + targetMarathonUrl = marathonurl or util.rget(service.document, 'marathon', 'url') + if not targetMarathonUrl: + raise RuntimeError("No Marathon URL defined for service %s" % service.filename) + + parsedMarathonUrl = urlparse(targetMarathonUrl) + appurl = get_marathon_url(targetMarathonUrl, service.config['id'], force) # See if service config has changed prevConfig = get_marathon_app(appurl) @@ -203,14 +207,11 @@ def deploy(marathonurl, filenames, noop=False, force=False, targetdir=None): # Send Datadog deployment notification datadog = Datadog(util.rget(service.document, 'datadog', 'token')) datadog.notify( - title="Deployed %s to the %s environment" % (service.id, service.environment), - message="%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%" % (service.id, service.image, service.environment, parsedMarathonUrl.netloc), id=service.id, - tags=[ - "environment:%s" % service.environment, - "service:%s" % service.id - ] - ) + title="Deployed %s to the %s environment" % (service.id, service.environment), + message="%%%%%% \n Lighter deployed **%s** with image **%s** to **%s** (%s) \n %%%%%%" % ( + service.id, service.image, service.environment, parsedMarathonUrl.netloc), + tags=["environment:%s" % service.environment, "service:%s" % service.id]) except urllib2.HTTPError, e: raise RuntimeError("Failed to deploy %s HTTP %d (%s)" % (service.filename, e.code, e)), None, sys.exc_info()[2] diff --git a/src/lighter/test/deploy_test.py b/src/lighter/test/deploy_test.py index 6a216a6..4fbc6b0 100644 --- a/src/lighter/test/deploy_test.py +++ b/src/lighter/test/deploy_test.py @@ -4,6 +4,9 @@ from lighter.util import jsonRequest class DeployTest(unittest.TestCase): + def setUp(self): + self._called = False + def testParseService(self): service = lighter.parse_service('src/resources/yaml/staging/myservice.yml') self.assertEquals(service.document['hipchat']['token'], 'abc123') @@ -53,20 +56,37 @@ def testParseError(self): with self.assertRaises(RuntimeError): lighter.deploy('http://localhost:1/', filenames=['src/resources/yaml/staging/myservice.yml', 'src/resources/yaml/staging/myservice-broken.yml']) + def _createJsonRequestWrapper(self, marathonurl='http://localhost:1'): + appurl = '%s/v2/apps/myproduct/myservice' % marathonurl - def _resolvePost(self, url, data=None, *args, **kwargs): - if url.startswith('file:'): - return jsonRequest(url, data, *args, **kwargs) - if '/v2/apps' in url and data: - self.assertEquals(data['container']['docker']['image'], 'meltwater/myservice:1.0.0') - self._resolvePostCalled = True - return {'app': {}} - - def testResolve(self): - with patch('lighter.util.jsonRequest', wraps=self._resolvePost) as mock_jsonRequest: + def wrapper(url, method='GET', data=None, *args, **kwargs): + if url.startswith('file:'): + return jsonRequest(url, data, *args, **kwargs) + if url == appurl and method == 'PUT' and data: + self.assertEquals(data['container']['docker']['image'], 'meltwater/myservice:1.0.0') + self._called = True + return {} + if url == appurl and method == 'GET': + return {'app': {}} + return None + return wrapper + + def testResolveMavenJson(self): + with patch('lighter.util.jsonRequest', wraps=self._createJsonRequestWrapper()) as mock_jsonRequest: lighter.deploy('http://localhost:1/', filenames=['src/resources/yaml/integration/myservice.yml']) - self.assertTrue(self._resolvePostCalled) + self.assertTrue(self._called) + + def testDefaultMarathonUrl(self): + with patch('lighter.util.jsonRequest', wraps=self._createJsonRequestWrapper('http://defaultmarathon:2')) as mock_jsonRequest: + lighter.deploy(marathonurl=None, filenames=['src/resources/yaml/integration/myservice.yml']) + self.assertTrue(self._called) + def testNoMarathonUrlDefined(self): + with patch('lighter.util.jsonRequest', wraps=self._createJsonRequestWrapper()) as mock_jsonRequest: + with self.assertRaises(RuntimeError) as cm: + lighter.deploy(marathonurl=None, filenames=['src/resources/yaml/staging/myservice.yml']) + self.assertEqual("No Marathon URL defined for service src/resources/yaml/staging/myservice.yml", cm.exception.message) + def testUnresolvedVariable(self): service_yaml = 'src/resources/yaml/integration/myservice-unresolved-variable.yml' try: diff --git a/src/resources/yaml/integration/globals.yml b/src/resources/yaml/integration/globals.yml new file mode 100644 index 0000000..4db8b7e --- /dev/null +++ b/src/resources/yaml/integration/globals.yml @@ -0,0 +1,2 @@ +marathon: + url: 'http://defaultmarathon:2/' \ No newline at end of file From 399a87a024be848595a34ef78dc07ec4ab5c8e8f Mon Sep 17 00:00:00 2001 From: Mikael Johansson Date: Fri, 11 Dec 2015 13:26:56 +0100 Subject: [PATCH 2/2] Updated README with Marathon URL defaults. Fixed overwriting of deploy_parser variable --- README.md | 44 +++++++++++++++++++++++++------------------- src/lighter/main.py | 8 ++++---- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index a80ccca..6ce9f3e 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,17 @@ usage: lighter COMMAND [OPTIONS]... Marathon deployment tool positional arguments: - {deploy,verify} Available commands - deploy Deploy services to Marathon - verify Verify and generate Marathon configuration files + {deploy,verify} Available commands + deploy Deploy services to Marathon + verify Verify and generate Marathon configuration files optional arguments: - -h, --help show this help message and exit - -n, --noop Execute dry-run without modifying Marathon [default: False] - -v, --verbose Increase logging verbosity [default: False] + -h, --help show this help message and exit + -n, --noop Execute dry-run without modifying Marathon [default: + False] + -v, --verbose Increase logging verbosity [default: False] + -t TARGETDIR, --targetdir TARGETDIR + Directory to output rendered config files ``` ### Deploy Command @@ -42,7 +45,9 @@ positional arguments: optional arguments: -h, --help show this help message and exit -m MARATHON, --marathon MARATHON - Marathon url, e.g. "http://marathon-host:8080/" + Marathon URL like "http://marathon-host:8080/". + Overrides default Marathon URL's provided in config + files -f, --force Force deployment even if the service is already affected by a running deployment [default: False] ``` @@ -64,13 +69,23 @@ my-config-repo/ myservice.yml ``` -Running `lighter deploy -m http://marathon-host:8080 staging/services/myservice.yml` will +Running `lighter deploy staging/services/myservice.yml` will * Merge *myservice.yml* with environment defaults from *my-config-repo/staging/globals.yml* and *my-config-repo/globals.yml* * Fetch the *json* template for this service and version from the Maven repository * Expand the *json* template with variables and overrides from the *yml* files * Post the resulting *json* configuration into Marathon +## Marathon +Yaml files may contain a `marathon:` section with a default URL to reach Marathon at. The `-m/--marathon` +parameter will override this setting when given on the command-line. + +*globals.yml* +``` +marathon: + url: 'http://marathon-host:8080/' +``` + ## Maven The `maven:` section specifies where to fetch *json* templates from which are merged into the configuration. For example @@ -223,15 +238,6 @@ facts: environment: 'staging' ``` -## Marathon Defaults -Yaml files may contain a `marathon:` section with a default URL to reach Marathon at - -*globals.yml* -``` -marathon: - url: 'http://marathon-host:8080/' -``` - ## Installation Place a `lighter` script in the root of your configuration repo. Replace the LIGHTER_VERSION with a version from the [releases page](https://github.com/meltwater/lighter/releases). @@ -258,10 +264,10 @@ Use the script like cd my-config-repo # Deploy/sync all services (from Jenkins or other CI/CD server) -./lighter deploy -f -m http://marathon-host:8080 $(find staging -name \*.yml -not -name globals.yml) +./lighter deploy $(find staging -name \*.yml -not -name globals.yml) # Deploy single services -./lighter deploy -m http://marathon-host:8080 staging/myservice.yml staging/myservice2.yml +./lighter deploy staging/myservice.yml staging/myservice2.yml ``` ## Integrations diff --git a/src/lighter/main.py b/src/lighter/main.py index 6275b52..555e846 100755 --- a/src/lighter/main.py +++ b/src/lighter/main.py @@ -242,7 +242,7 @@ def verify(filenames, targetdir=None, verifySecrets=False): help='Deploy services to Marathon', description='Deploy services to Marathon') - deploy_parser.add_argument('-m', '--marathon', required=True, dest='marathon', help='Marathon url, e.g. "http://marathon-host:8080/"', + deploy_parser.add_argument('-m', '--marathon', required=True, dest='marathon', help='Marathon URL like "http://marathon-host:8080/". Overrides default Marathon URL\'s provided in config files', default=os.environ.get('MARATHON_URL', '')) deploy_parser.add_argument('-f', '--force', dest='force', help='Force deployment even if the service is already affected by a running deployment [default: %(default)s]', action='store_true', default=False) @@ -250,16 +250,16 @@ def verify(filenames, targetdir=None, verifySecrets=False): help='Service files to expand and deploy') # Create the parser for the "verify" command - deploy_parser = subparsers.add_parser('verify', + verify_parser = subparsers.add_parser('verify', prog='lighter', usage='%(prog)s verify YMLFILE...', help='Verify and generate Marathon configuration files', description='Verify and generate Marathon configuration files') - deploy_parser.add_argument('filenames', metavar='YMLFILE', nargs='+', + verify_parser.add_argument('filenames', metavar='YMLFILE', nargs='+', help='Service files to expand and deploy') - deploy_parser.add_argument('--verify-secrets', dest='verifySecrets', help='Fail verification if unencrypted secrets are found [default: %(default)s]', + verify_parser.add_argument('--verify-secrets', dest='verifySecrets', help='Fail verification if unencrypted secrets are found [default: %(default)s]', action='store_true', default=False) args = parser.parse_args()