release

#!/usr/bin/env python

"""
Quark release tool. Because quark involves a variety of different
support packages for backend languages, the release process involves
dealing with a lot of different packaging tools and environments. This
utility captures all this in a single place.

Usage:
  release [--dry] version [--dev | --prod] [<version>] [--doc=<version>]
  release [--dry] push-pkgs
  release [--dry] push-docs

  Options:
    --dry                  Dry run.

    --dev                  Change to development packages.
    --prod                 Change to production packages.

    --doc <version>        Documentation version.
"""

import json, os, random, re, subprocess, tempfile, urllib2
from docopt import docopt
from collections import OrderedDict

__dir__ = os.path.dirname(__file__)
__dry__ = False

def base(name=None):
    if name:
        return os.path.join(__dir__, name)
    else:
        return __dir__

metadata = {}
with open(base("quarkc/_metadata.py")) as fp:
    exec(fp.read(), metadata)
with open(base("docs/conf.py")) as fp:
    for line in fp:
        if line.startswith("__doc_version__"):
            exec(line, metadata)

## utilities

class ReleaseError(Exception):
    """An anticipated error."""

def check(*commands):
    """Check whether the supplied commands are available."""
    cmd = ["which"] + list(commands)
    try:
        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError, e:
        raise ReleaseError("Please install and configure missing prereqs: %s\n\n%s" %
                           (", ".join(commands), e.output))


def pipcheck(*packages):
    missing = []
    for pkg in packages:
        try:
            subprocess.check_output(["pip", "show", pkg], stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError, e:
            missing.append(pkg)
    if missing:
        raise ReleaseError("Please install missing python packages: %s" % ", ".join(missing))

def run(*command, **kwargs):
    """Execute the supplied command."""
    dry = kwargs.pop("dry", __dry__)
    print " ".join(command), kwargs
    if dry: return
    subprocess.check_call(command, **kwargs)

def call(*command, **kwargs):
    """Execute the supplied command and return the output."""
    dry = kwargs.pop("dry", __dry__)
    print " ".join(command), kwargs
    if dry: return
    return subprocess.check_output(command, **kwargs)

def update_json(name, **kwargs):
    """Update a json file with supplied overrides."""
    with open(name, "r") as fd:
        original = fd.read()
        obj = json.loads(original, object_pairs_hook=OrderedDict)
        obj.update(kwargs)
    updated = json.dumps(obj, indent=4, separators=(',', ': ')) + "\n"
    if updated != original:
        print "Updating json %s: %s" % (name, kwargs)
        if __dry__: return
        with open(name, "w") as fd:
            fd.write(updated)

def substitute(line, vars):
    """
    Substitute a line of python code declaring a variable of the form
    __<name>__ = <value>.  The name must be at the beginning of the
    line in order to match.
    """
    for key in vars:
        varname = "__%s__" % key
        if line.startswith(varname):
            return "%s = %s\n" % (varname, repr(vars[key]))
    else:
        return line

def update_python(name, **kwargs):
    """Update a python file with supplied overrides."""
    lines = []
    orig_lines = []
    with open(name, "r") as fd:
        for line in fd:
            orig_lines.append(line)
            lines.append(substitute(line, kwargs))
    updated = "".join(lines)
    original = "".join(orig_lines)
    if updated != original:
        print "Updating python %s: %s" % (name, kwargs)
        if __dry__: return True
        with open(name, "w") as fd:
            fd.write(updated)
        return True
    else:
        return False

def push_wheel(path):
    dest = tempfile.mkdtemp()
    run("python", "setup.py", "-q", "clean", "bdist_wheel", "-d", dest, cwd=base(path), dry=False)
    for name in os.listdir(dest):
        run("twine", "upload", "--skip-existing", os.path.join(dest, name), cwd=base())

## version update logic

def quark_version(version, doc_version, dev):
    if dev:
        title = "datawire-quarkdev"
    else:
        title = "datawire-quark"

    subs = {"title": title,
            "version": version,
            "doc_version": doc_version}

    updated = []
    for fname in ("quarkc/_metadata.py", "docs/conf.py"):
        if update_python(base(fname), **subs):
            updated.append(fname)

    return title, updated

def is_dev():
    return metadata["__title__"].endswith("dev")

## push logic

def push_quark():
    push_wheel(base())

## subcommands

def show_metadata():
    names = metadata["__all__"] + ["__doc_version__"]
    width = max(map(lambda n: len(n.strip("_")), names))
    for name in names:
        print "%*s: %s" % (width, name.strip("_"), metadata[name])

def version(args):
    if not (args["--dev"] or args["--prod"] or args["--doc"] or args["<version>"]):
        return show_metadata()

    if args["--dev"]:
        dev = True
    elif args["--prod"]:
        dev = False
    else:
        dev = is_dev()

    ver = args["<version>"]
    
    if ver:
        doc_default = "1"
    else:
        ver = metadata["__version__"]
        doc_default = metadata["__doc_version__"]

    docver = args["--doc"] or doc_default

    title, updated = quark_version(ver, docver, dev)
    for fname in updated:
        run("git", "add", fname, cwd=base())
    run("git", "commit", "-m", "Changed version to %s, %s (doc %s)." % (title, ver, docver),
        cwd=base())

def randstr():
    result = ""
    for i in range(4):
        result += chr(ord('a') + random.randint(0, 25))
    return result

def challenge():
    chal = randstr()
    text = raw_input('Please type "%s" to confirm: ' % chal)
    return text == chal

release_document_template = """
README file: https://github.com/datawire/quark/blob/%(tag)s/README.md

Full documentation: http://datawire.github.io/quark/%(short)s/index.html

We would appreciate your feedback! Please file issues here on Github.

For more information get in touch by email at hello@datawire.io or by Twitter at @datawireio.
"""

def make_github_release(tag, commitish=""):
    # Assumes that the tag already exists or the commit-ish is provided.
    major, minor, _ = tag.split(".", 2)
    short = major + "." + minor
    name = "Quark %s" % tag
    body = release_document_template.strip() % locals()
    post_data = dict(tag_name=tag, target_commitish=commitish, draft=False, prerelease=False, name=name, body=body)

    # Visit https://github.com/settings/tokens
    # Select "Generate new token"
    # Only required scope is "public_repo"
    # Paste resulting token (40 chars as of Jan 2016) into:
    token_path = "~/.github_public_repo_token"
    try:
        token = open(os.path.expanduser(token_path)).read().strip()
    except IOError as exc:
        print "Github release failed: Could not find Github auth token in %s: %s" % (token_path, exc)
        return False

    url = "https://api.github.com/repos/datawire/quark/releases?access_token=%(token)s" % locals()
    try:
        urllib2.urlopen(url, json.dumps(post_data))
        return True
    except Exception as exc:
        print "Github release failed: %s" % exc

    return False

def push_pkgs(args):
    print "You are about to push the following package live:"
    print
    print "    %s: %s" % (metadata["__title__"], metadata["__version__"])
    print
    if challenge():
        print "Pushing"
        push_quark()
    else:
        print "Canceled"

def push_docs(args):
    version = metadata["__version__"]
    short = ".".join(version.split(".")[:2])
    docver = metadata["__doc_version__"]
    if is_dev():
        docdest = os.path.join("dev", short)
    else:
        docdest = short
    print "You are about to push the documentation live:"
    print
    print "    version: %s" % version
    print "        doc: %s" % docver
    print "       dest: %s" % docdest
    print
    if challenge():
        print "Pushing docs"
        dest = tempfile.mkdtemp()
        run("git", "clone", call("git", "config", "--get", "remote.origin.url", dry=False).strip(),
            dest)
        run("git", "checkout", "gh-pages", cwd=dest)
        run("touch", ".nojekyll", cwd=dest)
        run("git", "rm", "-r", "--ignore-unmatch", docdest, cwd=dest)
        run("sphinx-build", "-q", "docs", os.path.join(dest, docdest), cwd=base())
        run(base("docs/substitute.sh"), dest)
        run("git", "add", ".", cwd=dest)
        run("git", "commit", "-m", "Documentation update %s-%s." % (version, docver), cwd=dest)
        run("git", "push", cwd=dest)
        print "Documentation is pushed."
    else:
        print "Canceled"

## main

def main(args):
    global __dry__
    __dry__ = args["--dry"]
    check("pip", "twine", "docker", "git")
    pipcheck("wheel", "sphinx", "sphinx-better-theme")
    if args["version"]:
        version(args)
    elif args["push-pkgs"]:
        push_pkgs(args)
    elif args["push-docs"]:
        push_docs(args)
    else:
        assert False

def call_main(args):
    try:
        return main(args)
    except ReleaseError, e:
        return e

if __name__ == "__main__":
    exit(call_main(docopt(__doc__)))