make_dist

#!/usr/bin/env python

# This software was produced by NIST, an agency of the U.S. government,
# and by statute is not subject to copyright in the United States.
# Recipients of this software assume all responsibilities associated
# with its operation, modification and maintenance. However, to
# facilitate maintenance we ask that before distributing modified
# versions of this software, you first contact the authors at
# oof_manager@nist.gov. 


# Script for building a distribution of OOF2 or OOF3D.  Takes care of
# git stuff, and sets the version number in the packaged source code.

# This script assumes that the version to be packaged is the HEAD of
# the master branch on github.  The previous cvs versions of this
# script allowed a release to be built from any cvs branch.  If we
# want the oof version number to be stored in the committed versions
# of setup.py and oofversion.py, however, then we can only set the oof
# version number on one branch.  If we released from different branch,
# we'd get a conflict if and when the branch was later merged into
# master.  This wasn't so much of a problem in cvs, because the
# repository was private, and so it didn't matter if the branch number
# was never set in the committed files.

# The options are:
#    --version     a string of nonpunctuation characters and dots (required)
#    --comment     a string containing no dots (optional)

import getopt
import os
import os.path
import shutil
import subprocess
import sys
import tempfile
import time


def osCall(*args):
    print >> sys.stderr, "--->", ' '.join(args)
    status = subprocess.call(args)
    if status != 0:
        print >> sys.stderr, "Failed to execute", ' '.join(args)
        print >> sys.stderr, "Aborting!"
        sys.exit(status)
        
options = ['version=', 'comment=', 'dryrun', 'help', 'noclean', 'branch=']

def state_options_and_quit():
    print >> sys.stderr,  \
"""Options are:
--version=<version number> Version number.
--comment=<comment>        Optional comment, cannot contain dots or spaces 

Debugging options are:
--noclean              Don't remove temp directory
--dryrun               Don't actually commit, tag, or push in git
--branch=<branchname>  Use branch, not master.  Implies --dryrun and --noclean
                       Real distributions are always made without --branch.
--help                 Print this
"""
    sys.exit()

version = None
comment = None
branch = None
dryrun = False
noclean = False
dimension = 2

try:
    optlist, args = getopt.getopt(sys.argv[1:], '', options)
except getopt.error, message:
    print message
    sys.exit()

for opt in optlist:
    if opt[0] == '--version':
        version = opt[1]
    elif opt[0] == '--comment':
        comment = opt[1]
    elif opt[0] == '--noclean':
        noclean = True
    elif opt[0] == '--dryrun':
        dryrun = True
    elif opt[0] == '--branch':
        branch = opt[1]
        dryrun = True
        noclean = True
    elif opt[0] == '--help':
        state_options_and_quit()

assert version is not None

startdir = os.getcwd()

# Are we building oof2 or oof3d? This info is stored in
# distname.distname.  We get this from the directory in which
# make_dist is being run.
import distname

print >> sys.stderr, \
    "Building", distname.distname, "distribution named", version


# Create a temp directory.
tempdir = tempfile.mkdtemp(prefix='oof-tempdir-'+version+'-')
print >> sys.stderr, "Using temp directory", tempdir

try:
    # Get the git repository location
    proc = subprocess.Popen(['git', 'remote', 'get-url', 'origin'],
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdoutdata, stderrdata = proc.communicate()
    if stderrdata:
        print "*** Failed to get git repository!", stderrdata.strip()
        sys.exit(1)
    giturl = stdoutdata.strip()

    # cd to the temp directory
    print >> sys.stderr, "Changing directory to", tempdir
    os.chdir(tempdir)

    # Clone the git repository into a directory whose name is given by
    # the oof version number: oof2-version or oof3d-version
    distdir = distname.distname + "-" + version
    # # Use --quiet so that stderr is empty if the command succeeds.
    # cmd = ['git', 'clone', '--quiet', giturl, distdir]
    if branch is None:
        cmd = ['git', 'clone', giturl, distdir]
    else:
        cmd = ['git', 'clone', '--branch', branch, giturl, distdir]
    osCall(*cmd)

    # cd to the cloned source code directory
    print >> sys.stderr, "Changing directory to", distdir
    os.chdir(distdir)

    # Construct the commit message and new branch name (aka tag)
    commit_msg = ("Building " + distname.distname + " release version " +
                  version)
    if comment:
        commit_msg += " -- " + comment
    tag = distname.distname + "-" + version
    if comment:
        oldtag = tag
        comment.replace(' ', '-')
        tag += "--" + comment
    # Check and/or fix the tag if it's not legal
    proc = subprocess.Popen(['git', 'check-ref-format', '--normalize',
                             '--allow-onelevel', tag],
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdoutdata, stderrdata = proc.communicate()
    if proc.returncode == 0:    # success
        tag = stdoutdata.strip()
    else:
        print >> sys.stderr, "Failed to convert '%s' to a legal tag." % tag
        print >> sys.stderr, "Using '%s' instead." % oldtag
        tag = oldtag
    
    # Create a branch for the release.

    # First see if the branch already exists.  "git ls-remote" returns
    # 2 if the branch doesn't exist, when given the --exit-code
    # argument.
    if subprocess.call(['git', 'ls-remote', '--exit-code', 'origin', tag],
                       stdout=None, stderr=None) == 2:
        # The branch doesn't already exist.
        newbranch = True
        osCall('git', 'checkout', '-b', tag)
    else:
        # The branch already exists.  Check it out, and merge the
        # current master into it.  There *should* be no conflicts in
        # the merge, since nobod [y finished writing this comment!]
        newbranch = False
        osCall('git', 'checkout', tag)
        print >> sys.stderr, "Merging master branch into", tag
        proc = subprocess.Popen(['git', 'merge', '--no-edit', 'master'],
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        print >> sys.stderr, stdoutdata
        stdoutdata, stderrdata = proc.communicate()
        if proc.returncode != 0:
            print >> sys.stderr, stderrdata
            print >> sys.stderr, "Failed to merge master into", tag
            sys.exit(1)

    # Set the version number in setup.py and SRC/common/oofversion.py.
    # The sed command finds a line beginning with
    # "version_from_make_dist" and replaces everything after the
    # equals sign with a string containing the version number.
    sedcmd = '/^version_from_make_dist/s/=.*/= \\\"%s\\\"/' % version
    tmpfile, tmpfilename = tempfile.mkstemp(dir=tempdir)
    if subprocess.call(["sed", sedcmd, "setup.py"], stdout=tmpfile) != 0:
        sys.exit(proc.returncode)
    os.close(tmpfile)
    print >> sys.stderr, "Moving", tmpfilename, "to setup.py"
    os.rename(tmpfilename, 'setup.py')

    tmpfile, tmpfilename = tempfile.mkstemp(dir=tempdir)
    if subprocess.call(['sed', sedcmd, 'SRC/common/oofversion.py'],
                       stdout=tmpfile) != 0:
        sys.exit(proc.returncode)
    os.close(tmpfile)
    print >> sys.stderr, "Moving", tmpfilename, "to SRC/common/oofversion.py"
    os.rename(tmpfilename, 'SRC/common/oofversion.py')

    # Check all the modifications into git, and tag with the version
    # number.  It's necessary to go back to the distribution directory
    # to use git.
    osCall('git', 'add', 'SRC/common/oofversion.py', 'setup.py')

    cmd = ['git', 'commit', '--allow-empty', '-m', commit_msg]
    print >> sys.stderr, '--->', ' '.join(cmd)
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdoutdata, stderrdata = proc.communicate()
    if proc.returncode != 0:
        print >> sys.stderr, "Back from commit, status=", proc.returncode
        print >> sys.stderr, "stdoutdata=", stdoutdata
        print >> sys.stderr, "stderrdata=", stderrdata
        sys.exit(0)

    if not newbranch:
        cmd = ['git', 'push', 'origin', tag]
    else:
        cmd = ['git', 'push', '--set-upstream', 'origin', tag]

    # Push the modifications to the server, unless this is just a dry
    # run.
    if not dryrun:
        osCall(*cmd)
    else:
        print >> sys.stderr, "Dry run! Not running:"
        print >> sys.stderr, " ".join(cmd)

    # The distribution includes two files, MANIFEST and package_date,
    # that aren't in git, and are therefore constructed only now that
    # all the git manipulations are complete.
        
    # Make a timestamp file for the distribution.
    timefile = file("package_date", "w")
    print >> timefile, time.ctime()
    timefile.close()
    
    # Make the MANIFEST file after moving back up to the temp
    # directory.  Moving up to the temp directory means that the path
    # names in the file will all start with "oof{2,3d}-version/",
    # which is nice and modular.
    os.chdir(tempdir)

    # We aren't using distutils to automatically create a MANIFEST
    # file because it wants to include the swig output files.  Once
    # we've built the file ourselves, there's really no need to use
    # distutils here at all.

    import localexclusions

    print >> sys.stderr, "Changing directory to", tempdir
    
    globalExcludeDirs = [
        ".git"
    ] + localexclusions.globalExcludeDirs

    excludeDirs = [
        "math", 
        "NOTES", 
        "3DSandbox",
        "SRC/DOC",
        "SRC/TEST-DATA", "SRC/TEST-SRC",
        "SRC/common/EXTRA",
        "SRC/engine/EXTRA",
        "SRC/engine/PETSc",
        "SRC/image/GRAINBDY", "SRC/image/imagemanip"
    ] + localexclusions.excludeDirs

    globalExcludeFiles = [
        ".gitignore"
    ] + localexclusions.globalExcludeFiles

    excludeFiles = [
        "localexclusions.py", "distname.py",
        "make_dist",
        "oof2-build", "oof2-clean",
        "SRC/header", "SRC/header.py",
        "SRC/maketags"
    ] + localexclusions.excludeFiles

    excludeDirs = [os.path.join(distdir, f) for f in excludeDirs]
    excludeFiles = [os.path.join(distdir, f) for f in excludeFiles]

    def getFiles(path, manifest):
        if os.path.isdir(path):
            files = os.listdir(path) # just file name, no path components
            for f in files:
                if path != ".":
                    fname = os.path.join(path, f)
                else:
                    fname = f
                if (os.path.isfile(fname) and f not in globalExcludeFiles and
                    fname not in excludeFiles) :
                    print >> manifest, fname
                if (os.path.isdir(fname) and f not in globalExcludeDirs and
                    fname not in excludeDirs):
                    getFiles(fname, manifest)

    print >> sys.stderr, "Building MANIFEST"
    manifest = open(os.path.join(distdir, "MANIFEST"), "w")
    getFiles(distdir, manifest)   
    manifest.close()

    # Build the distribution.
    distfilename = distdir + ".tar.gz"
    print >> sys.stderr, "Distribution file is", distfilename

    cmd = ['tar', '-T', os.path.join(distdir, 'MANIFEST'), '-czf', distfilename]
    osCall(*cmd)
    print >> sys.stderr, "Moving", distfilename, "to", startdir
    finaldistfilename = os.path.join(startdir, distfilename)
    os.rename(distfilename, finaldistfilename)

finally:
    if not noclean:
        print >> sys.stderr, "Removing", tempdir
        shutil.rmtree(tempdir)
    else:
        print >> sys.stderr, "Not removing", tempdir


print >> sys.stderr, "Done.", distfilename, "is ready."
print >> sys.stderr, \
"""To publish it, copy it to WEBPAGES/%(dist)s/source and edit
WEBPAGES/%(dist)s/index.html.  Remember to check that the README file in
WEBPAGES/%(dist)s/source is up to date.""" % {'dist':distname.distname}
osCall("openssl", "dgst", "-md5", finaldistfilename)
osCall("openssl", "dgst", "-rmd160", finaldistfilename)
osCall("openssl", "dgst", "-sha256", finaldistfilename)