From cf31cdec0f52fb6b222bab6e7fa4031202fd7440 Mon Sep 17 00:00:00 2001 From: Joel Croteau Date: Tue, 27 Feb 2018 13:17:33 -0800 Subject: [PATCH 1/3] Add deploy option and some other changes. This adds an option to upload the generated egg to a PyPI repository via the deploy goal. Also some rewrites to save on code reuse and adding support for a few different distribution types. Also fixed the install command. --- pom.xml | 329 +++++++++--------- .../distribute/AbstractSetupCommandMojo.java | 98 ++++++ .../github/mojos/distribute/DeployMojo.java | 57 +++ .../github/mojos/distribute/InstallMojo.java | 54 +-- .../github/mojos/distribute/PackageMojo.java | 92 ++--- .../mojos/distribute/ProcessSourcesMojo.java | 9 +- 6 files changed, 358 insertions(+), 281 deletions(-) create mode 100644 src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java create mode 100644 src/main/java/com/github/mojos/distribute/DeployMojo.java diff --git a/pom.xml b/pom.xml index 1ed5889..a1c1bcc 100644 --- a/pom.xml +++ b/pom.xml @@ -1,175 +1,180 @@ - 4.0.0 - com.github.UltimateDogg - maven-python-distribute-plugin - 0.2.1 - maven-plugin - Maven Python Distribute Plugin - https://github.com/UltimateDogg/maven-python-distribute-plugin - Allows to package and distribute Python modules during Maven builds + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" + xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + 4.0.0 + com.github.UltimateDogg + maven-python-distribute-plugin + 0.2.1 + maven-plugin + Maven Python Distribute Plugin + https://github.com/UltimateDogg/maven-python-distribute-plugin + Allows to package and distribute Python modules during Maven builds - - - Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + - - UTF-8 - + + UTF-8 + - - - Jacek Furmankiewicz - jacek99@gmail.com - - + + + Jacek Furmankiewicz + jacek99@gmail.com + + + Joel Croteau + jcroteau@gmail.com + LiveRamp + + - - - org.apache.maven - maven-project - 2.0.6 - compile - - - org.apache.maven - maven-plugin-api - 2.0 - compile - - - org.projectlombok - lombok - 0.11.2 - compile - - - commons-io - commons-io - 2.0 - - - com.google.collections - google-collections - 1.0 - compile - - - org.javatuples - javatuples - 1.1 - compile - - - junit - junit - 4.7 - test - - - org.apache.maven - maven-core - 2.2.1 - - + + + org.apache.maven + maven-project + 2.0.6 + compile + + + org.apache.maven + maven-plugin-api + 2.0 + compile + + + org.projectlombok + lombok + 0.11.2 + compile + + + commons-io + commons-io + 2.0 + + + com.google.collections + google-collections + 1.0 + compile + + + org.javatuples + javatuples + 1.1 + compile + + + junit + junit + 4.7 + test + + + org.apache.maven + maven-core + 2.2.1 + + - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - - + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + + - - org.apache.maven.plugins - maven-plugin-plugin - 2.5.1 - - distribute - - - - generated-helpmojo - - helpmojo - - - - + + org.apache.maven.plugins + maven-plugin-plugin + 2.5.1 + + distribute + + + + generated-helpmojo + + helpmojo + + + + - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.4 - - - attach-javadocs - verify - - jar - - - - - - http://docs.oracle.com/javase/8/docs/api/ - - org.apache.calcite.sql.parser.impl - -Xdoclint:none - - - goal - a - Goal: - - - phase - a - Phase: - - - requiresProject - a - Requires Project: - - - - + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.4 + + + attach-javadocs + verify + + jar + + + + + + http://docs.oracle.com/javase/8/docs/api/ + + org.apache.calcite.sql.parser.impl + -Xdoclint:none + + + goal + a + Goal: + + + phase + a + Phase: + + + requiresProject + a + Requires Project: + + + + - - org.apache.maven.plugins - maven-source-plugin - 2.1.1 - - - attach-sources - verify - - jar-no-fork - - - - - - - - org.jvnet.wagon-svn - wagon-svn - 1.9 - - - + + org.apache.maven.plugins + maven-source-plugin + 2.1.1 + + + attach-sources + verify + + jar-no-fork + + + + + + + + org.jvnet.wagon-svn + wagon-svn + 1.9 + + + diff --git a/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java b/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java new file mode 100644 index 0000000..62f7cd0 --- /dev/null +++ b/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java @@ -0,0 +1,98 @@ +package com.github.mojos.distribute; + +/* + * Copyright 2001-2018 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.project.MavenProject; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Abstract class to run a setup.py command + */ +public abstract class AbstractSetupCommandMojo extends AbstractMojo { + /** + * @parameter default-value="${project}" + * @required + * @readonly + */ + private MavenProject project; + + /** + * @parameter default-value="python" + * @required + */ + private String pythonExecutable; + + /** + * Implementations should add any commands and arguments they wish to pass to setup.py here. + * + * @param args List to add setup.py commands to. + * @throws MojoExecutionException + */ + abstract void addSetupArgs(List args) throws MojoExecutionException; + + /* (non-Javadoc) + * @see org.apache.maven.plugin.AbstractMojo#execute() + */ + @Override + public void execute() throws MojoExecutionException { + final File buildDirectory = Paths.get(project.getBuild().getDirectory(), "maven-python").toFile(); + final String setupOutputCanonicalPath = project.getProperties().getProperty("python.distribute.plugin.setup.path"); + + try { + List args = new ArrayList<>(); + args.add(pythonExecutable); + args.add(setupOutputCanonicalPath); + addSetupArgs(args); + //execute setup script + ProcessBuilder processBuilder = new ProcessBuilder(args.toArray(new String[args.size()])); + processBuilder.directory(buildDirectory); + + Process pr = processBuilder.start(); + BufferedReader stdout = new BufferedReader(new InputStreamReader(pr.getInputStream())); + BufferedReader stderr = new BufferedReader(new InputStreamReader(pr.getErrorStream())); + while (!pr.waitFor(100, TimeUnit.MILLISECONDS)) { + stdout.lines().forEachOrdered(line -> getLog().debug(line)); + stderr.lines().forEachOrdered(line -> getLog().warn(line)); + } + + stdout.lines().forEachOrdered(line -> getLog().debug(line)); + stderr.lines().forEachOrdered(line -> getLog().warn(line)); + + int exitCode = pr.exitValue(); + if (exitCode != 0) { + throw new MojoExecutionException("'" + String.join(" ", processBuilder.command()) + "' returned error code " + exitCode); + } + } catch (FileNotFoundException e) { + throw new MojoExecutionException("Unable to find " + setupOutputCanonicalPath, e); + } catch (IOException e) { + throw new MojoExecutionException("Unable to read " + setupOutputCanonicalPath, e); + } catch (InterruptedException e) { + throw new MojoExecutionException("Unable to execute python " + setupOutputCanonicalPath, e); + } + } +} diff --git a/src/main/java/com/github/mojos/distribute/DeployMojo.java b/src/main/java/com/github/mojos/distribute/DeployMojo.java new file mode 100644 index 0000000..d9d5e38 --- /dev/null +++ b/src/main/java/com/github/mojos/distribute/DeployMojo.java @@ -0,0 +1,57 @@ +package com.github.mojos.distribute; + +/* + * Copyright 2001-2018 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.maven.plugin.MojoExecutionException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Uploads a Python module to PyPI using distribute + * + * @goal deploy + * @phase deploy + */ +public class DeployMojo extends AbstractSetupCommandMojo { + /** + * @parameter + */ + private String repository; + + /** + * @parameter + */ + private String distributionType; + + static List builtDistributionTypes = Collections.synchronizedList(new ArrayList<>()); + + @Override + void addSetupArgs(List args) throws MojoExecutionException { + if (distributionType != null) { + args.add(PackageMojo.getDistributionTypeArg(distributionType)); + } else { + args.addAll(builtDistributionTypes); + } + args.add("upload"); + if (repository != null) { + args.add("-r"); + args.add(repository); + } + } +} diff --git a/src/main/java/com/github/mojos/distribute/InstallMojo.java b/src/main/java/com/github/mojos/distribute/InstallMojo.java index b34adac..4b645db 100644 --- a/src/main/java/com/github/mojos/distribute/InstallMojo.java +++ b/src/main/java/com/github/mojos/distribute/InstallMojo.java @@ -1,7 +1,7 @@ package com.github.mojos.distribute; /* - * Copyright 2001-2005 The Apache Software Foundation. + * Copyright 2001-2018 The Apache Software Foundation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,55 +16,17 @@ * limitations under the License. */ -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStreamReader; - -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugin.MojoFailureException; +import java.util.List; /** * Installs a Python module using distribute - * + * * @goal install * @phase install */ -public class InstallMojo extends AbstractMojo { - - /* (non-Javadoc) - * @see org.apache.maven.plugin.AbstractMojo#execute() - */ - public void execute() throws MojoExecutionException, MojoFailureException { - - File setup = new File("src/main/python/setup.py"); - - try { - //execute setup script - ProcessBuilder t = new ProcessBuilder("python","setup.py","install"); - t.directory(new File("src/test/python")); - t.redirectErrorStream(true); - - Process pr = t.start(); - int exitCode = pr.waitFor(); - BufferedReader buf = new BufferedReader(new InputStreamReader(pr.getInputStream())); - String line = ""; - while ((line = buf.readLine()) != null) { - getLog().info(line); - } - - if (exitCode != 0) { - throw new MojoExecutionException("'python setup.py install' returned error code " + exitCode); - } - - } catch (FileNotFoundException e) { - throw new MojoExecutionException("Unable to find " + setup.getPath(),e); - } catch (IOException e) { - throw new MojoExecutionException("Unable to read " + setup.getPath(),e); - } catch (InterruptedException e) { - throw new MojoExecutionException("Unable to execute python " + setup.getPath(),e); - } - } +public class InstallMojo extends AbstractSetupCommandMojo { + @Override + void addSetupArgs(List args) { + args.add("install"); + } } diff --git a/src/main/java/com/github/mojos/distribute/PackageMojo.java b/src/main/java/com/github/mojos/distribute/PackageMojo.java index 61be848..fb160cb 100644 --- a/src/main/java/com/github/mojos/distribute/PackageMojo.java +++ b/src/main/java/com/github/mojos/distribute/PackageMojo.java @@ -1,7 +1,7 @@ package com.github.mojos.distribute; /* - * Copyright 2001-2005 The Apache Software Foundation. + * Copyright 2001-2018 The Apache Software Foundation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,17 +16,9 @@ * limitations under the License. */ -import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.project.MavenProject; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; +import java.util.List; /** * Packages a Python module using distribute @@ -34,72 +26,38 @@ * @goal package * @phase package */ -public class PackageMojo extends AbstractMojo { - /** - * @parameter default-value="${project}" - * @required - * @readonly - */ - private MavenProject project; - - /** - * @parameter default-value="python" - * @required - */ - private String pythonExecutable; - +public class PackageMojo extends AbstractSetupCommandMojo { /** * @parameter default-value="egg" * @required */ private String distributionType; + static String getDistributionTypeArg(String distributionType) throws MojoExecutionException { + switch (distributionType) { + case "egg": + case "wheel": + case "wininst": + case "rpm": + return "bdist_" + distributionType; + case "bdist": + return "bdist"; + case "source": + return "sdist"; + case "docs": + return "build_sphinx"; + default: + throw new MojoExecutionException("Invalid distributionType (egg, wheel, wininst, rpm, bdist, source, or docs supported): " + distributionType); + } + } + /* (non-Javadoc) * @see org.apache.maven.plugin.AbstractMojo#execute() */ @Override - public void execute() throws MojoExecutionException { - final File buildDirectory = Paths.get(project.getBuild().getDirectory(), "maven-python").toFile(); - final String setupOutputCanonicalPath = project.getProperties().getProperty("python.distribute.plugin.setup.path"); - - try { - String bdistName; - switch(distributionType) { - case "egg": - bdistName = "bdist_egg"; - break; - case "wheel": - bdistName = "bdist_wheel"; - break; - default: - throw new MojoExecutionException("invalid distributionType (egg or wheel supported): " + distributionType); - } - - //execute setup script - ProcessBuilder processBuilder = new ProcessBuilder(pythonExecutable, setupOutputCanonicalPath, bdistName); - processBuilder.directory(buildDirectory); - - Process pr = processBuilder.start(); - BufferedReader stdout = new BufferedReader(new InputStreamReader(pr.getInputStream())); - BufferedReader stderr = new BufferedReader(new InputStreamReader(pr.getErrorStream())); - while (!pr.waitFor(100, TimeUnit.MILLISECONDS)) { - stdout.lines().forEachOrdered(line->getLog().info(line)); - stderr.lines().forEachOrdered(line->getLog().error(line)); - } - - stdout.lines().forEachOrdered(line->getLog().debug(line)); - stderr.lines().forEachOrdered(line->getLog().warn(line)); - - int exitCode = pr.exitValue(); - if (exitCode != 0) { - throw new MojoExecutionException("python setup.py returned error code " + exitCode); - } - } catch (FileNotFoundException e) { - throw new MojoExecutionException("Unable to find " + setupOutputCanonicalPath, e); - } catch (IOException e) { - throw new MojoExecutionException("Unable to read " + setupOutputCanonicalPath, e); - } catch (InterruptedException e) { - throw new MojoExecutionException("Unable to execute python " + setupOutputCanonicalPath, e); - } + public void addSetupArgs(List args) throws MojoExecutionException { + String distributionTypeArg = getDistributionTypeArg(distributionType); + args.add(distributionTypeArg); + DeployMojo.builtDistributionTypes.add(distributionTypeArg); } } \ No newline at end of file diff --git a/src/main/java/com/github/mojos/distribute/ProcessSourcesMojo.java b/src/main/java/com/github/mojos/distribute/ProcessSourcesMojo.java index 49b4b9b..e2fc27a 100644 --- a/src/main/java/com/github/mojos/distribute/ProcessSourcesMojo.java +++ b/src/main/java/com/github/mojos/distribute/ProcessSourcesMojo.java @@ -1,7 +1,7 @@ package com.github.mojos.distribute; /* - * Copyright 2001-2005 The Apache Software Foundation. + * Copyright 2001-2018 The Apache Software Foundation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -107,12 +107,9 @@ public void execute() throws MojoExecutionException, MojoFailureException { try { if (setupTemplateProvided) { //update VERSION to latest version - List lines = new ArrayList(); - final InputStream inputStream = new BufferedInputStream(new FileInputStream(setupTemplate)); - try { + List lines = new ArrayList<>(); + try (InputStream inputStream = new BufferedInputStream(new FileInputStream(setupTemplate))) { lines.addAll(IOUtils.readLines(inputStream)); - } finally { - inputStream.close(); } int index = 0; From 04b0eeea464984826d65c8c747a53abeef326354 Mon Sep 17 00:00:00 2001 From: Joel Croteau Date: Tue, 27 Feb 2018 19:13:56 -0800 Subject: [PATCH 2/3] Attempt to log errors as errors --- .../github/mojos/distribute/AbstractSetupCommandMojo.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java b/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java index 62f7cd0..612ad4c 100644 --- a/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java +++ b/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java @@ -95,4 +95,11 @@ public void execute() throws MojoExecutionException { throw new MojoExecutionException("Unable to execute python " + setupOutputCanonicalPath, e); } } + + private void logErrorOrWarning(String line) { + if (line.toLowerCase().contains("error")) + getLog().error(line); + else + getLog().warn(line); + } } From 001220556ecb8e463dbe70232aa4b49e60bc6f58 Mon Sep 17 00:00:00 2001 From: Joel Croteau Date: Wed, 28 Feb 2018 11:15:16 -0800 Subject: [PATCH 3/3] Several improvements Add support for bdist_dumb packaging, set default path for setup.py, and add documentation of new features to readme file. --- README.rst | 64 ++++++++++++++----- .../distribute/AbstractSetupCommandMojo.java | 2 +- .../github/mojos/distribute/PackageMojo.java | 3 +- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index 8ab9b27..8247f4b 100644 --- a/README.rst +++ b/README.rst @@ -13,15 +13,15 @@ which is useful for IT shops that develop in both of these languages. Functionality ------------- -* keeps the *setup.py* version in sync with the Maven project version by updating setup.py in the **process-sources** phase -* packages the Python module during the Maven **package** phase -* allows specifying which format should the Python module be distributed as: source, RPM, egg, tar, zip, etc. +* Keeps the *setup.py* version in sync with the Maven project version by updating setup.py in the **process-sources** phase. +* Packages the Python module during the Maven **package** phase. +* Allows specifying which format should the Python module be distributed as: source, RPM, egg, tar, zip, etc. +* Supports uploading packages to PyPI during the **deploy** phase. Configuration ------------- -Add the following to your *pom.xml* build section: -:: +Add the following to your *pom.xml* build section:: maven-python-mojos @@ -43,6 +43,45 @@ Add the following to your *pom.xml* build section: +This defaults to building an egg file. If you would like to use another distribution type, you may specify something else:: + + + package + + package + + + rpm + + + +If you wish to build multiple different distribtion types, you can add multiple ```` blocks with different types. +Supported types are egg, wheel, wininst, rpm, bdist, dumb, source, or docs. + +If you wish to upload your packaged files to PyPI, add the following:: + + + deploy + + deploy + + + +This will default to uploading all of the packages generated by the **package** goal. If you don't want that, you can specify +a distribution type in the same way as you can for packaging. The deploy goal supports all distribution types except for docs. + +You can also optionally specify a repository to upload to:: + + + deploy + + deploy + + + https://pypi.myserver.com + + + setup.py -------- @@ -50,12 +89,13 @@ To make the code runnable outside maven you can have a setup.py. If a setup-temp your source root setup.py will be replaced. setup-template.py --------- +----------------- -setup template allows for using maven controlled variables in your setup.py file. +Setup template allows for using maven controlled variables in your setup.py file. Set the *version* field in your *setup-template.py* to a hardcoded constant of **${VERSION}**, e.g. Set the *name* field in your *setup-template.py* to a hardcoded constant of **${PROJECT_NAME}**, e.g. :: + from setuptools import setup, find_packages setup( @@ -65,13 +105,10 @@ Set the *name* field in your *setup-template.py* to a hardcoded constant of **${ packages = find_packages('.') ) - Maven Repository ---------------- -Add the following plugin repository to your *pom.xml* in order to use this plugin: - -:: +Add the following plugin repository to your *pom.xml* in order to use this plugin:: @@ -79,8 +116,3 @@ Add the following plugin repository to your *pom.xml* in order to use this plugi https://jitpack.io - - - - - diff --git a/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java b/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java index 612ad4c..c38669e 100644 --- a/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java +++ b/src/main/java/com/github/mojos/distribute/AbstractSetupCommandMojo.java @@ -61,7 +61,7 @@ public abstract class AbstractSetupCommandMojo extends AbstractMojo { @Override public void execute() throws MojoExecutionException { final File buildDirectory = Paths.get(project.getBuild().getDirectory(), "maven-python").toFile(); - final String setupOutputCanonicalPath = project.getProperties().getProperty("python.distribute.plugin.setup.path"); + final String setupOutputCanonicalPath = project.getProperties().getProperty("python.distribute.plugin.setup.path", "src/main/python/setup.py"); try { List args = new ArrayList<>(); diff --git a/src/main/java/com/github/mojos/distribute/PackageMojo.java b/src/main/java/com/github/mojos/distribute/PackageMojo.java index fb160cb..710eb3b 100644 --- a/src/main/java/com/github/mojos/distribute/PackageMojo.java +++ b/src/main/java/com/github/mojos/distribute/PackageMojo.java @@ -39,6 +39,7 @@ static String getDistributionTypeArg(String distributionType) throws MojoExecuti case "wheel": case "wininst": case "rpm": + case "dumb": return "bdist_" + distributionType; case "bdist": return "bdist"; @@ -47,7 +48,7 @@ static String getDistributionTypeArg(String distributionType) throws MojoExecuti case "docs": return "build_sphinx"; default: - throw new MojoExecutionException("Invalid distributionType (egg, wheel, wininst, rpm, bdist, source, or docs supported): " + distributionType); + throw new MojoExecutionException("Invalid distributionType (egg, wheel, wininst, rpm, bdist, dumb, source, or docs supported): " + distributionType); } }