#488: Add support for Matrix builds

.github/workflows/ci-build.yml

@@ -35,9 +35,8 @@ jobs:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ matrix.go }}-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go-${{ matrix.go }}
             ${{ runner.os }}-go-
       - name: Install Go tools
         run: go install github.com/google/[email protected]

README.md

@@ -101,16 +101,45 @@ The syntax for a replacement is `broken-url|replacement`.
 
 Project-keeper will then use the replacement in the `dependencies.md` file instead of the original url.
 
-### GitHub Runner Operating System
+### CI Build Configuration
 
-Some projects require to run the integration tests in the CI build on an operating system other than the default `ubuntu-latest`. In this case you can add the following to file `.project-keeper.yml`:
+PK allows configuring the generated CI-Build workflow scripts using the `build` section in file `.project-keeper.yml`.
+
+#### GitHub Runner Operating System
+
+Some projects require to run the integration tests in the CI build on an operating system other than the default `ubuntu-latest`. In this case you can use the following:
 
 ```yml
-ciBuildRunnerOS: ubuntu-20.04
+build:
+  runnerOs: ubuntu-20.04
 ```
 
 PK will use this setting for GitHub workflows `ci-build.yml` and `release_droid_prepare_original_checksum.yml` which run integration tests. The other workflows don't run integration tests and will stick to the default `ubuntu-latest`.
 
+#### Free Disk Space
+
+Some projects need more disk space during build, e.g. for Docker images. You can free up disk space before the build like this:
+
+```yml
+build:
+  freeDiskSpace: true
+```
+
+This will slow down the build by about one minute.
+
+#### Matrix Build with Exasol DB Versions
+
+To CI build as a matrix build with multiple Exasol DB versions you can add the following:
+
+```yml
+build:
+  exasolDbVersions:
+    - "7.1.24"
+    - "8.23.1"
+```
+
+Sonar will only run for the first version in the list.
+
 ## POM File
 
 For maven projects, project-keeper generates a `pk_generated_parent.pom` file. This file contains all the required plugins, dependencies and configurations. PK configures your `pom.xml` to use this file as a parent pom. By that, your `pom.xml` inherits all the configuration.

doc/changes/changes_2.9.16.md

@@ -1,21 +1,30 @@
-# Project Keeper 2.9.16, released 2023-??-??
+# Project Keeper 2.9.16, released 2023-11-13
 
 Code name: Support Matrix builds & enable compiler warnings
 
 ## Summary
 
 This release enables linter warnings for the Java compiler by adding argument `-Xlint:all`. This will log warnings but the build won't fail.
 
+The release also adds the following options for the build script:
+* Free up disk space before build
+* Select operating system for build
+* Select Exasol DB versions for which to run the build
+
+See the [user guide](../../README.md#ci-build-configuration) for details.
+
 ## Features
 
 * #497: Enabled linter warnings
+* #488: Added support for Matrix builds
 
 ## Dependency Updates
 
 ### Project-Keeper Shared Model Classes
 
 #### Test Dependency Updates
 
+* Added `com.jparams:to-string-verifier:1.4.8`
 * Updated `nl.jqno.equalsverifier:equalsverifier:3.15.2` to `3.15.3`
 * Updated `org.junit.jupiter:junit-jupiter-engine:5.10.0` to `5.10.1`
 * Updated `org.junit.jupiter:junit-jupiter-params:5.10.0` to `5.10.1`
@@ -39,6 +48,7 @@ This release enables linter warnings for the Java compiler by adding argument `-
 #### Test Dependency Updates
 
 * Updated `com.exasol:project-keeper-shared-test-setup:2.9.15` to `2.9.16`
+* Added `com.jparams:to-string-verifier:1.4.8`
 * Updated `nl.jqno.equalsverifier:equalsverifier:3.15.2` to `3.15.3`
 * Updated `org.junit.jupiter:junit-jupiter-engine:5.10.0` to `5.10.1`
 * Updated `org.junit.jupiter:junit-jupiter-params:5.10.0` to `5.10.1`

parent-pom/pom.xml

@@ -207,6 +207,12 @@
                 <version>3.15.3</version>
                 <scope>test</scope>
             </dependency>
+            <dependency>
+                <groupId>com.jparams</groupId>
+                <artifactId>to-string-verifier</artifactId>
+                <version>1.4.8</version>
+                <scope>test</scope>
+            </dependency>
             <!-- overriding vulnerable version -->
             <dependency>
                 <groupId>junit</groupId>

project-keeper/error_code_config.yml

@@ -2,4 +2,4 @@ error-tags:
   PK-CORE:
     packages:
       - com.exasol.projectkeeper
-    highest-index: 169
+    highest-index: 170

project-keeper/pom.xml

@@ -92,6 +92,11 @@
             <artifactId>equalsverifier</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.jparams</groupId>
+            <artifactId>to-string-verifier</artifactId>
+            <scope>test</scope>
+        </dependency>
         <!-- Fix warning from SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".-->
         <dependency>
             <groupId>org.slf4j</groupId>

project-keeper/src/main/java/com/exasol/projectkeeper/ProjectKeeper.java

@@ -134,7 +134,7 @@ private ValidationPhase phase2(final ValidationPhase.Provision provision) {
                 .logger(this.logger) //
                 .projectKeeperVersion(this.ownVersion) //
                 .hasNpmModule(hasSourceOfType(sources, SourceType.NPM)) //
-                .ciBuildRunnerOS(config.getCiBuildRunnerOS()) //
+                .ciBuildOptions(config.getCiBuildConfig()) //
                 .build();
         final List<Validator> validators = List.of( //
                 projectFilesValidator,

project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReader.java

@@ -12,6 +12,7 @@
 import org.yaml.snakeyaml.error.YAMLException;
 
 import com.exasol.errorreporting.ExaError;
+import com.exasol.projectkeeper.config.ProjectKeeperRawConfig.Build;
 import com.exasol.projectkeeper.shared.config.*;
 
 /**
@@ -79,14 +80,13 @@ private ProjectKeeperConfig parseRawConfig(final ProjectKeeperRawConfig rawConfi
         final List<String> excludes = convertExcludes(rawConfig.getExcludes());
         final List<String> linkReplacements = Objects.requireNonNullElseGet(rawConfig.getLinkReplacements(),
                 Collections::emptyList);
-        final VersionConfig version = parseVersion(rawConfig.getVersion(), projectDir);
 
         return ProjectKeeperConfig.builder() //
                 .sources(sources) //
                 .linkReplacements(linkReplacements) //
                 .excludes(excludes) //
-                .versionConfig(version) //
-                .ciBuildRunnerOS(rawConfig.getCiBuildRunnerOS()) //
+                .versionConfig(parseVersion(rawConfig.getVersion(), projectDir)) //
+                .buildOptions(convertBuildOptions(rawConfig.getBuild())) //
                 .build();
     }
 
@@ -109,6 +109,17 @@ private VersionConfig parseVersion(final Object rawVersion, final Path projectDi
                 .toString());
     }
 
+    private BuildOptions convertBuildOptions(final Build build) {
+        if (build == null) {
+            return BuildOptions.builder().build();
+        }
+        return BuildOptions.builder() //
+                .runnerOs(build.getRunnerOs()) //
+                .freeDiskSpace(build.shouldFreeDiskSpace()) //
+                .exasolDbVersions(build.getExasolDbVersions()) //
+                .build();
+    }
+
     private List<String> convertExcludes(final List<Object> rawExcludes) {
         if (rawExcludes == null) {
             return Collections.emptyList();

project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperRawConfig.java

@@ -1,5 +1,7 @@
 package com.exasol.projectkeeper.config;
 
+import static java.util.Collections.emptyList;
+
 import java.util.List;
 import java.util.Map;
 
@@ -10,18 +12,18 @@
  * </p>
  */
 public class ProjectKeeperRawConfig {
-    private List<ProjectKeeperRawConfig.Source> sources;
+    private List<Source> sources;
     private List<String> linkReplacements;
     private List<Object> excludes;
     private Object version;
-    private String ciBuildRunnerOS;
+    private Build build;
 
     /**
      * Get the sources.
      * 
      * @return sources
      */
-    public List<ProjectKeeperRawConfig.Source> getSources() {
+    public List<Source> getSources() {
         return sources;
     }
 
@@ -103,21 +105,21 @@ public void setVersion(final Object version) {
     }
 
     /**
-     * Get CI build runner operating system, e.g. {@code ubuntu-20.04}.
+     * Get the CI build configuration.
      * 
-     * @return CI build runner operating system
+     * @return build configuration
      */
-    public String getCiBuildRunnerOS() {
-        return ciBuildRunnerOS;
+    public Build getBuild() {
+        return build;
     }
 
     /**
-     * Set CI build runner operating system, e.g. {@code ubuntu-20.04}. Default: {@code ubuntu-latest}.
+     * Set the CI build configuration.
      * 
-     * @param ciBuildRunnerOS CI build runner operating system
+     * @param build build configuration
      */
-    public void setCiBuildRunnerOS(final String ciBuildRunnerOS) {
-        this.ciBuildRunnerOS = ciBuildRunnerOS;
+    public void setBuild(final Build build) {
+        this.build = build;
     }
 
     /**
@@ -307,4 +309,54 @@ public void setRelativePath(final String relativePath) {
             this.relativePath = relativePath;
         }
     }
+
+    /**
+     * Intermediate class for de-serializing build options from PK's YAML configuration file
+     * {@link ProjectKeeperConfigReader#CONFIG_FILE_NAME}.
+     * <p>
+     * SnakeYML requires this class to be public.
+     */
+    public static class Build {
+        private String runnerOs;
+        private boolean freeDiskSpace = false;
+        private List<String> exasolDbVersions = emptyList();
+
+        /**
+         * Get CI build runner operating system, e.g. {@code ubuntu-20.04}.
+         * 
+         * @return CI build runner operating system
+         */
+        public String getRunnerOs() {
+            return runnerOs;
+        }
+
+        /**
+         * Set CI build runner operating system, e.g. {@code ubuntu-20.04}. Default: {@code ubuntu-latest}.
+         * 
+         * @param runnerOs CI build runner operating system
+         */
+        public void setRunnerOs(final String runnerOs) {
+            this.runnerOs = runnerOs;
+        }
+
+        /** @return {@code true} if the CI build should free disk space before running the build */
+        public boolean shouldFreeDiskSpace() {
+            return freeDiskSpace;
+        }
+
+        /** @param freeDiskSpace {@code true} if the CI build should free disk space before running the build */
+        public void setFreeDiskSpace(final boolean freeDiskSpace) {
+            this.freeDiskSpace = freeDiskSpace;
+        }
+
+        /** @return Exasol DB versions for which to run the CI build */
+        public List<String> getExasolDbVersions() {
+            return exasolDbVersions;
+        }
+
+        /** @param exasolDbVersions Exasol DB versions for which to run the CI build */
+        public void setExasolDbVersions(final List<String> exasolDbVersions) {
+            this.exasolDbVersions = exasolDbVersions;
+        }
+    }
 }

project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java

@@ -0,0 +1,51 @@
+package com.exasol.projectkeeper.validators.files;
+
+import static com.exasol.projectkeeper.validators.files.FileTemplate.Validation.REQUIRE_EXACT;
+import static java.util.stream.Collectors.joining;
+
+import com.exasol.projectkeeper.shared.config.BuildOptions;
+
+class CiBuildWorkflowGenerator {
+    private final BuildOptions buildOptions;
+
+    CiBuildWorkflowGenerator(final BuildOptions buildOptions) {
+        this.buildOptions = buildOptions;
+    }
+
+    FileTemplate createReleaseDroidPrepareOriginalChecksumWorkflow() {
+        return new FileTemplateFromResource(".github/workflows/release_droid_prepare_original_checksum.yml",
+                REQUIRE_EXACT) //
+                .replacing("ciBuildRunnerOS", buildOptions.getRunnerOs())
+                .replacing("freeDiskSpace", String.valueOf(buildOptions.shouldFreeDiskSpace()));
+    }
+
+    FileTemplateFromResource createCiBuildWorkflow() {
+        final FileTemplateFromResource template = new FileTemplateFromResource(
+                "templates/.github/workflows/" + getCiBuildTemplate(), ".github/workflows/ci-build.yml", REQUIRE_EXACT)
+                .replacing("ciBuildRunnerOS", buildOptions.getRunnerOs())
+                .replacing("freeDiskSpace", String.valueOf(buildOptions.shouldFreeDiskSpace()));
+
+        if (isMatrixBuild()) {
+            template.replacing("matrixExasolDbVersions",
+                    buildOptions.getExasolDbVersions().stream().map(this::quote).collect(joining(", ")));
+            template.replacing("defaultExasolDbVersion", quote(buildOptions.getExasolDbVersions().get(0)));
+        }
+        return template;
+    }
+
+    private String quote(final String value) {
+        return '"' + value + '"';
+    }
+
+    private String getCiBuildTemplate() {
+        if (isMatrixBuild()) {
+            return "ci-build-db-version-matrix.yml";
+        } else {
+            return "ci-build.yml";
+        }
+    }
+
+    private boolean isMatrixBuild() {
+        return !buildOptions.getExasolDbVersions().isEmpty();
+    }
+}

project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/FileTemplateFromResource.java

@@ -41,14 +41,20 @@ public FileTemplateFromResource(final String templateResource, final String path
     }
 
     /**
-     * Add a replacement definition. {@link FileTemplateFromResource} identifies variables in the template by prefix "$"
-     * and replaces each variable with the value provided beforehand.
+     * Add a replacement definition. {@link FileTemplateFromResource} identifies variables in the template by prefix
+     * {@code $} and replaces each variable with the value provided beforehand.
      *
      * @param name        name of the variable
      * @param replacement text that should be used to replace the variable
      * @return this for fluent programming
      */
     public FileTemplateFromResource replacing(final String name, final String replacement) {
+        if (this.replacements.containsKey(name)) {
+            throw new IllegalStateException(ExaError.messageBuilder("E-PK-CORE-170").message(
+                    "Cannot add replacement with value {{new replacement}} because {{name}} is already registered as replacement with value {{existing value}}.",
+                    replacement, name, replacements.get(name)).mitigation("Remove or rename one of the replacements.")
+                    .toString());
+        }
         this.replacements.put(name, replacement);
         return this;
     }