From 76b695ab32123a85cc5b8cdf306321eea768c16a Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 16:29:41 +0100 Subject: [PATCH 01/14] #597: Move workflows into ci-build.yml --- .github/workflows/broken_links_checker.yml | 4 - project-keeper/error_code_config.yml | 2 +- .../config/ProjectKeeperConfigReader.java | 6 ++ .../config/ProjectKeeperRawConfig.java | 2 + .../validators/DeletedFilesValidator.java | 1 + .../files/CiBuildWorkflowGenerator.java | 26 ++++--- .../files/FileTemplatesFactory.java | 25 ++----- .../validators/files/GitHubWorkflow.java | 16 +++- .../GitHubWorkflowJavaVersionCustomizer.java | 20 +++-- .../files/GitHubWorkflowStepCustomizer.java | 6 +- .../files/JavaVersionExtractor.java | 6 +- .../files/ProjectFilesValidator.java | 2 +- .../workflows/project-keeper-verify.yml | 1 + .../workflows/broken_links_checker.yml | 4 - .../workflows/ci-build-db-version-matrix.yml | 50 ++++++++++++- .../workflows/ci-build-native-build.yml | 1 + .../.github/workflows/ci-build-next-java.yml | 34 --------- .../templates/.github/workflows/ci-build.yml | 75 ++++++++++++++++++- .../main/resources/templates/gitattributes | 1 - .../config/ProjectKeeperConfigReaderTest.java | 46 ++++++++++-- .../files/CiBuildWorkflowGeneratorTest.java | 69 +++++++++-------- .../files/FileTemplatesFactoryTest.java | 33 ++++---- ...tHubWorkflowJavaVersionCustomizerTest.java | 16 ++-- .../GitHubWorkflowStepCustomizerTest.java | 31 +++++--- .../validators/files/GitHubWorkflowTest.java | 8 +- .../files/JavaVersionExtractorTest.java | 5 -- .../config/workflow/StepCustomization.java | 30 +++++++- .../config/workflow/CustomWorkflowTest.java | 8 +- 28 files changed, 342 insertions(+), 186 deletions(-) delete mode 100644 project-keeper/src/main/resources/templates/.github/workflows/ci-build-next-java.yml diff --git a/.github/workflows/broken_links_checker.yml b/.github/workflows/broken_links_checker.yml index 39612b76..90488caa 100644 --- a/.github/workflows/broken_links_checker.yml +++ b/.github/workflows/broken_links_checker.yml @@ -5,10 +5,6 @@ name: Broken Links Checker on: schedule: - cron: "0 5 * * 0" - push: - branches: - - main - pull_request: jobs: linkChecker: diff --git a/project-keeper/error_code_config.yml b/project-keeper/error_code_config.yml index c634c6c1..4ec71b22 100644 --- a/project-keeper/error_code_config.yml +++ b/project-keeper/error_code_config.yml @@ -2,4 +2,4 @@ error-tags: PK-CORE: packages: - com.exasol.projectkeeper - highest-index: 207 + highest-index: 208 diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReader.java b/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReader.java index cf6f08f5..d92ceeab 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReader.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReader.java @@ -170,6 +170,11 @@ private List convertSteps(final List st } private StepCustomization convertStep(final RawStepCustomization step) { + if (step.job == null || step.job.isBlank()) { + throw new IllegalArgumentException(ExaError.messageBuilder("E-PK-CORE-208") + .message("Missing job in step customization of file {{config file}}.", CONFIG_FILE_NAME) + .mitigation("Add job to the step customization.").toString()); + } if (step.action == null) { throw new IllegalArgumentException(ExaError.messageBuilder("E-PK-CORE-200") .message("Missing action in step customization of file {{config file}}.", CONFIG_FILE_NAME) @@ -188,6 +193,7 @@ private StepCustomization convertStep(final RawStepCustomization step) { .mitigation("Add content to the step customization.").toString()); } return StepCustomization.builder() // + .jobId(step.job) // .type(step.action) // .stepId(step.stepId) // .step(WorkflowStep.createStep(step.content)) // diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperRawConfig.java b/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperRawConfig.java index 2939860c..64c8534f 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperRawConfig.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperRawConfig.java @@ -460,6 +460,8 @@ public RawStepCustomization() { // Required for specifying Javadoc } + /** ID of the job to customize. */ + public String job; /** Customization type (insert/replace). */ public StepCustomization.Type action; /** ID of the step to replace or after which to insert. */ diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/DeletedFilesValidator.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/DeletedFilesValidator.java index 5a4d48f2..e6017447 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/DeletedFilesValidator.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/DeletedFilesValidator.java @@ -25,6 +25,7 @@ public class DeletedFilesValidator implements Validator { Path.of(GITHUB_WORKFLOWS, "release_droid_release_on_maven_central.yml"), RELEASE_DROID_WORKFLOW_WARNING, Path.of(GITHUB_WORKFLOWS, "release_droid_upload_github_release_assets.yml"), RELEASE_DROID_WORKFLOW_WARNING, Path.of(GITHUB_WORKFLOWS, "release_droid_prepare_original_checksum.yml"), RELEASE_DROID_WORKFLOW_WARNING, + Path.of(GITHUB_WORKFLOWS, "ci-build-next-java.yml"), "Next Java build is now included in ci-build.yml", Path.of("release_config.yml"), "Release-droid configuration is replaced by release.yml"); private final Path projectDirectory; diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java index 7859ba8d..db8c6f60 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java @@ -18,10 +18,13 @@ class CiBuildWorkflowGenerator { private static final String CI_BUILD_PATH_IN_PROJECT = WORKFLOW_PATH + CI_BUILD_WORKFLOW_NAME; private final BuildOptions buildOptions; private final List javaVersions; + private final String nextJavaVersion; - CiBuildWorkflowGenerator(final BuildOptions buildOptions, final List javaVersions) { + CiBuildWorkflowGenerator(final BuildOptions buildOptions, final List javaVersions, + final String nextJavaVersion) { this.buildOptions = Objects.requireNonNull(buildOptions, "buildOptions"); this.javaVersions = Objects.requireNonNull(javaVersions, "javaVersions"); + this.nextJavaVersion = Objects.requireNonNull(nextJavaVersion, "nextJavaVersion"); } FileTemplate createCiBuildWorkflow() { @@ -31,6 +34,7 @@ FileTemplate createCiBuildWorkflow() { CI_BUILD_PATH_IN_PROJECT, REQUIRE_EXACT); template.replacing("ciBuildRunnerOS", buildOptions.getRunnerOs()); template.replacing("freeDiskSpace", String.valueOf(buildOptions.shouldFreeDiskSpace())); + template.replacing("nextJavaVersion", nextJavaVersion); if (buildType == CiTemplateType.EXASOL_VERSION_MATRIX) { template.replacing("matrixExasolDbVersions", @@ -50,13 +54,13 @@ private FileTemplate createTemplate(final FileTemplateFromResource template, fin return new ContentCustomizingTemplate(template, new GitHubWorkflowCustomizer( // javaVersionCustomizer(), // [impl->dsn~customize-build-process.ci-build~0] - new GitHubWorkflowStepCustomizer(customizations, buildJobId), + new GitHubWorkflowStepCustomizer(customizations), // [impl->dsn~customize-build-process.ci-build.environment~1] new GitHubWorkflowEnvironmentCustomizer(buildJobId, environmentName))); } private GitHubWorkflowJavaVersionCustomizer javaVersionCustomizer() { - return new GitHubWorkflowJavaVersionCustomizer(javaVersions); + return new GitHubWorkflowJavaVersionCustomizer(javaVersions, nextJavaVersion); } private List findCustomizations(final String workflowName) { @@ -81,7 +85,7 @@ private CiTemplateType getCiBuildType() { FileTemplate createReleaseWorkflow(final List sources) { final Consumer templateCustomizer = template -> template .replacing("mavenCentralDeployment", mavenCentralDeploymentRequired(sources) ? "true" : "false"); - return createCustomizedWorkflow("release.yml", "release", templateCustomizer); + return createCustomizedWorkflow("release.yml", templateCustomizer); } private boolean mavenCentralDeploymentRequired(final List sources) { @@ -94,32 +98,32 @@ private boolean mavenCentralDeploymentRequired(final List source // [impl->dsn~customize-build-process.dependency-check~0] FileTemplate createDependenciesCheckWorkflow() { - return createCustomizedWorkflow("dependencies_check.yml", "report_security_issues"); + return createCustomizedWorkflow("dependencies_check.yml"); } // [impl->dsn~dependency-updater.workflow.generate~1] // [impl->dsn~customize-build-process.dependency-update~0] FileTemplate createDependenciesUpdateWorkflow() { - return createCustomizedWorkflow("dependencies_update.yml", "update_dependencies"); + return createCustomizedWorkflow("dependencies_update.yml"); } - private FileTemplate createCustomizedWorkflow(final String workflowName, final String jobName) { - return createCustomizedWorkflow(workflowName, jobName, template -> { + private FileTemplate createCustomizedWorkflow(final String workflowName) { + return createCustomizedWorkflow(workflowName, template -> { }); } - private FileTemplate createCustomizedWorkflow(final String workflowName, final String jobName, + private FileTemplate createCustomizedWorkflow(final String workflowName, final Consumer templateCustomizer) { final FileTemplateFromResource template = new FileTemplateFromResource(WORKFLOW_PATH + workflowName, REQUIRE_EXACT); templateCustomizer.accept(template); final List customizations = findCustomizations(workflowName); return new ContentCustomizingTemplate(template, new GitHubWorkflowCustomizer(javaVersionCustomizer(), - new GitHubWorkflowStepCustomizer(customizations, jobName))); + new GitHubWorkflowStepCustomizer(customizations))); } enum CiTemplateType { - DEFAULT(CI_BUILD_WORKFLOW_NAME, "build"), + DEFAULT(CI_BUILD_WORKFLOW_NAME, "build-and-test"), EXASOL_VERSION_MATRIX("ci-build-db-version-matrix.yml", "matrix-build"); private final String templateName; diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactory.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactory.java index 1dceb90b..80eee2c5 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactory.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactory.java @@ -24,6 +24,7 @@ class FileTemplatesFactory { private final boolean hasNpmModule; private final BuildOptions buildOptions; private final CiBuildWorkflowGenerator workflowGenerator; + private final List sources; public FileTemplatesFactory(final Logger logger, final String ownVersion, final boolean hasNpmModule, final BuildOptions buildOptions, final List sources) { @@ -31,11 +32,13 @@ public FileTemplatesFactory(final Logger logger, final String ownVersion, final this.ownVersion = Objects.requireNonNull(ownVersion, "ownVersion"); this.hasNpmModule = hasNpmModule; this.buildOptions = Objects.requireNonNull(buildOptions, "buildOptions"); - this.workflowGenerator = new CiBuildWorkflowGenerator(this.buildOptions, - new JavaVersionExtractor(sources).extractVersions()); + this.sources = sources; + final JavaVersionExtractor javaVersionExtractor = new JavaVersionExtractor(sources); + this.workflowGenerator = new CiBuildWorkflowGenerator(this.buildOptions, javaVersionExtractor.extractVersions(), + javaVersionExtractor.getNextVersion()); } - List getGlobalTemplates(final List sources) { + List getGlobalTemplates() { final List templates = new ArrayList<>(); templates.add(new FileTemplateFromResource(".github/workflows/broken_links_checker.yml", REQUIRE_EXACT)); templates.add(new FileTemplateFromResource(".vscode/settings.json", REQUIRE_EXIST)); @@ -43,7 +46,7 @@ List getGlobalTemplates(final List sources) { templates.add(new FileTemplateFromResource("templates/gitattributes", ".gitattributes", REQUIRE_EXIST) .replacing("pomFiles", mvnRoot.isPresent() ? POM_FILES_GENERATED : "")); if (mvnRoot.isPresent()) { - templates.addAll(getGenericMavenTemplates(sources, mvnRoot.get().getModules())); + templates.addAll(getGenericMavenTemplates(mvnRoot.get().getModules())); } else { templates.addAll(getProjectKeeperVerifyWorkflowTemplates()); this.logger.warn(ExaError.messageBuilder("W-PK-CORE-91") @@ -53,27 +56,15 @@ List getGlobalTemplates(final List sources) { return templates; } - private List getGenericMavenTemplates(final List sources, - final Set modules) { + private List getGenericMavenTemplates(final Set modules) { final List templates = new ArrayList<>(); templates.add(getCiBuildTemplate(modules)); - templates.add(getNextJavaWorkflow(modules, new JavaVersionExtractor(sources).getNextVersion())); templates.add(this.workflowGenerator.createDependenciesCheckWorkflow()); templates.add(this.workflowGenerator.createDependenciesUpdateWorkflow()); templates.add(this.workflowGenerator.createReleaseWorkflow(sources)); return templates; } - private FileTemplate getNextJavaWorkflow(final Set modules, final String nextJavaVersion) { - final FileTemplateFromResource template = new FileTemplateFromResource( - ".github/workflows/ci-build-next-java.yml", REQUIRE_EXACT) // - .replacing("skipNativeImage", - modules.contains(ProjectKeeperModule.NATIVE_IMAGE) ? "-P skipNativeImage" : "") - .replacing("nextJavaVersion", nextJavaVersion); - return new ContentCustomizingTemplate(template, - new GitHubWorkflowCustomizer(new GitHubWorkflowJavaVersionCustomizer(List.of(nextJavaVersion)))); - } - private FileTemplate getCiBuildTemplate(final Set modules) { if (modules.contains(NATIVE_IMAGE)) { return new FileTemplateFromResource("templates/.github/workflows/ci-build-native-build.yml", diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java index fbcbf1f7..201ff444 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java @@ -35,7 +35,7 @@ Job getJob(final String jobId) { if (rawJob == null) { return null; } - return new Job(asMap(rawJob)); + return new Job(jobId, asMap(rawJob)); } private Map getJobMap() { @@ -43,7 +43,7 @@ private Map getJobMap() { } List getJobs() { - return getJobMap().values().stream().map(GitHubWorkflow::asMap).map(Job::new).toList(); + return getJobMap().entrySet().stream().map(entry -> new Job(entry.getKey(), asMap(entry.getValue()))).toList(); } Map getOnTrigger() { @@ -59,12 +59,18 @@ Object getRawObject() { * well as insert and replace steps. */ static class Job { + private final String id; private final Map rawJob; - private Job(final Map rawJob) { + private Job(final String id, final Map rawJob) { + this.id = id; this.rawJob = rawJob; } + String getId() { + return id; + } + Step getStep(final String id) { return getSteps().stream() // .filter(hasId(id)) // @@ -137,7 +143,9 @@ private int findStepIndex(final String stepId) { .filter(index -> steps.get(index).getId().equals(stepId)) // .findFirst() // .orElseThrow(() -> new IllegalStateException(ExaError.messageBuilder("E-PK-CORE-205") - .message("No step found for id {{step id}} in {{raw job}}", stepId, rawJob).toString())); + .message("No step found for id {{step id}} in {{available step ids}}", stepId, + steps.stream().map(Step::getId).toList()) + .toString())); } } diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizer.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizer.java index 2b146b64..f8249079 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizer.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizer.java @@ -10,22 +10,30 @@ class GitHubWorkflowJavaVersionCustomizer implements GitHubWorkflowCustomizer.WorkflowCustomizer { private final List javaVersions; + private final String nextJavaVersion; - GitHubWorkflowJavaVersionCustomizer(final List javaVersions) { + GitHubWorkflowJavaVersionCustomizer(final List javaVersions, final String nextJavaVersion) { this.javaVersions = javaVersions; + this.nextJavaVersion = nextJavaVersion; } @Override public void applyCustomization(final GitHubWorkflow workflow) { - workflow.getJobs().forEach(this::updateJavaVersion); + for (final Job job : workflow.getJobs()) { + if ("next-java-compatibility".equals(job.getId())) { + updateJavaVersion(job, nextJavaVersion); + } else { + updateJavaVersion(job, formatJavaVersions()); + } + } } - private void updateJavaVersion(final Job job) { - job.getSteps().stream().filter(this::isSetupJavaStep).forEach(this::updateJavaVersion); + private void updateJavaVersion(final Job job, final String version) { + job.getSteps().stream().filter(this::isSetupJavaStep).forEach(step -> updateJavaVersion(step, version)); } - private void updateJavaVersion(final Step step) { - step.getWith().put("java-version", formatJavaVersions()); + private void updateJavaVersion(final Step step, final String version) { + step.getWith().put("java-version", version); } private String formatJavaVersions() { diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java index c9a27417..b16af519 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java @@ -7,17 +7,15 @@ class GitHubWorkflowStepCustomizer implements GitHubWorkflowCustomizer.WorkflowCustomizer { private final List customizations; - private final String jobId; - GitHubWorkflowStepCustomizer(final List customizations, final String jobId) { + GitHubWorkflowStepCustomizer(final List customizations) { this.customizations = customizations; - this.jobId = jobId; } @Override public void applyCustomization(final GitHubWorkflow workflow) { - final Job job = workflow.getJob(jobId); for (final StepCustomization customization : customizations) { + final Job job = workflow.getJob(customization.getJobId()); applyCustomization(job, customization); } } diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractor.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractor.java index 4a24fa7f..dfd94514 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractor.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractor.java @@ -39,6 +39,9 @@ private static double parseVersion(final String version) { } private Set getSourceJavaVersions() { + if (sources.isEmpty()) { + throw new IllegalStateException("No sources, can't get java versions"); + } return sources.stream().filter(AnalyzedMavenSource.class::isInstance) // .map(AnalyzedMavenSource.class::cast) // .map(AnalyzedMavenSource::getJavaVersion) // @@ -63,7 +66,8 @@ private double getCurrentLatestVersion() { return getSourceJavaVersions().stream() // .map(JavaVersionExtractor::parseVersion) // .reduce(JavaVersionExtractor::takeLast) // - .orElseThrow(); + .orElseThrow(() -> new IllegalStateException( + "No latest java version found in source java versions " + getSourceJavaVersions())); } private static double takeLast(final double first, final double second) { diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/ProjectFilesValidator.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/ProjectFilesValidator.java index 3fcd1b46..b4cb2a5a 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/ProjectFilesValidator.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/ProjectFilesValidator.java @@ -57,7 +57,7 @@ private List validateTemplatesRelativeToSource(final FileTemp } private List validateTemplatesRelativeToRepo(final FileTemplatesFactory templatesFactory) { - final List templates = templatesFactory.getGlobalTemplates(this.sources); + final List templates = templatesFactory.getGlobalTemplates(); return runValidation(templates, this.projectDirectory); } diff --git a/project-keeper/src/main/resources/non_maven_templates/.github/workflows/project-keeper-verify.yml b/project-keeper/src/main/resources/non_maven_templates/.github/workflows/project-keeper-verify.yml index 08f9390d..296232b9 100644 --- a/project-keeper/src/main/resources/non_maven_templates/.github/workflows/project-keeper-verify.yml +++ b/project-keeper/src/main/resources/non_maven_templates/.github/workflows/project-keeper-verify.yml @@ -16,6 +16,7 @@ jobs: cancel-in-progress: true steps: - name: Checkout the repository + id: checkout uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/project-keeper/src/main/resources/templates/.github/workflows/broken_links_checker.yml b/project-keeper/src/main/resources/templates/.github/workflows/broken_links_checker.yml index 39612b76..90488caa 100644 --- a/project-keeper/src/main/resources/templates/.github/workflows/broken_links_checker.yml +++ b/project-keeper/src/main/resources/templates/.github/workflows/broken_links_checker.yml @@ -5,10 +5,6 @@ name: Broken Links Checker on: schedule: - cron: "0 5 * * 0" - push: - branches: - - main - pull_request: jobs: linkChecker: diff --git a/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml b/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml index 801937f8..872a7e2b 100644 --- a/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml +++ b/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml @@ -119,8 +119,53 @@ jobs: path: ${{ steps.build-pk-verify.outputs.release-artifacts }} retention-days: 5 + - name: Configure broken links checker + id: configure-link-check + run: | + mkdir -p ./target + echo '{"aliveStatusCodes": [429, 200], "ignorePatterns": [' \ + '{"pattern": "^https?://(www|dev).mysql.com/"},' \ + '{"pattern": "^https?://(www.)?opensource.org"}' \ + '{"pattern": "^https?://(www.)?eclipse.org"}' \ + '{"pattern": "^https?://projects.eclipse.org"}' \ + ']}' > ./target/broken_links_checker.json + - uses: gaurav-nelson/github-action-markdown-link-check@v1 + id: run-link-check + with: + use-quiet-mode: "yes" + use-verbose-mode: "yes" + config-file: ./target/broken_links_checker.json + + next-java-compatibility: + runs-on: ubuntu-latest + defaults: + run: + shell: "bash" + permissions: + contents: read + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - name: Checkout the repository + id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up JDK $nextJavaVersion + id: setup-java + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: $nextJavaVersion + cache: "maven" + - name: Run tests and build with Maven $nextJavaVersion + run: mvn --batch-mode clean package -DtrimStackTrace=false -Djava.version=$nextJavaVersion + build: - needs: matrix-build + needs: + - matrix-build + - next-java-compatibility runs-on: ubuntu-latest defaults: run: @@ -132,10 +177,12 @@ jobs: release-required: ${{ steps.check-release.outputs.release-required }} steps: - name: Checkout the repository + id: checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up JDKs + id: setup-java uses: actions/setup-java@v4 with: distribution: "temurin" @@ -145,6 +192,7 @@ jobs: cache: "maven" - name: Check if release is needed id: check-release + if: ${{ github.ref == 'refs/heads/main' }} run: | if mvn --batch-mode com.exasol:project-keeper-maven-plugin:verify-release --projects .; then echo "### ✅ Release preconditions met, start release" >> "$GITHUB_STEP_SUMMARY" diff --git a/project-keeper/src/main/resources/templates/.github/workflows/ci-build-native-build.yml b/project-keeper/src/main/resources/templates/.github/workflows/ci-build-native-build.yml index 07e17a51..23aba08e 100644 --- a/project-keeper/src/main/resources/templates/.github/workflows/ci-build-native-build.yml +++ b/project-keeper/src/main/resources/templates/.github/workflows/ci-build-native-build.yml @@ -29,6 +29,7 @@ jobs: sudo rm -rf /usr/local/lib/android sudo rm -rf /usr/share/dotnet - name: Checkout the repository + id: checkout uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/project-keeper/src/main/resources/templates/.github/workflows/ci-build-next-java.yml b/project-keeper/src/main/resources/templates/.github/workflows/ci-build-next-java.yml deleted file mode 100644 index 0af95546..00000000 --- a/project-keeper/src/main/resources/templates/.github/workflows/ci-build-next-java.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Generated by Project Keeper -# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/ci-build-next-java.yml -name: CI Build next Java -on: - push: - branches: - - main - pull_request: - -jobs: - next-java-compatibility: - runs-on: ubuntu-latest - defaults: - run: - shell: "bash" - permissions: - contents: read - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - steps: - - name: Checkout the repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up JDK $nextJavaVersion - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: $nextJavaVersion - cache: "maven" - - name: Run tests and build with Maven $nextJavaVersion - run: | - mvn --batch-mode clean package -DtrimStackTrace=false $skipNativeImage -Djava.version=$nextJavaVersion diff --git a/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml b/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml index b10243b5..c774b920 100644 --- a/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml +++ b/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - build: + build-and-test: runs-on: $ciBuildRunnerOS defaults: run: @@ -110,6 +110,77 @@ jobs: path: ${{ steps.build-pk-verify.outputs.release-artifacts }} retention-days: 5 + - name: Configure link check + id: configure-link-check + run: | + mkdir -p ./target + echo '{"aliveStatusCodes": [429, 200], "ignorePatterns": [' \ + '{"pattern": "^https?://(www|dev).mysql.com/"},' \ + '{"pattern": "^https?://(www.)?opensource.org"}' \ + '{"pattern": "^https?://(www.)?eclipse.org"}' \ + '{"pattern": "^https?://projects.eclipse.org"}' \ + ']}' > ./target/broken_links_checker.json + - uses: gaurav-nelson/github-action-markdown-link-check@v1 + id: run-link-check + with: + use-quiet-mode: "yes" + use-verbose-mode: "yes" + config-file: ./target/broken_links_checker.json + + next-java-compatibility: + runs-on: ubuntu-latest + defaults: + run: + shell: "bash" + permissions: + contents: read + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - name: Checkout the repository + id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up JDK $nextJavaVersion + id: setup-java + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: $nextJavaVersion + cache: "maven" + - name: Run tests and build with Maven $nextJavaVersion + run: mvn --batch-mode clean package -DtrimStackTrace=false -Djava.version=$nextJavaVersion + + build: + needs: + - build-and-test + - next-java-compatibility + runs-on: ubuntu-latest + defaults: + run: + shell: "bash" + permissions: + contents: read + issues: read # Required for PK verify-release + outputs: + release-required: ${{ steps.check-release.outputs.release-required }} + steps: + - name: Checkout the repository + id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up JDKs + id: setup-java + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: | + 11 + 17 + cache: "maven" - name: Check if release is needed id: check-release if: ${{ github.ref == 'refs/heads/main' }} @@ -118,7 +189,7 @@ jobs: echo "### ✅ Release preconditions met, start release" >> "$GITHUB_STEP_SUMMARY" echo "release-required=true" >> "$GITHUB_OUTPUT" else - echo "### 🛑 Release precondition not met, skipping release" >> "$GITHUB_STEP_SUMMARY" + echo "### 🛑 Not all release preconditions met, skipping release" >> "$GITHUB_STEP_SUMMARY" echo "See log output for details." >> "$GITHUB_STEP_SUMMARY" echo "release-required=false" >> "$GITHUB_OUTPUT" fi diff --git a/project-keeper/src/main/resources/templates/gitattributes b/project-keeper/src/main/resources/templates/gitattributes index 59e28bad..db2c44c7 100644 --- a/project-keeper/src/main/resources/templates/gitattributes +++ b/project-keeper/src/main/resources/templates/gitattributes @@ -5,7 +5,6 @@ $pomFiles dependencies.md linguist-generated=true doc/changes/changelog.md linguist-generated=true .github/workflows/broken_links_checker.yml linguist-generated=true -.github/workflows/ci-build-next-java.yml linguist-generated=true .github/workflows/ci-build.yml linguist-generated=true .github/workflows/dependencies_check.yml linguist-generated=true .github/workflows/dependencies_update.yml linguist-generated=true diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReaderTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReaderTest.java index 942e9ca5..c51dae15 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReaderTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReaderTest.java @@ -201,7 +201,7 @@ static Stream invalidConfig() { - stepCustomizations: """, equalTo( "E-PK-CORE-199: Missing workflow name in file '.project-keeper.yml'. Add a workflow name to the workflow configuration.")), - Arguments.of("workflow: missing step id", """ + Arguments.of("workflow: missing job id", """ build: workflows: - name: ci-build.yml @@ -210,13 +210,36 @@ static Stream invalidConfig() { content: id: new-step """, equalTo( + "E-PK-CORE-208: Missing job in step customization of file '.project-keeper.yml'. Add job to the step customization.")), + Arguments.of("workflow: empty job id", """ + build: + workflows: + - name: ci-build.yml + stepCustomizations: + - job: '' + action: REPLACE + content: + id: new-step + """, equalTo( + "E-PK-CORE-208: Missing job in step customization of file '.project-keeper.yml'. Add job to the step customization.")), + Arguments.of("workflow: missing step id", """ + build: + workflows: + - name: ci-build.yml + stepCustomizations: + - job: myJob + action: REPLACE + content: + id: new-step + """, equalTo( "E-PK-CORE-201: Missing stepId in step customization of file '.project-keeper.yml'. Add stepId to the step customization.")), Arguments.of("workflow: empty step id", """ build: workflows: - name: ci-build.yml stepCustomizations: - - action: REPLACE + - job: myJob + action: REPLACE stepId: '' content: id: new-step @@ -227,7 +250,8 @@ static Stream invalidConfig() { workflows: - name: ci-build.yml stepCustomizations: - - stepId: step-id-to-replace + - job: myJob + stepId: step-id-to-replace content: id: new-step """, equalTo( @@ -237,7 +261,8 @@ static Stream invalidConfig() { workflows: - name: ci-build.yml stepCustomizations: - - action: INVALID + - job: myJob + action: INVALID stepId: step-id-to-replace content: id: new-step @@ -247,7 +272,8 @@ static Stream invalidConfig() { workflows: - name: ci-build.yml stepCustomizations: - - action: REPLACE + - job: myJob + action: REPLACE stepId: step-id-to-replace """, equalTo( "E-PK-CORE-202: Missing content in step customization of file '.project-keeper.yml'. Add content to the step customization.")), @@ -256,7 +282,8 @@ static Stream invalidConfig() { workflows: - name: ci-build.yml stepCustomizations: - - action: REPLACE + - job: myJob + action: REPLACE stepId: step-id-to-replace content: """, equalTo( @@ -291,7 +318,8 @@ void readWorkflowCustomization() throws IOException { workflows: - name: ci-build.yml stepCustomizations: - - action: REPLACE + - job: myJob + action: REPLACE stepId: step-id-to-replace content: name: New Step @@ -303,6 +331,7 @@ void readWorkflowCustomization() throws IOException { assertThat(readConfig().getCiBuildConfig().getWorkflows(), contains(CustomWorkflow.builder() // .workflowName("ci-build.yml") // .addStep(StepCustomization.builder() // + .jobId("myJob") // .type(StepCustomization.Type.REPLACE) // .stepId("step-id-to-replace") // .step(WorkflowStep.createStep(Map.of("name", "New Step", "id", "new-step", "run", @@ -319,7 +348,8 @@ void readWorkflowStepAction(final String action, final StepCustomization.Type ex workflows: - name: ci-build.yml stepCustomizations: - - action: ${action} + - job: job + action: ${action} stepId: step-id-to-replace content: id: new-step diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java index ba0fe72d..4f838a0c 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java @@ -34,62 +34,60 @@ void doesNotModifyOnTrigger() { @Test void defaultRunnerOs() { final GitHubWorkflow workflow = ciBuildContent(BuildOptions.builder()); - assertThat(workflow.getJob("build").getRunnerOS(), equalTo("ubuntu-latest")); + assertThat(workflow.getJob("build-and-test").getRunnerOS(), equalTo("ubuntu-latest")); } @Test void customRunnerOs() { final GitHubWorkflow workflow = ciBuildContent(BuildOptions.builder().runnerOs("my-runner-os")); - assertThat(workflow.getJob("build").getRunnerOS(), equalTo("my-runner-os")); + assertThat(workflow.getJob("build-and-test").getRunnerOS(), equalTo("my-runner-os")); } @ParameterizedTest @ValueSource(booleans = { true, false }) void ciBuildFreeDiskSpace(final boolean freeDiskSpace) { final GitHubWorkflow workflow = ciBuildContent(BuildOptions.builder().freeDiskSpace(freeDiskSpace)); - assertThat(workflow.getJob("build").getStep("free-disk-space").getIfCondition(), + assertThat(workflow.getJob("build-and-test").getStep("free-disk-space").getIfCondition(), equalTo("${{ " + freeDiskSpace + " }}")); } // [utest->dsn~customize-build-process.ci-build~0] @Test void customizeBuildStepsNonMatrixBuild() { - final Job job = ciBuildContent(BuildOptions.builder().workflows(List.of(CustomWorkflow.builder() - .workflowName("ci-build.yml") - .addStep(StepCustomization.builder().type(Type.INSERT_AFTER).stepId("sonar-analysis") + final Job job = ciBuildContent(BuildOptions.builder() + .workflows(List.of(CustomWorkflow.builder().workflowName("ci-build.yml").addStep(StepCustomization + .builder().jobId("build-and-test").type(Type.INSERT_AFTER).stepId("sonar-analysis") .step(WorkflowStep.createStep(Map.of("id", "inserted-step", "name", "Inserted Step"))).build()) - .build()))).getJob("build"); + .build()))) + .getJob("build-and-test"); final List stepIds = job.getSteps().stream().map(Step::getId).toList(); assertAll(() -> assertThat(job.getSteps(), hasSize(greaterThanOrEqualTo(10))), () -> assertThat(job.getStep("inserted-step").getName(), equalTo("Inserted Step")), - () -> assertThat(stepIds, - contains("free-disk-space", "checkout", "setup-java", "cache-sonar", - "enable-testcontainer-reuse", "build-pk-verify", "sonar-analysis", "inserted-step", - "verify-release-artifacts", "upload-artifacts", "check-release"))); + () -> assertThat(stepIds, contains("free-disk-space", "checkout", "setup-java", "cache-sonar", + "enable-testcontainer-reuse", "build-pk-verify", "sonar-analysis", "inserted-step", + "verify-release-artifacts", "upload-artifacts", "configure-link-check", "run-link-check"))); } // [utest->dsn~customize-build-process.ci-build~0] @Test void customizeBuildStepsMatrixBuild() { final Job job = ciBuildContent(BuildOptions.builder().exasolDbVersions(List.of("v1", "v2")) - .workflows(List.of(CustomWorkflow.builder().workflowName("ci-build.yml") - .addStep(StepCustomization.builder().type(Type.INSERT_AFTER).stepId("sonar-analysis") - .step(WorkflowStep.createStep(Map.of("id", "inserted-step", "name", "Inserted Step"))) - .build()) + .workflows(List.of(CustomWorkflow.builder().workflowName("ci-build.yml").addStep(StepCustomization + .builder().jobId("matrix-build").type(Type.INSERT_AFTER).stepId("sonar-analysis") + .step(WorkflowStep.createStep(Map.of("id", "inserted-step", "name", "Inserted Step"))).build()) .build()))) .getJob("matrix-build"); final List stepIds = job.getSteps().stream().map(Step::getId).toList(); assertAll(() -> assertThat(job.getSteps(), hasSize(greaterThanOrEqualTo(10))), () -> assertThat(job.getStep("inserted-step").getName(), equalTo("Inserted Step")), - () -> assertThat(stepIds, - contains("free-disk-space", "checkout", "setup-java", "cache-sonar", - "enable-testcontainer-reuse", "build-pk-verify", "sonar-analysis", "inserted-step", - "verify-release-artifacts", "upload-artifacts"))); + () -> assertThat(stepIds, contains("free-disk-space", "checkout", "setup-java", "cache-sonar", + "enable-testcontainer-reuse", "build-pk-verify", "sonar-analysis", "inserted-step", + "verify-release-artifacts", "upload-artifacts", "configure-link-check", "run-link-check"))); } @Test void ciBuildContentNonMatrixBuild() { - final Job job = ciBuildContent(BuildOptions.builder()).getJob("build"); + final Job job = ciBuildContent(BuildOptions.builder()).getJob("build-and-test"); assertAll( () -> assertThat(job.getConcurrency(), equalTo(Map.of("group", "${{ github.workflow }}-${{ github.ref }}", "cancel-in-progress", @@ -138,13 +136,13 @@ void ciBuildMatrixBuildSingleVersion() { void ciBuildCustomEnvironment() { final Job job = ciBuildContent(BuildOptions.builder() .workflows(List.of(CustomWorkflow.builder().workflowName("ci-build.yml").environment("aws").build()))) - .getJob("build"); + .getJob("build-and-test"); assertThat(job.getEnvironment(), equalTo("aws")); } @Test void ciBuildDefaultHasNoEnvironment() { - final Job job = ciBuildContent(BuildOptions.builder()).getJob("build"); + final Job job = ciBuildContent(BuildOptions.builder()).getJob("build-and-test"); assertThat(job.getEnvironment(), nullValue()); } @@ -184,7 +182,7 @@ void releaseBuildWithoutMavenRelease() { void releaseBuildCustomized() { final Job job = releaseBuildContent(BuildOptions.builder() .workflows(List.of(CustomWorkflow.builder().workflowName("release.yml") - .addStep(StepCustomization.builder().type(Type.INSERT_AFTER).stepId("build") + .addStep(StepCustomization.builder().jobId("release").type(Type.INSERT_AFTER).stepId("build") .step(WorkflowStep.createStep(Map.of("id", "new-step", "name", "New Step"))).build()) .build())) .build()).getJob("release"); @@ -197,7 +195,8 @@ void releaseBuildCustomized() { void dependencyCheckBuildCustomized() { final Job job = dependencyCheckBuildContent(BuildOptions.builder() .workflows(List.of(CustomWorkflow.builder().workflowName("dependencies_check.yml") - .addStep(StepCustomization.builder().type(Type.INSERT_AFTER).stepId("setup-jdks") + .addStep(StepCustomization.builder().jobId("report_security_issues").type(Type.INSERT_AFTER) + .stepId("setup-jdks") .step(WorkflowStep.createStep(Map.of("id", "new-step", "name", "New Step"))).build()) .build())) .build()).getJob("report_security_issues"); @@ -210,7 +209,8 @@ void dependencyCheckBuildCustomized() { void dependencyUpdateBuildCustomized() { final Job job = dependencyUpdateBuildContent(BuildOptions.builder() .workflows(List.of(CustomWorkflow.builder().workflowName("dependencies_update.yml") - .addStep(StepCustomization.builder().type(Type.INSERT_AFTER).stepId("setup-jdks") + .addStep(StepCustomization.builder().jobId("update_dependencies").type(Type.INSERT_AFTER) + .stepId("setup-jdks") .step(WorkflowStep.createStep(Map.of("id", "new-step", "name", "New Step"))).build()) .build())) .build()).getJob("update_dependencies"); @@ -227,7 +227,7 @@ void ciBuildMatrixBuildAllStepsHaveId() { @Test void ciBuildNonMatrixAllStepsHaveId() { - final Job job = ciBuildContent(BuildOptions.builder()).getJob("build"); + final Job job = ciBuildContent(BuildOptions.builder()).getJob("build-and-test"); assertThat(job.getSteps(), hasSize(greaterThanOrEqualTo(10))); job.getSteps().forEach(step -> assertThat(step.getId(), notNullValue())); } @@ -256,13 +256,12 @@ void dependencyUpdateBuildAllStepsHaveId() { @Test void customizeSetupJavaStepInCiBuild() { final Map setupJavaStep = setupJavaStep("custom-version"); - final Job job = ciBuildContent( - BuildOptions.builder() - .workflows(List.of(CustomWorkflow.builder().workflowName("ci-build.yml") - .addStep(StepCustomization.builder().type(Type.REPLACE).stepId("setup-java") - .step(WorkflowStep.createStep(setupJavaStep)).build()) - .build()))) - .getJob("build"); + final Job job = ciBuildContent(BuildOptions.builder() + .workflows(List.of(CustomWorkflow.builder().workflowName("ci-build.yml") + .addStep(StepCustomization.builder().jobId("build-and-test").type(Type.REPLACE) + .stepId("setup-java").step(WorkflowStep.createStep(setupJavaStep)).build()) + .build()))) + .getJob("build-and-test"); final String customJavaVersion = (String) job.getStep("setup-java").getWith().get("java-version"); assertThat(customJavaVersion, equalTo("custom-version")); } @@ -272,7 +271,7 @@ void customizeSetupJavaStepInReleaseBuild() { final Map setupJavaStep = setupJavaStep("custom-version"); final Job job = releaseBuildContent(BuildOptions.builder() .workflows(List.of(CustomWorkflow.builder().workflowName("release.yml") - .addStep(StepCustomization.builder().type(Type.REPLACE).stepId("setup-jdks") + .addStep(StepCustomization.builder().jobId("release").type(Type.REPLACE).stepId("setup-jdks") .step(WorkflowStep.createStep(setupJavaStep)).build()) .build())) .build()).getJob("release"); @@ -347,7 +346,7 @@ private void validateYamlSyntax(final String content) { } private CiBuildWorkflowGenerator testee(final BuildOptions options) { - return new CiBuildWorkflowGenerator(options, List.of("11", "17")); + return new CiBuildWorkflowGenerator(options, List.of("11", "17"), "21"); } private GitHubWorkflow parse(final String yaml) { diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactoryTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactoryTest.java index 8097c54c..e2003054 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactoryTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactoryTest.java @@ -1,6 +1,5 @@ package com.exasol.projectkeeper.validators.files; -import static java.util.Collections.emptyList; import static java.util.Collections.emptySet; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; @@ -35,7 +34,7 @@ class FileTemplatesFactoryTest { void testGetCiBuildTemplatesForMavenProject() { final Set modules = Collections.emptySet(); final List sources = getMavenSourceWithModules(modules); - final List templates = testee().getGlobalTemplates(sources); + final List templates = testee(sources).getGlobalTemplates(); assertContainsTemplate(templates, ".github/workflows/ci-build.yml"); assertContainsTemplate(templates, ".vscode/settings.json"); final Optional gitattributes = findTemplate(templates, ".gitattributes"); @@ -48,22 +47,22 @@ void testGetCiBuildTemplatesForMavenProject() { void testGenerateDependencyUpdateWorkflow() { final Set modules = Collections.emptySet(); final List sources = getMavenSourceWithModules(modules); - final List templates = testee().getGlobalTemplates(sources); + final List templates = testee(sources).getGlobalTemplates(); assertContainsTemplate(templates, ".github/workflows/dependencies_check.yml"); assertContainsTemplate(templates, ".github/workflows/dependencies_update.yml"); } - private FileTemplatesFactory testee() { - return testee(true); + private FileTemplatesFactory testee(final List sources) { + return testee(true, sources); } - private FileTemplatesFactory testee(final boolean hasNpmModule) { + private FileTemplatesFactory testee(final boolean hasNpmModule, final List sources) { return new FileTemplatesFactory(this.loggerMock, OWN_VERSION, hasNpmModule, - BuildOptions.builder().runnerOs("ci-build-runner-os").build(), emptyList()); + BuildOptions.builder().runnerOs("ci-build-runner-os").build(), sources); } private List getMavenSourceWithModules(final Set modules) { - return List.of(AnalyzedMavenSource.builder().modules(modules).isRootProject(true).build()); + return List.of(AnalyzedMavenSource.builder().modules(modules).isRootProject(true).javaVersion("11").build()); } /* @@ -74,7 +73,7 @@ private List getMavenSourceWithModules(final Set sources = List .of(AnalyzedMavenSource.builder().modules(Collections.emptySet()).isRootProject(false).build()); - final List templates = testee().getGlobalTemplates(sources); + final List templates = testee(sources).getGlobalTemplates(); assertFalse(() -> templates.stream() .anyMatch(template -> template.getPathInProject().equals(Path.of(".github/workflows/ci-build.yml")))); verify(this.loggerMock).warn( @@ -84,7 +83,7 @@ void testGetCiBuildTemplatesForNonAggregatedMvnProject() { @Test void testReleaseWorkflow() { final List sources = getMavenSourceWithModules(Set.of()); - final List templates = testee().getGlobalTemplates(sources); + final List templates = testee(sources).getGlobalTemplates(); assertContainsTemplate(templates, ".github/workflows/release.yml"); } @@ -93,7 +92,7 @@ void testReleaseWorkflow() { void testProjectKeeperVerifyYml(final boolean hasNpmModule) { final List sources = List.of(AnalyzedMavenSource.builder() // .modules(Collections.emptySet()).isRootProject(false).build()); - final List actual = testee(hasNpmModule).getGlobalTemplates(sources); + final List actual = testee(hasNpmModule, sources).getGlobalTemplates(); final Optional template = findTemplate(actual, ".github/workflows/project-keeper-verify.yml"); assertTrue(template.isPresent()); final String content = template.get().getContent(); @@ -107,12 +106,12 @@ void testProjectKeeperVerifyYml(final boolean hasNpmModule) { void testCiBuildNextJava() { final List sources = List .of(AnalyzedMavenSource.builder().javaVersion("17").isRootProject(true).modules(emptySet()).build()); - final List actual = testee(false).getGlobalTemplates(sources); - final Optional template = findTemplate(actual, ".github/workflows/ci-build-next-java.yml"); + final List actual = testee(false, sources).getGlobalTemplates(); + final Optional template = findTemplate(actual, ".github/workflows/ci-build.yml"); assertTrue(template.isPresent()); final String content = template.get().getContent(); - assertThat(content, allOf(not(containsString("skipNativeImage")), // - containsString("java-version: '21',"), containsString("-Djava.version=21"))); + System.out.println(content); + assertThat(content, allOf(containsString("java-version: '21',"), containsString("-Djava.version=21"))); } @ParameterizedTest @@ -124,7 +123,7 @@ void testCiBuildNextJava() { }) void testGetTemplatesPerSource(final ProjectKeeperModule module, final String expectedTemplate) { final AnalyzedMavenSource source = AnalyzedMavenSource.builder().modules(Set.of(module)).build(); - final List templates = testee().getTemplatesForSource(source); + final List templates = testee(List.of(source)).getTemplatesForSource(source); assertContainsTemplate(templates, expectedTemplate); } @@ -135,7 +134,7 @@ void testSettingsOrgEclipseJdtUiPrefs(final String javaVersion, final String exp .modules(Set.of(ProjectKeeperModule.DEFAULT)) // .javaVersion(javaVersion) // .build(); - final List templates = testee().getTemplatesForSource(source); + final List templates = testee(List.of(source)).getTemplatesForSource(source); final FileTemplate template = findTemplate(templates, ".settings/org.eclipse.jdt.ui.prefs").get(); template.getContent().contains("org.eclipse.jdt.core.compiler.codegen.targetPlatform=" + expected); } diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizerTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizerTest.java index d1515d6b..c894e41b 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizerTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizerTest.java @@ -24,7 +24,7 @@ void testUpdateWithActionVersion4() { old version cache: "maven" - """, List.of("v1")); + """, List.of("v1"), "v2"); assertThat(workflow.getJob("build").getStep("setup-java").getWith().get("java-version"), equalTo("v1")); } @@ -41,7 +41,7 @@ void testUpdateWithActionVersion5() { distribution: "temurin" java-version: old cache: "maven" - """, List.of("v1")); + """, List.of("v1"), "v2"); assertThat(workflow.getJob("build").getStep("setup-java").getWith().get("java-version"), equalTo("v1")); } @@ -55,7 +55,7 @@ void testUpdateEmptyJavaVersion() { id: setup-java uses: actions/setup-java@v4 with: - """, List.of("v1")); + """, List.of("v1"), "v2"); assertThat(workflow.getJob("build").getStep("setup-java").getWith().get("java-version"), equalTo("v1")); } @@ -74,7 +74,7 @@ void testUpdateToMultipleJavaVersions() { old version cache: "maven" - """, List.of("11", "17", "21")); + """, List.of("11", "17", "21"), "23"); assertThat(workflow.getJob("build").getStep("setup-java").getWith().get("java-version"), equalTo("11\n17\n21")); } @@ -89,7 +89,7 @@ void testNoUpdateOfUnknownAction() { uses: actions/unknown@v4 with: java-version: old version - """, List.of("11", "17", "21")); + """, List.of("11", "17", "21"), "23"); assertThat(workflow.getJob("build").getStep("setup-java").getWith().get("java-version"), equalTo("old version")); } @@ -103,13 +103,13 @@ void ignoreStepWithoutUsesDirective() { - name: Set up JDKs id: setup-java run: echo Hello - """, List.of("11", "17", "21")); + """, List.of("11", "17", "21"), "23"); assertThat(workflow.getJob("build").getStep("setup-java").getWith().size(), is(0)); } - GitHubWorkflow customize(final String yaml, final List javaVersions) { + GitHubWorkflow customize(final String yaml, final List javaVersions, final String nextJavaVersion) { final GitHubWorkflow workflow = GitHubWorkflowIO.create().loadWorkflow(yaml); - new GitHubWorkflowJavaVersionCustomizer(javaVersions).applyCustomization(workflow); + new GitHubWorkflowJavaVersionCustomizer(javaVersions, nextJavaVersion).applyCustomization(workflow); return workflow; } } diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java index 6e2d0ba8..165cf187 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java @@ -95,7 +95,8 @@ void replaceStep() { id: replaced-step - name: step2 id: step2 - """, StepCustomization.builder().type(Type.REPLACE).stepId("replaced-step").step(newStep).build()); + """, StepCustomization.builder().jobId("build").type(Type.REPLACE).stepId("replaced-step").step(newStep) + .build()); assertThat(getStepIds(workflow.getJob("build")), contains("step0", "custom-step", "step2")); } @@ -112,7 +113,8 @@ void replaceStepNextStepHasMissingId() { - name: step1 id: replaced-step - name: step2 - """, StepCustomization.builder().type(Type.REPLACE).stepId("replaced-step").step(newStep).build()); + """, StepCustomization.builder().jobId("build").type(Type.REPLACE).stepId("replaced-step").step(newStep) + .build()); assertThat(getStepNames(workflow.getJob("build")), contains("step0", "Custom Step", "step2")); } @@ -120,8 +122,8 @@ void replaceStepNextStepHasMissingId() { void replaceStepWrongBuildStepId() { final WorkflowStep customBuildStep = WorkflowStep .createStep(Map.of("name", "Custom Step", "id", "custom-step", "run", "echo custom-step")); - final StepCustomization customization = StepCustomization.builder().type(Type.REPLACE).stepId("missing-step") - .step(customBuildStep).build(); + final StepCustomization customization = StepCustomization.builder().jobId("build").type(Type.REPLACE) + .stepId("missing-step").step(customBuildStep).build(); final IllegalStateException exception = assertThrows(IllegalStateException.class, () -> validate(""" jobs: build: @@ -130,14 +132,14 @@ void replaceStepWrongBuildStepId() { id: wrong-id """, customization)); assertThat(exception.getMessage(), - equalTo("E-PK-CORE-205: No step found for id 'missing-step' in {steps=[{name=step1, id=wrong-id}]}")); + equalTo("E-PK-CORE-205: No step found for id 'missing-step' in ['wrong-id']")); } @Test void insertStepWrongBuildStepId() { final WorkflowStep customBuildStep = WorkflowStep .createStep(Map.of("name", "Custom Step", "id", "custom-step", "run", "echo custom-step")); - final StepCustomization customization = StepCustomization.builder().type(Type.INSERT_AFTER) + final StepCustomization customization = StepCustomization.builder().jobId("build").type(Type.INSERT_AFTER) .stepId("missing-step").step(customBuildStep).build(); final IllegalStateException exception = assertThrows(IllegalStateException.class, () -> validate(""" jobs: @@ -147,7 +149,7 @@ void insertStepWrongBuildStepId() { id: wrong-id """, customization)); assertThat(exception.getMessage(), - equalTo("E-PK-CORE-205: No step found for id 'missing-step' in {steps=[{name=step1, id=wrong-id}]}")); + equalTo("E-PK-CORE-205: No step found for id 'missing-step' in ['wrong-id']")); } // [utest->dsn~customize-build-process.insert-step-after~0] @@ -163,7 +165,8 @@ void insertStepInTheMiddle() { id: step0 - name: Step 1 id: step1 - """, StepCustomization.builder().type(Type.INSERT_AFTER).stepId("step0").step(customBuildStep).build()); + """, StepCustomization.builder().jobId("build").type(Type.INSERT_AFTER).stepId("step0") + .step(customBuildStep).build()); assertThat(getStepIds(workflow.getJob("build")), contains("step0", "custom-step", "step1")); } @@ -179,7 +182,8 @@ void insertStepAtTheEnd() { id: step0 - name: Step 1 id: step1 - """, StepCustomization.builder().type(Type.INSERT_AFTER).stepId("step1").step(customBuildStep).build()); + """, StepCustomization.builder().jobId("build").type(Type.INSERT_AFTER).stepId("step1") + .step(customBuildStep).build()); assertThat(getStepIds(workflow.getJob("build")), contains("step0", "step1", "custom-step")); } @@ -195,8 +199,11 @@ void insertStepAfterReplacedStep() { id: step0 - name: Step 1 id: step1 - """, StepCustomization.builder().type(Type.REPLACE).stepId("step0").step(replacedStep).build(), - StepCustomization.builder().type(Type.INSERT_AFTER).stepId("replaced-step").step(insertedStep).build()); + """, + StepCustomization.builder().jobId("build").type(Type.REPLACE).stepId("step0").step(replacedStep) + .build(), + StepCustomization.builder().jobId("build").type(Type.INSERT_AFTER).stepId("replaced-step") + .step(insertedStep).build()); assertThat(getStepIds(workflow.getJob("build")), contains("replaced-step", "inserted-step", "step1")); } @@ -214,7 +221,7 @@ private GitHubWorkflow validate(final String workflowTemplate, final StepCustomi } private String getCustomizedContent(final String workflowTemplate, final StepCustomization... customizations) { - return new GitHubWorkflowCustomizer(new GitHubWorkflowStepCustomizer(asList(customizations), "build")) + return new GitHubWorkflowCustomizer(new GitHubWorkflowStepCustomizer(asList(customizations))) .customizeContent(workflowTemplate); } } diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowTest.java index 6d3995a9..b9170c6e 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowTest.java @@ -288,8 +288,8 @@ void replaceStepMissing() { """).getJob("build"); final IllegalStateException exception = assertThrows(IllegalStateException.class, () -> job.replaceStep("missing", null)); - assertThat(exception.getMessage(), equalTo( - "E-PK-CORE-205: No step found for id 'missing' in {steps=[{id=step-to-replace, name=Old Step}]}")); + assertThat(exception.getMessage(), + equalTo("E-PK-CORE-205: No step found for id 'missing' in ['step-to-replace']")); } @Test @@ -318,8 +318,8 @@ void insertStepMissing() { """).getJob("build"); final IllegalStateException exception = assertThrows(IllegalStateException.class, () -> job.insertStepAfter("missing", null)); - assertThat(exception.getMessage(), equalTo( - "E-PK-CORE-205: No step found for id 'missing' in {steps=[{id=step-to-replace, name=Old Step}]}")); + assertThat(exception.getMessage(), + equalTo("E-PK-CORE-205: No step found for id 'missing' in ['step-to-replace']")); } @Test diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java index 8f8407fb..d79b07df 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java @@ -15,11 +15,6 @@ class JavaVersionExtractorTest { - @Test - void noSources() { - assertThat(extract(), contains("17")); - } - @Test void nonJavaSource() { assertThat(extract(AnalyzedSourceImpl.builder().build()), contains("17")); diff --git a/shared-model-classes/src/main/java/com/exasol/projectkeeper/shared/config/workflow/StepCustomization.java b/shared-model-classes/src/main/java/com/exasol/projectkeeper/shared/config/workflow/StepCustomization.java index 884f7c0b..0f0c2318 100644 --- a/shared-model-classes/src/main/java/com/exasol/projectkeeper/shared/config/workflow/StepCustomization.java +++ b/shared-model-classes/src/main/java/com/exasol/projectkeeper/shared/config/workflow/StepCustomization.java @@ -7,16 +7,27 @@ */ public final class StepCustomization { + private final String jobId; private final Type type; private final String stepId; private final WorkflowStep step; private StepCustomization(final Builder builder) { + this.jobId = Objects.requireNonNull(builder.jobId, "jobId"); this.type = Objects.requireNonNull(builder.type, "type"); this.stepId = Objects.requireNonNull(builder.stepId, "stepId"); this.step = Objects.requireNonNull(builder.step, "step"); } + /** + * Get the job ID. + * + * @return job ID + */ + public String getJobId() { + return jobId; + } + /** * Get the type of customization. * @@ -55,6 +66,7 @@ public static Builder builder() { /** Builder class for {@link StepCustomization} instances. */ public static class Builder { + private String jobId; private Type type; private String stepId; private WorkflowStep step; @@ -62,6 +74,17 @@ public static class Builder { private Builder() { } + /** + * Set the ID of the build job to customize. + * + * @param jobId job ID + * @return {@code this} for fluent programming + */ + public Builder jobId(final String jobId) { + this.jobId = jobId; + return this; + } + /** * Set the type of customization. * @@ -107,12 +130,12 @@ public StepCustomization build() { @Override public String toString() { - return "StepCustomization [type=" + type + ", stepId=" + stepId + ", step=" + step + "]"; + return "StepCustomization [jobId=" + jobId + ", type=" + type + ", stepId=" + stepId + ", step=" + step + "]"; } @Override public int hashCode() { - return Objects.hash(type, stepId, step); + return Objects.hash(jobId, type, stepId, step); } @Override @@ -127,7 +150,8 @@ public boolean equals(final Object obj) { return false; } final StepCustomization other = (StepCustomization) obj; - return type == other.type && Objects.equals(stepId, other.stepId) && Objects.equals(step, other.step); + return Objects.equals(jobId, other.jobId) && type == other.type && Objects.equals(stepId, other.stepId) + && Objects.equals(step, other.step); } /** diff --git a/shared-model-classes/src/test/java/com/exasol/projectkeeper/shared/config/workflow/CustomWorkflowTest.java b/shared-model-classes/src/test/java/com/exasol/projectkeeper/shared/config/workflow/CustomWorkflowTest.java index a6d8af41..037be2fc 100644 --- a/shared-model-classes/src/test/java/com/exasol/projectkeeper/shared/config/workflow/CustomWorkflowTest.java +++ b/shared-model-classes/src/test/java/com/exasol/projectkeeper/shared/config/workflow/CustomWorkflowTest.java @@ -34,14 +34,16 @@ void builderDefaultCustomizationEmpty() { @Test void builderAddCustomization() { final CustomWorkflow options = CustomWorkflow.builder().workflowName("name").addStep(StepCustomization.builder() - .type(Type.REPLACE).stepId("stepId").step(WorkflowStep.createStep(emptyMap())).build()).build(); + .jobId("myJob").type(Type.REPLACE).stepId("stepId").step(WorkflowStep.createStep(emptyMap())).build()) + .build(); assertThat(options.getSteps(), hasSize(1)); } @Test void builderSetCustomizations() { - final CustomWorkflow options = CustomWorkflow.builder().workflowName("name").steps(List.of(StepCustomization - .builder().type(Type.REPLACE).stepId("stepId").step(WorkflowStep.createStep(emptyMap())).build())) + final CustomWorkflow options = CustomWorkflow + .builder().workflowName("name").steps(List.of(StepCustomization.builder().jobId("myJob") + .type(Type.REPLACE).stepId("stepId").step(WorkflowStep.createStep(emptyMap())).build())) .build(); assertThat(options.getSteps(), hasSize(1)); } From 27f90a4d52e93b83167686195d745fe4682fdb5c Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 16:50:18 +0100 Subject: [PATCH 02/14] Improve usability: add available jobs to error message --- .github/workflows/ci-build.yml | 86 ++++++++++++++++++- .project-keeper.yml | 6 ++ .../config/ProjectKeeperConfigReader.java | 5 -- .../files/CiBuildWorkflowGenerator.java | 4 +- .../validators/files/GitHubWorkflow.java | 5 ++ .../files/GitHubWorkflowStepCustomizer.java | 24 +++++- .../config/ProjectKeeperConfigReaderTest.java | 48 ++++++----- .../GitHubWorkflowStepCustomizerTest.java | 44 +++++++++- .../config/workflow/StepCustomization.java | 7 +- 9 files changed, 193 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 41b11332..88d54afd 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -9,7 +9,7 @@ on: pull_request: null workflow_dispatch: null jobs: - build: + build-and-test: runs-on: ubuntu-latest defaults: run: { @@ -119,6 +119,88 @@ jobs: path: '${{ steps.build-pk-verify.outputs.release-artifacts }}', retention-days: 5 } + - name: Configure link check + id: configure-link-check + run: | + mkdir -p ./target + echo '{"aliveStatusCodes": [429, 200], "ignorePatterns": [' \ + '{"pattern": "^https?://(www|dev).mysql.com/"},' \ + '{"pattern": "^https?://(www.)?opensource.org"}' \ + '{"pattern": "^https?://(www.)?eclipse.org"}' \ + '{"pattern": "^https?://projects.eclipse.org"}' \ + ']}' > ./target/broken_links_checker.json + - uses: gaurav-nelson/github-action-markdown-link-check@v1 + id: run-link-check + with: { + use-quiet-mode: yes, + use-verbose-mode: yes, + config-file: ./target/broken_links_checker.json + } + next-java-compatibility: + runs-on: ubuntu-latest + defaults: + run: { + shell: bash + } + permissions: { + contents: read + } + concurrency: { + group: '${{ github.workflow }}-${{ github.ref }}', + cancel-in-progress: true + } + steps: + - name: Checkout the repository + id: checkout + uses: actions/checkout@v4 + with: { + fetch-depth: 0 + } + - name: Set up JDK 21 + id: setup-java + uses: actions/setup-java@v4 + with: { + distribution: temurin, + java-version: '21', + cache: maven + } + - { + name: Run tests and build with Maven 21, + run: mvn --batch-mode clean package -DtrimStackTrace=false -Djava.version=21 + } + build: + needs: [ + build-and-test, + next-java-compatibility + ] + runs-on: ubuntu-latest + defaults: + run: { + shell: bash + } + permissions: { + contents: read, + issues: read + } + outputs: { + release-required: '${{ steps.check-release.outputs.release-required }}' + } + steps: + - name: Checkout the repository + id: checkout + uses: actions/checkout@v4 + with: { + fetch-depth: 0 + } + - name: Set up JDKs + id: setup-java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: |- + 11 + 17 + cache: maven - name: Check if release is needed id: check-release if: ${{ github.ref == 'refs/heads/main' }} @@ -127,7 +209,7 @@ jobs: echo "### ✅ Release preconditions met, start release" >> "$GITHUB_STEP_SUMMARY" echo "release-required=true" >> "$GITHUB_OUTPUT" else - echo "### 🛑 Release precondition not met, skipping release" >> "$GITHUB_STEP_SUMMARY" + echo "### 🛑 Not all release preconditions met, skipping release" >> "$GITHUB_STEP_SUMMARY" echo "See log output for details." >> "$GITHUB_STEP_SUMMARY" echo "release-required=false" >> "$GITHUB_OUTPUT" fi diff --git a/.project-keeper.yml b/.project-keeper.yml index 9f0bd284..8535ebea 100644 --- a/.project-keeper.yml +++ b/.project-keeper.yml @@ -89,6 +89,7 @@ build: - name: "ci-build.yml" stepCustomizations: - action: INSERT_AFTER + job: build-and-test stepId: setup-java content: name: Set up Go @@ -98,12 +99,14 @@ build: go-version: "1.22" cache-dependency-path: .project-keeper.yml - action: INSERT_AFTER + job: build-and-test stepId: setup-go content: name: Install Go tools id: install-go-tools run: go install github.com/google/go-licenses@v1.6.0 - action: REPLACE + job: build-and-test stepId: build-pk-verify content: name: Run tests and build with Maven @@ -115,12 +118,14 @@ build: env: GITHUB_TOKEN: ${{ github.token }} # Required for integration tests - action: INSERT_AFTER + job: build-and-test stepId: sonar-analysis content: name: Run project-keeper itself id: build-pk-verify run: mvn --batch-mode com.exasol:project-keeper-maven-plugin:verify --projects . - action: INSERT_AFTER + job: build-and-test stepId: build-pk-verify content: name: Verify that metrics.json was created @@ -134,6 +139,7 @@ build: - name: "dependencies_check.yml" stepCustomizations: - action: INSERT_AFTER + job: report_security_issues stepId: setup-jdks content: name: Install Project Keeper diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReader.java b/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReader.java index d92ceeab..ea5f3823 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReader.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReader.java @@ -170,11 +170,6 @@ private List convertSteps(final List st } private StepCustomization convertStep(final RawStepCustomization step) { - if (step.job == null || step.job.isBlank()) { - throw new IllegalArgumentException(ExaError.messageBuilder("E-PK-CORE-208") - .message("Missing job in step customization of file {{config file}}.", CONFIG_FILE_NAME) - .mitigation("Add job to the step customization.").toString()); - } if (step.action == null) { throw new IllegalArgumentException(ExaError.messageBuilder("E-PK-CORE-200") .message("Missing action in step customization of file {{config file}}.", CONFIG_FILE_NAME) diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java index db8c6f60..19cff0d0 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java @@ -54,7 +54,7 @@ private FileTemplate createTemplate(final FileTemplateFromResource template, fin return new ContentCustomizingTemplate(template, new GitHubWorkflowCustomizer( // javaVersionCustomizer(), // [impl->dsn~customize-build-process.ci-build~0] - new GitHubWorkflowStepCustomizer(customizations), + new GitHubWorkflowStepCustomizer(workflowName, customizations), // [impl->dsn~customize-build-process.ci-build.environment~1] new GitHubWorkflowEnvironmentCustomizer(buildJobId, environmentName))); } @@ -119,7 +119,7 @@ private FileTemplate createCustomizedWorkflow(final String workflowName, templateCustomizer.accept(template); final List customizations = findCustomizations(workflowName); return new ContentCustomizingTemplate(template, new GitHubWorkflowCustomizer(javaVersionCustomizer(), - new GitHubWorkflowStepCustomizer(customizations))); + new GitHubWorkflowStepCustomizer(workflowName, customizations))); } enum CiTemplateType { diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java index 201ff444..de6983f7 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java @@ -197,4 +197,9 @@ private Optional getOptionalString(final String key) { return Optional.ofNullable((String) rawStep.get(key)); } } + + public Job getJob(final Optional jobId) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'getJob'"); + } } diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java index b16af519..c2bf40d4 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java @@ -2,24 +2,44 @@ import java.util.List; +import com.exasol.errorreporting.ExaError; +import com.exasol.projectkeeper.config.ProjectKeeperConfigReader; import com.exasol.projectkeeper.shared.config.workflow.StepCustomization; import com.exasol.projectkeeper.validators.files.GitHubWorkflow.Job; class GitHubWorkflowStepCustomizer implements GitHubWorkflowCustomizer.WorkflowCustomizer { + private final String workflowName; private final List customizations; - GitHubWorkflowStepCustomizer(final List customizations) { + GitHubWorkflowStepCustomizer(final String workflowName, final List customizations) { + this.workflowName = workflowName; this.customizations = customizations; } @Override public void applyCustomization(final GitHubWorkflow workflow) { for (final StepCustomization customization : customizations) { - final Job job = workflow.getJob(customization.getJobId()); + final Job job = getJob(workflow, customization); applyCustomization(job, customization); } } + private Job getJob(final GitHubWorkflow workflow, final StepCustomization customization) { + if (customization.getJobId().isPresent()) { + return workflow.getJob(customization.getJobId().get()); + } + final List allJobs = workflow.getJobs(); + if (allJobs.size() == 1) { + return allJobs.get(0); + } + throw new IllegalStateException(ExaError.messageBuilder("E-PK-CORE-208") + .message("Missing job in step customization of workflow {{workflow}} in file {{config file}}.", + workflowName, ProjectKeeperConfigReader.CONFIG_FILE_NAME) + .mitigation("Add job with one of the following values: {{available jobs}}.", + allJobs.stream().map(Job::getId).toList()) + .toString()); + } + // [impl->dsn~customize-build-process.insert-step-after~0] // [impl->dsn~customize-build-process.replace-step~0] private void applyCustomization(final Job job, final StepCustomization customization) { diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReaderTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReaderTest.java index c51dae15..86cfe2a5 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReaderTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReaderTest.java @@ -201,27 +201,6 @@ static Stream invalidConfig() { - stepCustomizations: """, equalTo( "E-PK-CORE-199: Missing workflow name in file '.project-keeper.yml'. Add a workflow name to the workflow configuration.")), - Arguments.of("workflow: missing job id", """ - build: - workflows: - - name: ci-build.yml - stepCustomizations: - - action: REPLACE - content: - id: new-step - """, equalTo( - "E-PK-CORE-208: Missing job in step customization of file '.project-keeper.yml'. Add job to the step customization.")), - Arguments.of("workflow: empty job id", """ - build: - workflows: - - name: ci-build.yml - stepCustomizations: - - job: '' - action: REPLACE - content: - id: new-step - """, equalTo( - "E-PK-CORE-208: Missing job in step customization of file '.project-keeper.yml'. Add job to the step customization.")), Arguments.of("workflow: missing step id", """ build: workflows: @@ -340,6 +319,33 @@ void readWorkflowCustomization() throws IOException { .build())); } + @Test + void jobIdIsOptional() throws IOException { + writeProjectKeeperConfig(""" + build: + workflows: + - name: ci-build.yml + stepCustomizations: + - action: REPLACE + stepId: step-id-to-replace + content: + name: New Step + id: new-step + run: echo 'new step' + env: + ENV_VARIABLE: 'value' + """); + assertThat(readConfig().getCiBuildConfig().getWorkflows(), contains(CustomWorkflow.builder() // + .workflowName("ci-build.yml") // + .addStep(StepCustomization.builder() // + .type(StepCustomization.Type.REPLACE) // + .stepId("step-id-to-replace") // + .step(WorkflowStep.createStep(Map.of("name", "New Step", "id", "new-step", "run", + "echo 'new step'", "env", Map.of("ENV_VARIABLE", "value")))) // + .build()) // + .build())); + } + @ParameterizedTest @CsvSource({ "REPLACE, REPLACE", "INSERT_AFTER, INSERT_AFTER" }) void readWorkflowStepAction(final String action, final StepCustomization.Type expected) throws IOException { diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java index 165cf187..1df0a8af 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java @@ -19,6 +19,8 @@ class GitHubWorkflowStepCustomizerTest { + private static final String WORKFLOW_NAME = "workflow.yml"; + @Test void startsWithGeneratedComment() { final String yaml = getCustomizedContent(""" @@ -100,6 +102,46 @@ void replaceStep() { assertThat(getStepIds(workflow.getJob("build")), contains("step0", "custom-step", "step2")); } + @Test + void missingJobConfigurationWithSingleJob() { + final WorkflowStep newStep = WorkflowStep + .createStep(Map.of("name", "Custom Step", "id", "custom-step", "run", "echo custom-step")); + final GitHubWorkflow workflow = validate(""" + jobs: + build: + steps: + - name: step0 + id: step0 + - name: step1 + id: replaced-step + - name: step2 + id: step2 + """, StepCustomization.builder().jobId(null).type(Type.REPLACE).stepId("replaced-step").step(newStep) + .build()); + assertThat(getStepIds(workflow.getJob("build")), contains("step0", "custom-step", "step2")); + } + + @Test + void missingJobConfigurationWithMultipleJobsFails() { + final WorkflowStep newStep = WorkflowStep + .createStep(Map.of("name", "Custom Step", "id", "custom-step", "run", "echo custom-step")); + final IllegalStateException exception = assertThrows(IllegalStateException.class, () -> validate(""" + jobs: + build1: + steps: + - name: step0 + id: step0 + build2: + steps: + - name: step0 + id: step0 + """, StepCustomization.builder().jobId(null).type(Type.REPLACE).stepId("replaced-step").step(newStep) + .build())); + assertThat(exception.getMessage(), equalTo( + "E-PK-CORE-208: Missing job in step customization of workflow 'workflow.yml' in file '.project-keeper.yml'. " + + "Add job with one of the following values: ['build1', 'build2'].")); + } + @Test void replaceStepNextStepHasMissingId() { final WorkflowStep newStep = WorkflowStep @@ -221,7 +263,7 @@ private GitHubWorkflow validate(final String workflowTemplate, final StepCustomi } private String getCustomizedContent(final String workflowTemplate, final StepCustomization... customizations) { - return new GitHubWorkflowCustomizer(new GitHubWorkflowStepCustomizer(asList(customizations))) + return new GitHubWorkflowCustomizer(new GitHubWorkflowStepCustomizer(WORKFLOW_NAME, asList(customizations))) .customizeContent(workflowTemplate); } } diff --git a/shared-model-classes/src/main/java/com/exasol/projectkeeper/shared/config/workflow/StepCustomization.java b/shared-model-classes/src/main/java/com/exasol/projectkeeper/shared/config/workflow/StepCustomization.java index 0f0c2318..92a5fa8d 100644 --- a/shared-model-classes/src/main/java/com/exasol/projectkeeper/shared/config/workflow/StepCustomization.java +++ b/shared-model-classes/src/main/java/com/exasol/projectkeeper/shared/config/workflow/StepCustomization.java @@ -1,6 +1,7 @@ package com.exasol.projectkeeper.shared.config.workflow; import java.util.Objects; +import java.util.Optional; /** * This class defines how a GitHub workflow should be modified by inserting or replacing a step. @@ -13,7 +14,7 @@ public final class StepCustomization { private final WorkflowStep step; private StepCustomization(final Builder builder) { - this.jobId = Objects.requireNonNull(builder.jobId, "jobId"); + this.jobId = builder.jobId; this.type = Objects.requireNonNull(builder.type, "type"); this.stepId = Objects.requireNonNull(builder.stepId, "stepId"); this.step = Objects.requireNonNull(builder.step, "step"); @@ -24,8 +25,8 @@ private StepCustomization(final Builder builder) { * * @return job ID */ - public String getJobId() { - return jobId; + public Optional getJobId() { + return Optional.ofNullable(jobId); } /** From 8e5f1dc0e9d421321b64865173c7a43936b1ee2b Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 16:56:13 +0100 Subject: [PATCH 03/14] Increment minor version --- .project-keeper.yml | 2 +- doc/changes/changelog.md | 2 +- .../{changes_4.4.1.md => changes_4.5.0.md} | 21 ++++++++++--------- parent-pom/pom.xml | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) rename doc/changes/{changes_4.4.1.md => changes_4.5.0.md} (89%) diff --git a/.project-keeper.yml b/.project-keeper.yml index 8535ebea..54b6c0f4 100644 --- a/.project-keeper.yml +++ b/.project-keeper.yml @@ -73,7 +73,7 @@ excludes: # PK can't use itself as a Maven plugin - regex: "W-PK-CORE-151: Pom file .* contains no reference to project-keeper-maven-plugin." # Build with Java 21 fails due to missing JavaDoc comments, see https://github.com/exasol/project-keeper/issues/596 - - "E-PK-CORE-17: Missing required file: '.github/workflows/ci-build-next-java.yml'" + ###- "E-PK-CORE-17: Missing required file: '.github/workflows/ci-build-next-java.yml'" # No configuration necessary for aggregator POM - "E-PK-CORE-103: Missing parent declaration in 'pom.xml'" - "E-PK-CORE-17: Missing required file: 'pk_generated_parent.pom'" diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 66c7c704..7c6525cb 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,6 +1,6 @@ # Changes -* [4.4.1](changes_4.4.1.md) +* [4.5.0](changes_4.5.0.md) * [4.4.0](changes_4.4.0.md) * [4.3.3](changes_4.3.3.md) * [4.3.2](changes_4.3.2.md) diff --git a/doc/changes/changes_4.4.1.md b/doc/changes/changes_4.5.0.md similarity index 89% rename from doc/changes/changes_4.4.1.md rename to doc/changes/changes_4.5.0.md index 8d402c3e..9adc57a4 100644 --- a/doc/changes/changes_4.4.1.md +++ b/doc/changes/changes_4.5.0.md @@ -1,14 +1,15 @@ -# Project Keeper 4.4.1, released 2024-??-?? +# Project Keeper 4.5.0, released 2024-11-22 Code name: Fix java customization ## Summary -This release allows customization of the java version in `actions/setup-java` steps. +This release allows customization of the java version in `actions/setup-java` steps. It also migrates generated workflows `broken_links_checker.yml` and `ci-build-next-java.yml` into `ci-build.yml`. This allows simplifying branch protection rules. Now only `build` is required, you can remove `next-java-compatibility` and `linkChecker`. ## Features * #602: Fixed customization of java version in `actions/setup-java` +* #597: Migrate workflows `broken_links_checker.yml` and `ci-build-next-java.yml` into `ci-build.yml` ## Bugfixes @@ -41,16 +42,16 @@ This release allows customization of the java version in `actions/setup-java` st #### Compile Dependency Updates -* Updated `com.exasol:project-keeper-shared-model-classes:4.4.0` to `4.4.1` +* Updated `com.exasol:project-keeper-shared-model-classes:4.4.0` to `4.5.0` #### Runtime Dependency Updates -* Updated `com.exasol:project-keeper-java-project-crawler:4.4.0` to `4.4.1` +* Updated `com.exasol:project-keeper-java-project-crawler:4.4.0` to `4.5.0` #### Test Dependency Updates * Updated `com.exasol:maven-project-version-getter:1.2.0` to `1.2.1` -* Updated `com.exasol:project-keeper-shared-test-setup:4.4.0` to `4.4.1` +* Updated `com.exasol:project-keeper-shared-test-setup:4.4.0` to `4.5.0` * Updated `nl.jqno.equalsverifier:equalsverifier:3.17.1` to `3.17.3` #### Plugin Dependency Updates @@ -67,12 +68,12 @@ This release allows customization of the java version in `actions/setup-java` st #### Compile Dependency Updates -* Updated `com.exasol:project-keeper-core:4.4.0` to `4.4.1` +* Updated `com.exasol:project-keeper-core:4.4.0` to `4.5.0` #### Test Dependency Updates * Updated `com.exasol:maven-project-version-getter:1.2.0` to `1.2.1` -* Updated `com.exasol:project-keeper-shared-test-setup:4.4.0` to `4.4.1` +* Updated `com.exasol:project-keeper-shared-test-setup:4.4.0` to `4.5.0` #### Plugin Dependency Updates @@ -88,7 +89,7 @@ This release allows customization of the java version in `actions/setup-java` st #### Compile Dependency Updates -* Updated `com.exasol:project-keeper-core:4.4.0` to `4.4.1` +* Updated `com.exasol:project-keeper-core:4.4.0` to `4.5.0` #### Test Dependency Updates @@ -110,7 +111,7 @@ This release allows customization of the java version in `actions/setup-java` st #### Compile Dependency Updates -* Updated `com.exasol:project-keeper-shared-model-classes:4.4.0` to `4.4.1` +* Updated `com.exasol:project-keeper-shared-model-classes:4.4.0` to `4.5.0` #### Test Dependency Updates @@ -132,7 +133,7 @@ This release allows customization of the java version in `actions/setup-java` st #### Compile Dependency Updates -* Updated `com.exasol:project-keeper-shared-model-classes:4.4.0` to `4.4.1` +* Updated `com.exasol:project-keeper-shared-model-classes:4.4.0` to `4.5.0` #### Plugin Dependency Updates diff --git a/parent-pom/pom.xml b/parent-pom/pom.xml index 6e49b2f8..46be26a1 100644 --- a/parent-pom/pom.xml +++ b/parent-pom/pom.xml @@ -28,7 +28,7 @@ - 4.4.1 + 4.5.0 17 3.9.9 From 6fda61ec5cc6daa3df49708d4c4aeb827088bb51 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 17:04:31 +0100 Subject: [PATCH 04/14] Fix compiler warnings with Java 21 --- .../test/java/com/exasol/projectkeeper/TestMavenModel.java | 1 + .../com/exasol/projectkeeper/mavenrepo/MavenRepository.java | 4 ++-- .../java/com/exasol/projectkeeper/test/TestMavenModel.java | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/maven-project-crawler/src/test/java/com/exasol/projectkeeper/TestMavenModel.java b/maven-project-crawler/src/test/java/com/exasol/projectkeeper/TestMavenModel.java index a62be5e7..de7d4282 100644 --- a/maven-project-crawler/src/test/java/com/exasol/projectkeeper/TestMavenModel.java +++ b/maven-project-crawler/src/test/java/com/exasol/projectkeeper/TestMavenModel.java @@ -15,6 +15,7 @@ public class TestMavenModel extends Model { public static final String PROJECT_VERSION = "0.1.0"; public static final String PROJECT_GROUP_ID = "com.example"; + @SuppressWarnings("this-escape") // Required by 3rd party parent class public TestMavenModel() { this.setBuild(new Build()); this.setVersion(PROJECT_VERSION); diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/mavenrepo/MavenRepository.java b/project-keeper/src/main/java/com/exasol/projectkeeper/mavenrepo/MavenRepository.java index c5c42a84..678bdbbb 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/mavenrepo/MavenRepository.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/mavenrepo/MavenRepository.java @@ -4,7 +4,7 @@ import java.io.IOException; import java.io.InputStream; -import java.net.URL; +import java.net.URI; import javax.xml.XMLConstants; import javax.xml.parsers.*; @@ -104,7 +104,7 @@ public String getLatestVersion() factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); final DocumentBuilder db = factory.newDocumentBuilder(); - try (InputStream stream = new URL(this.url).openStream()) { + try (InputStream stream = URI.create(this.url).toURL().openStream()) { return getLatestVersion(db.parse(stream)); } } diff --git a/shared-test-setup/src/main/java/com/exasol/projectkeeper/test/TestMavenModel.java b/shared-test-setup/src/main/java/com/exasol/projectkeeper/test/TestMavenModel.java index c34830c8..ba972c56 100644 --- a/shared-test-setup/src/main/java/com/exasol/projectkeeper/test/TestMavenModel.java +++ b/shared-test-setup/src/main/java/com/exasol/projectkeeper/test/TestMavenModel.java @@ -18,6 +18,7 @@ public class TestMavenModel extends Model { public static final String PROJECT_GROUP_ID = "com.exasol"; private static final String PROJECT_DESCRIPTION = "My project description"; + @SuppressWarnings("this-escape") // Required by 3rd party parent class public TestMavenModel() { this.setBuild(new Build()); this.setVersion(PROJECT_VERSION); From ed431feea335918ce5c5ff851b7958345bff5c69 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 17:17:00 +0100 Subject: [PATCH 05/14] Fix Java 21 build --- .github/workflows/ci-build.yml | 3 ++- .project-keeper.yml | 12 ++++++++++-- .../.github/workflows/ci-build-db-version-matrix.yml | 1 + .../templates/.github/workflows/ci-build.yml | 1 + 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 88d54afd..a5848b67 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -166,7 +166,8 @@ jobs: } - { name: Run tests and build with Maven 21, - run: mvn --batch-mode clean package -DtrimStackTrace=false -Djava.version=21 + id: build-next-java, + run: mvn --batch-mode clean test -DtrimStackTrace=false -Djava.version=21 } build: needs: [ diff --git a/.project-keeper.yml b/.project-keeper.yml index 54b6c0f4..926a43e8 100644 --- a/.project-keeper.yml +++ b/.project-keeper.yml @@ -72,8 +72,6 @@ version: excludes: # PK can't use itself as a Maven plugin - regex: "W-PK-CORE-151: Pom file .* contains no reference to project-keeper-maven-plugin." - # Build with Java 21 fails due to missing JavaDoc comments, see https://github.com/exasol/project-keeper/issues/596 - ###- "E-PK-CORE-17: Missing required file: '.github/workflows/ci-build-next-java.yml'" # No configuration necessary for aggregator POM - "E-PK-CORE-103: Missing parent declaration in 'pom.xml'" - "E-PK-CORE-17: Missing required file: 'pk_generated_parent.pom'" @@ -135,6 +133,16 @@ build: ls -lh coverage-aggregator/target/metrics.json cat coverage-aggregator/target/metrics.json + # Build with Java 21 fails due to missing JavaDoc comments, see https://github.com/exasol/project-keeper/issues/596 + - action: REPLACE + job: next-java-compatibility + stepId: build-next-java + content: + name: Run tests and build with Maven 21 + id: build-next-java + # Only run tests as javadoc has warnings with Java 21 + run: mvn --batch-mode clean test -DtrimStackTrace=false -Djava.version=21 + # [itest->dsn~customize-build-process.dependency-check~0] - name: "dependencies_check.yml" stepCustomizations: diff --git a/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml b/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml index 872a7e2b..4185c7f0 100644 --- a/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml +++ b/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml @@ -160,6 +160,7 @@ jobs: java-version: $nextJavaVersion cache: "maven" - name: Run tests and build with Maven $nextJavaVersion + id: build-next-java run: mvn --batch-mode clean package -DtrimStackTrace=false -Djava.version=$nextJavaVersion build: diff --git a/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml b/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml index c774b920..ac4aa44c 100644 --- a/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml +++ b/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml @@ -151,6 +151,7 @@ jobs: java-version: $nextJavaVersion cache: "maven" - name: Run tests and build with Maven $nextJavaVersion + id: build-next-java run: mvn --batch-mode clean package -DtrimStackTrace=false -Djava.version=$nextJavaVersion build: From b9f592ac3431469c5c331b64d64644946c59eb86 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 17:22:04 +0100 Subject: [PATCH 06/14] Fix cancelled jobs due to concurrency --- .github/workflows/ci-build.yml | 4 ++-- .../.github/workflows/ci-build-db-version-matrix.yml | 2 +- .../main/resources/templates/.github/workflows/ci-build.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index a5848b67..17817712 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -20,7 +20,7 @@ jobs: issues: read } concurrency: { - group: '${{ github.workflow }}-${{ github.ref }}', + group: '${{ github.workflow }}-build-and-test-${{ github.ref }}', cancel-in-progress: true } outputs: { @@ -146,7 +146,7 @@ jobs: contents: read } concurrency: { - group: '${{ github.workflow }}-${{ github.ref }}', + group: '${{ github.workflow }}-next-java-${{ github.ref }}', cancel-in-progress: true } steps: diff --git a/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml b/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml index 4185c7f0..10861a01 100644 --- a/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml +++ b/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml @@ -144,7 +144,7 @@ jobs: permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-next-java-${{ github.ref }} cancel-in-progress: true steps: - name: Checkout the repository diff --git a/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml b/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml index ac4aa44c..db9e837a 100644 --- a/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml +++ b/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml @@ -15,7 +15,7 @@ jobs: contents: read issues: read # Required for PK verify-release concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-build-and-test-${{ github.ref }} cancel-in-progress: true outputs: release-required: ${{ steps.check-release.outputs.release-required }} @@ -135,7 +135,7 @@ jobs: permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-next-java-${{ github.ref }} cancel-in-progress: true steps: - name: Checkout the repository From 4f15235f75b757f01b88b4646597f4d8b054880e Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 17:29:12 +0100 Subject: [PATCH 07/14] Adapt unit test --- .../files/CiBuildWorkflowGeneratorTest.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java index 4f838a0c..ae25e829 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java @@ -87,15 +87,19 @@ void customizeBuildStepsMatrixBuild() { @Test void ciBuildContentNonMatrixBuild() { - final Job job = ciBuildContent(BuildOptions.builder()).getJob("build-and-test"); + final Job buildJob = ciBuildContent(BuildOptions.builder()).getJob("build-and-test"); + final Job nextJavaJob = ciBuildContent(BuildOptions.builder()).getJob("next-java-compatibility"); assertAll( - () -> assertThat(job.getConcurrency(), - equalTo(Map.of("group", "${{ github.workflow }}-${{ github.ref }}", "cancel-in-progress", - true))), - () -> assertThat(job.getStrategy(), nullValue()), () -> assertThat(job.getEnv(), nullValue()), - () -> assertThat(job.getStep("build-pk-verify").getRunCommand(), + () -> assertThat(buildJob.getConcurrency(), + equalTo(Map.of("group", "${{ github.workflow }}-build-and-test-${{ github.ref }}", + "cancel-in-progress", true))), + () -> assertThat(nextJavaJob.getConcurrency(), + equalTo(Map.of("group", "${{ github.workflow }}-next-java-${{ github.ref }}", + "cancel-in-progress", true))), + () -> assertThat(buildJob.getStrategy(), nullValue()), () -> assertThat(buildJob.getEnv(), nullValue()), + () -> assertThat(buildJob.getStep("build-pk-verify").getRunCommand(), not(containsString("-Dcom.exasol.dockerdb.image"))), - () -> assertThat(job.getStep("sonar-analysis").getIfCondition(), + () -> assertThat(buildJob.getStep("sonar-analysis").getIfCondition(), equalTo("${{ env.SONAR_TOKEN != null }}"))); } From b15d0055621859af5562725be74ca69d69551dd0 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 17:41:32 +0100 Subject: [PATCH 08/14] Fix integration tests --- .../validators/files/CiBuildWorkflowGenerator.java | 9 +++++---- .../validators/files/FileTemplatesFactory.java | 2 +- .../validators/files/CiBuildWorkflowGeneratorTest.java | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java index 19cff0d0..52cd8be9 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java @@ -5,6 +5,7 @@ import java.util.*; import java.util.function.Consumer; +import java.util.function.Supplier; import com.exasol.projectkeeper.shared.config.BuildOptions; import com.exasol.projectkeeper.shared.config.ProjectKeeperModule; @@ -18,10 +19,10 @@ class CiBuildWorkflowGenerator { private static final String CI_BUILD_PATH_IN_PROJECT = WORKFLOW_PATH + CI_BUILD_WORKFLOW_NAME; private final BuildOptions buildOptions; private final List javaVersions; - private final String nextJavaVersion; + private final Supplier nextJavaVersion; CiBuildWorkflowGenerator(final BuildOptions buildOptions, final List javaVersions, - final String nextJavaVersion) { + final Supplier nextJavaVersion) { this.buildOptions = Objects.requireNonNull(buildOptions, "buildOptions"); this.javaVersions = Objects.requireNonNull(javaVersions, "javaVersions"); this.nextJavaVersion = Objects.requireNonNull(nextJavaVersion, "nextJavaVersion"); @@ -34,7 +35,7 @@ FileTemplate createCiBuildWorkflow() { CI_BUILD_PATH_IN_PROJECT, REQUIRE_EXACT); template.replacing("ciBuildRunnerOS", buildOptions.getRunnerOs()); template.replacing("freeDiskSpace", String.valueOf(buildOptions.shouldFreeDiskSpace())); - template.replacing("nextJavaVersion", nextJavaVersion); + template.replacing("nextJavaVersion", nextJavaVersion.get()); if (buildType == CiTemplateType.EXASOL_VERSION_MATRIX) { template.replacing("matrixExasolDbVersions", @@ -60,7 +61,7 @@ private FileTemplate createTemplate(final FileTemplateFromResource template, fin } private GitHubWorkflowJavaVersionCustomizer javaVersionCustomizer() { - return new GitHubWorkflowJavaVersionCustomizer(javaVersions, nextJavaVersion); + return new GitHubWorkflowJavaVersionCustomizer(javaVersions, nextJavaVersion.get()); } private List findCustomizations(final String workflowName) { diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactory.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactory.java index 80eee2c5..205e4bb9 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactory.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactory.java @@ -35,7 +35,7 @@ public FileTemplatesFactory(final Logger logger, final String ownVersion, final this.sources = sources; final JavaVersionExtractor javaVersionExtractor = new JavaVersionExtractor(sources); this.workflowGenerator = new CiBuildWorkflowGenerator(this.buildOptions, javaVersionExtractor.extractVersions(), - javaVersionExtractor.getNextVersion()); + javaVersionExtractor::getNextVersion); } List getGlobalTemplates() { diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java index ae25e829..10c52e98 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGeneratorTest.java @@ -350,7 +350,7 @@ private void validateYamlSyntax(final String content) { } private CiBuildWorkflowGenerator testee(final BuildOptions options) { - return new CiBuildWorkflowGenerator(options, List.of("11", "17"), "21"); + return new CiBuildWorkflowGenerator(options, List.of("11", "17"), () -> "21"); } private GitHubWorkflow parse(final String yaml) { From affcc7fcbe0e5e5bb4b5a6695c5458bf1e86d3f6 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 17:53:24 +0100 Subject: [PATCH 09/14] Fix sonar warnings --- .../projectkeeper/validators/files/GitHubWorkflow.java | 5 ----- .../validators/files/GitHubWorkflowStepCustomizer.java | 4 +++- .../validators/files/GitHubWorkflowStepCustomizerTest.java | 5 +++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java index de6983f7..201ff444 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java @@ -197,9 +197,4 @@ private Optional getOptionalString(final String key) { return Optional.ofNullable((String) rawStep.get(key)); } } - - public Job getJob(final Optional jobId) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'getJob'"); - } } diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java index c2bf40d4..bec10fcf 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java @@ -1,6 +1,7 @@ package com.exasol.projectkeeper.validators.files; import java.util.List; +import java.util.Optional; import com.exasol.errorreporting.ExaError; import com.exasol.projectkeeper.config.ProjectKeeperConfigReader; @@ -25,7 +26,8 @@ public void applyCustomization(final GitHubWorkflow workflow) { } private Job getJob(final GitHubWorkflow workflow, final StepCustomization customization) { - if (customization.getJobId().isPresent()) { + final Optional jobId = customization.getJobId(); + if (jobId.isPresent()) { return workflow.getJob(customization.getJobId().get()); } final List allJobs = workflow.getJobs(); diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java index 1df0a8af..9d10b974 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizerTest.java @@ -125,6 +125,8 @@ void missingJobConfigurationWithSingleJob() { void missingJobConfigurationWithMultipleJobsFails() { final WorkflowStep newStep = WorkflowStep .createStep(Map.of("name", "Custom Step", "id", "custom-step", "run", "echo custom-step")); + final StepCustomization stepCustomization = StepCustomization.builder().jobId(null).type(Type.REPLACE) + .stepId("replaced-step").step(newStep).build(); final IllegalStateException exception = assertThrows(IllegalStateException.class, () -> validate(""" jobs: build1: @@ -135,8 +137,7 @@ void missingJobConfigurationWithMultipleJobsFails() { steps: - name: step0 id: step0 - """, StepCustomization.builder().jobId(null).type(Type.REPLACE).stepId("replaced-step").step(newStep) - .build())); + """, stepCustomization)); assertThat(exception.getMessage(), equalTo( "E-PK-CORE-208: Missing job in step customization of workflow 'workflow.yml' in file '.project-keeper.yml'. " + "Add job with one of the following values: ['build1', 'build2'].")); From 1f1273106afed8a464afec019e0dfe15315d58e1 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 17:53:42 +0100 Subject: [PATCH 10/14] Rename job --- .github/workflows/test_on_windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_on_windows.yml b/.github/workflows/test_on_windows.yml index 9292d3a4..61e3845d 100644 --- a/.github/workflows/test_on_windows.yml +++ b/.github/workflows/test_on_windows.yml @@ -7,7 +7,7 @@ on: pull_request: jobs: - test: + test-on-windows: runs-on: windows-latest permissions: contents: read From 35066386ac9ca25906453c1dcf9f973c8dd97092 Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Thu, 21 Nov 2024 18:02:01 +0100 Subject: [PATCH 11/14] Add unit tests --- ...tHubWorkflowJavaVersionCustomizerTest.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizerTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizerTest.java index c894e41b..26d952ee 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizerTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizerTest.java @@ -59,6 +59,19 @@ void testUpdateEmptyJavaVersion() { assertThat(workflow.getJob("build").getStep("setup-java").getWith().get("java-version"), equalTo("v1")); } + @Test + void testUpdateAnyJobs() { + final GitHubWorkflow workflow = customize(""" + jobs: + any-job-name: + steps: + - name: Set up JDKs + id: setup-java + uses: actions/setup-java@v4 + """, List.of("v1"), "v2"); + assertThat(workflow.getJob("any-job-name").getStep("setup-java").getWith().get("java-version"), equalTo("v1")); + } + @Test void testUpdateToMultipleJavaVersions() { final GitHubWorkflow workflow = customize(""" @@ -78,6 +91,26 @@ void testUpdateToMultipleJavaVersions() { assertThat(workflow.getJob("build").getStep("setup-java").getWith().get("java-version"), equalTo("11\n17\n21")); } + @Test + void testUpdateNextJavaBuild() { + final GitHubWorkflow workflow = customize(""" + jobs: + next-java-compatibility: + steps: + - name: Set up JDKs + id: setup-java + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: | + old + version + cache: "maven" + """, List.of("11", "17", "21"), "23"); + assertThat(workflow.getJob("next-java-compatibility").getStep("setup-java").getWith().get("java-version"), + equalTo("23")); + } + @Test void testNoUpdateOfUnknownAction() { final GitHubWorkflow workflow = customize(""" From bf8b567c0a5defbf6d9904c01e0be359f71f3c1f Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 22 Nov 2024 07:45:38 +0100 Subject: [PATCH 12/14] Fix sonar finding --- .../validators/files/GitHubWorkflowStepCustomizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java index bec10fcf..6246a7e0 100644 --- a/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java +++ b/project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java @@ -28,7 +28,7 @@ public void applyCustomization(final GitHubWorkflow workflow) { private Job getJob(final GitHubWorkflow workflow, final StepCustomization customization) { final Optional jobId = customization.getJobId(); if (jobId.isPresent()) { - return workflow.getJob(customization.getJobId().get()); + return workflow.getJob(jobId.get()); } final List allJobs = workflow.getJobs(); if (allJobs.size() == 1) { From 1b720f5b8fabe51074b99f621be9e747074b2eab Mon Sep 17 00:00:00 2001 From: Christoph Pirkl Date: Fri, 22 Nov 2024 07:49:41 +0100 Subject: [PATCH 13/14] Add unit tests --- .../files/JavaVersionExtractorTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java index d79b07df..9f3ffed6 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java @@ -4,6 +4,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.util.List; @@ -15,6 +16,12 @@ class JavaVersionExtractorTest { + @Test + void nonSource() { + final IllegalStateException exception = assertThrows(IllegalStateException.class, () -> extract()); + assertThat(exception.getMessage(), equalTo("No sources, can't get java versions")); + } + @Test void nonJavaSource() { assertThat(extract(AnalyzedSourceImpl.builder().build()), contains("17")); @@ -25,6 +32,16 @@ void javaSource() { assertThat(extract(mavenSource("11")), contains("11", "17")); } + @Test + void versionWithDot() { + assertThat(extract(mavenSource("1.2")), contains("1.2", "17")); + } + + @Test + void versionWithTwoDot() { + assertThat(extract(mavenSource("1.2.3")), contains("1.2.3", "17")); + } + @Test void sortedAscending() { assertThat(extract(mavenSource("21"), mavenSource("17"), mavenSource("11")), contains("11", "17", "21")); From e42cee2918726a587822c4809b01497e3698d40c Mon Sep 17 00:00:00 2001 From: Christoph Pirkl <4711730+kaklakariada@users.noreply.github.com> Date: Fri, 22 Nov 2024 08:40:39 +0100 Subject: [PATCH 14/14] Update project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java Co-authored-by: Christoph Kuhnke --- .../validators/files/JavaVersionExtractorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java index 9f3ffed6..ac8d78eb 100644 --- a/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java +++ b/project-keeper/src/test/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractorTest.java @@ -17,7 +17,7 @@ class JavaVersionExtractorTest { @Test - void nonSource() { + void noSources() { final IllegalStateException exception = assertThrows(IllegalStateException.class, () -> extract()); assertThat(exception.getMessage(), equalTo("No sources, can't get java versions")); }