From 6d34a6838e83779d12fc882eafd1bfdb6c192a92 Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Thu, 12 Oct 2023 12:05:12 +0100 Subject: [PATCH 1/2] Remove unused code --- .../analyser/deploy/DeployCommand.java | 262 +++++++++--------- 1 file changed, 127 insertions(+), 135 deletions(-) diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java index 2302afc7c..1cd5afe10 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java @@ -103,154 +103,146 @@ public DeployCommand(BeanManager beanManager, public void run() { try { - java.nio.file.Path modified = Files.createTempFile("deployment", ".tar.gz"); - try { - - Set gavs = new HashSet<>(); - - Map> contaminatedPaths = new HashMap<>(); - Map> contaminatedGavs = new HashMap<>(); - // Represents directories that should not be deployed i.e. if a single artifact (barring test jars) is - // contaminated then none of the artifacts will be deployed. - Set toRemove = new HashSet<>(); - Map jarFiles = new HashMap<>(); - Files.walkFileTree(deploymentPath, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - String name = deploymentPath.relativize(file).toString(); - Optional gav = getGav(name); - gav.ifPresent( - gav1 -> gavs.add(gav1.getGroupId() + ":" + gav1.getArtifactId() + ":" + gav1.getVersion())); - Log.debugf("Checking %s for contaminants with GAV %s", name, gav); - //we check every file as we also want to catch .tar.gz etc - var info = ClassFileTracker.readTrackingDataFromFile(Files.newInputStream(file), name); - for (var i : info) { - if (!allowedSources.contains(i.source)) { - Log.errorf("%s was contaminated by %s from %s", name, i.gav, i.source); - if (ALLOWED_CONTAMINANTS.stream().noneMatch(a -> file.getFileName().toString().endsWith(a))) { - gav.ifPresent(g -> contaminatedGavs.computeIfAbsent(i.gav, s -> new HashSet<>()) - .add(g.getGroupId() + ":" + g.getArtifactId() + ":" + g.getVersion())); - int index = name.lastIndexOf("/"); - if (index != -1) { - contaminatedPaths.computeIfAbsent(name.substring(0, index), - s -> new HashSet<>()).add(i.gav); - } else { - contaminatedPaths.computeIfAbsent("", s -> new HashSet<>()).add(i.gav); - } - toRemove.add(file.getParent()); + Set gavs = new HashSet<>(); + + Map> contaminatedPaths = new HashMap<>(); + Map> contaminatedGavs = new HashMap<>(); + // Represents directories that should not be deployed i.e. if a single artifact (barring test jars) is + // contaminated then none of the artifacts will be deployed. + Set toRemove = new HashSet<>(); + Map jarFiles = new HashMap<>(); + Files.walkFileTree(deploymentPath, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + String name = deploymentPath.relativize(file).toString(); + Optional gav = getGav(name); + gav.ifPresent( + gav1 -> gavs.add(gav1.getGroupId() + ":" + gav1.getArtifactId() + ":" + gav1.getVersion())); + Log.debugf("Checking %s for contaminants with GAV %s", name, gav); + //we check every file as we also want to catch .tar.gz etc + var info = ClassFileTracker.readTrackingDataFromFile(Files.newInputStream(file), name); + for (var i : info) { + if (!allowedSources.contains(i.source)) { + Log.errorf("%s was contaminated by %s from %s", name, i.gav, i.source); + if (ALLOWED_CONTAMINANTS.stream().noneMatch(a -> file.getFileName().toString().endsWith(a))) { + gav.ifPresent(g -> contaminatedGavs.computeIfAbsent(i.gav, s -> new HashSet<>()) + .add(g.getGroupId() + ":" + g.getArtifactId() + ":" + g.getVersion())); + int index = name.lastIndexOf("/"); + if (index != -1) { + contaminatedPaths.computeIfAbsent(name.substring(0, index), + s -> new HashSet<>()).add(i.gav); } else { - Log.debugf("Ignoring contaminant for %s", file.getFileName()); + contaminatedPaths.computeIfAbsent("", s -> new HashSet<>()).add(i.gav); } + toRemove.add(file.getParent()); + } else { + Log.debugf("Ignoring contaminant for %s", file.getFileName()); } } - if (gav.isPresent()) { - //now add our own tracking data - if (name.endsWith(".jar") && !name.endsWith("-sources.jar") && !name.endsWith("-javadoc.jar")) { - jarFiles.put(file, gav.get()); - } - } - return FileVisitResult.CONTINUE; } - }); - for (var e : jarFiles.entrySet()) { - Path file = e.getKey(); - Gav gav = e.getValue(); - try { - String fileName = file.getFileName().toString(); - Path temp = file.getParent().resolve(fileName + ".temp"); - ClassFileTracker.addTrackingDataToJar(Files.newInputStream(file), - new TrackingData( - gav.getGroupId() + ":" + gav.getArtifactId() + ":" - + gav.getVersion(), - "rebuilt", - Map.of("scm-uri", scmUri, "scm-commit", commit, "hermetic", - Boolean.toString(hermetic))), - Files.newOutputStream(temp), false); - Files.delete(file); - Files.move(temp, file); - try (Stream pathStream = Files.list(file.getParent())) { - pathStream.filter(s -> s.getFileName().toString().startsWith(fileName + ".")) - .forEach(f -> { - try { - Files.delete(f); - } catch (IOException ex) { - throw new RuntimeException(ex); - } - }); + if (gav.isPresent()) { + //now add our own tracking data + if (name.endsWith(".jar") && !name.endsWith("-sources.jar") && !name.endsWith("-javadoc.jar")) { + jarFiles.put(file, gav.get()); } - - Files.writeString(file.getParent().resolve(fileName + ".md5"), - HashUtil.md5(Files.newInputStream(file))); - Files.writeString(file.getParent().resolve(fileName + ".sha1"), - HashUtil.sha1(Files.newInputStream(file))); - } catch (Exception ex) { - Log.errorf(ex, "Failed to instrument %s", file); } + return FileVisitResult.CONTINUE; } - for (var i : toRemove) { - Log.errorf("Removing %s as it is contaminated", i); - FileUtil.deleteRecursive(i); - } - - //update the DB with contaminant information - Log.infof("Contaminants: %s", contaminatedPaths); - Log.infof("Contaminated GAVS: %s", contaminatedGavs); - Log.infof("GAVs to deploy: %s", gavs); - if (gavs.isEmpty()) { - Log.errorf("No content to deploy found in deploy directory"); - - Files.walkFileTree(deploymentPath, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Log.errorf("Contents: %s", file); - return FileVisitResult.CONTINUE; - } - }); - throw new RuntimeException("deploy failed"); - } - //we still deploy, but without the contaminates - java.nio.file.Path deployFile = deploymentPath; - // This means the build failed to produce any deployable output. - // If everything is contaminated we still need the task to succeed so we can resolve the contamination. - for (var i : contaminatedGavs.entrySet()) { - gavs.removeAll(i.getValue()); - } - generateBuildSbom(); - - if (!gavs.isEmpty()) { - try { - cleanBrokenSymlinks(sourcePath); - doDeployment(deployFile, sourcePath, logsPath, gavs); - } catch (Throwable t) { - Log.error("Deployment failed", t); - flushLogs(); - throw t; + }); + for (var e : jarFiles.entrySet()) { + Path file = e.getKey(); + Gav gav = e.getValue(); + try { + String fileName = file.getFileName().toString(); + Path temp = file.getParent().resolve(fileName + ".temp"); + ClassFileTracker.addTrackingDataToJar(Files.newInputStream(file), + new TrackingData( + gav.getGroupId() + ":" + gav.getArtifactId() + ":" + + gav.getVersion(), + "rebuilt", + Map.of("scm-uri", scmUri, "scm-commit", commit, "hermetic", + Boolean.toString(hermetic))), + Files.newOutputStream(temp), false); + Files.delete(file); + Files.move(temp, file); + try (Stream pathStream = Files.list(file.getParent())) { + pathStream.filter(s -> s.getFileName().toString().startsWith(fileName + ".")) + .forEach(f -> { + try { + Files.delete(f); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + }); } - } else { - Log.errorf("Skipped deploying from task run %s as all artifacts were contaminated", taskRun); + + Files.writeString(file.getParent().resolve(fileName + ".md5"), + HashUtil.md5(Files.newInputStream(file))); + Files.writeString(file.getParent().resolve(fileName + ".sha1"), + HashUtil.sha1(Files.newInputStream(file))); + } catch (Exception ex) { + Log.errorf(ex, "Failed to instrument %s", file); } - if (taskRun != null) { - - List newContaminates = new ArrayList<>(); - for (var i : contaminatedGavs.entrySet()) { - Contaminates contaminates = new Contaminates(); - contaminates.setContaminatedArtifacts(new ArrayList<>(i.getValue())); - contaminates.setGav(i.getKey()); - newContaminates.add(contaminates); + } + for (var i : toRemove) { + Log.errorf("Removing %s as it is contaminated", i); + FileUtil.deleteRecursive(i); + } + + //update the DB with contaminant information + Log.infof("Contaminants: %s", contaminatedPaths); + Log.infof("Contaminated GAVS: %s", contaminatedGavs); + Log.infof("GAVs to deploy: %s", gavs); + if (gavs.isEmpty()) { + Log.errorf("No content to deploy found in deploy directory"); + + Files.walkFileTree(deploymentPath, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Log.errorf("Contents: %s", file); + return FileVisitResult.CONTINUE; } - String serialisedContaminants = ResultsUpdater.MAPPER.writeValueAsString(newContaminates); - Log.infof("Updating results %s with contaminants %s and deployed resources %s", - taskRun, serialisedContaminants, gavs); - resultsUpdater.updateResults(taskRun, Map.of( - "CONTAMINANTS", serialisedContaminants, - "DEPLOYED_RESOURCES", String.join(",", gavs), - "IMAGE_URL", imageName == null ? "" : imageName, - "IMAGE_DIGEST", imageDigest == null ? "" : "sha256:" + imageDigest)); + }); + throw new RuntimeException("deploy failed"); + } + //we still deploy, but without the contaminates + java.nio.file.Path deployFile = deploymentPath; + // This means the build failed to produce any deployable output. + // If everything is contaminated we still need the task to succeed so we can resolve the contamination. + for (var i : contaminatedGavs.entrySet()) { + gavs.removeAll(i.getValue()); + } + generateBuildSbom(); + + if (!gavs.isEmpty()) { + try { + cleanBrokenSymlinks(sourcePath); + doDeployment(deployFile, sourcePath, logsPath, gavs); + } catch (Throwable t) { + Log.error("Deployment failed", t); + flushLogs(); + throw t; } - } finally { - if (Files.exists(modified)) { - Files.delete(modified); + } else { + Log.errorf("Skipped deploying from task run %s as all artifacts were contaminated", taskRun); + } + if (taskRun != null) { + + List newContaminates = new ArrayList<>(); + for (var i : contaminatedGavs.entrySet()) { + Contaminates contaminates = new Contaminates(); + contaminates.setContaminatedArtifacts(new ArrayList<>(i.getValue())); + contaminates.setGav(i.getKey()); + newContaminates.add(contaminates); } + String serialisedContaminants = ResultsUpdater.MAPPER.writeValueAsString(newContaminates); + Log.infof("Updating results %s with contaminants %s and deployed resources %s", + taskRun, serialisedContaminants, gavs); + resultsUpdater.updateResults(taskRun, Map.of( + "CONTAMINANTS", serialisedContaminants, + "DEPLOYED_RESOURCES", String.join(",", gavs), + "IMAGE_URL", imageName == null ? "" : imageName, + "IMAGE_DIGEST", imageDigest == null ? "" : "sha256:" + imageDigest)); } } catch (Exception e) { Log.error("Deployment failed", e); From 0f7545ed01421f12729d5c82b97aa0867a791789 Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Fri, 13 Oct 2023 16:13:16 +0100 Subject: [PATCH 2/2] Integrate maven repo deployment code --- deploy/base-development.sh | 6 +- .../base/jvmbuildservice.io_jbsconfigs.yaml | 7 + deploy/overlays/ci-template/config.yaml | 7 +- deploy/overlays/dev-template/config.yaml | 6 +- deploy/patch-yaml.sh | 8 ++ .../build-request-processor/pom.xml | 90 ++++-------- .../analyser/deploy/DeployCommand.java | 70 +++++++--- .../ContainerRegistryDeployer.java | 17 +-- .../MavenRepositoryDeployer.java | 120 ++++++++++++++++ .../hacbs/container/verifier/MavenUtils.java | 14 -- .../verifier/VerifyBuiltArtifactsCommand.java | 11 +- .../deploy/ContainerRegistryDeployerTest.java | 9 +- .../deploy/DeployContaminateTest.java | 12 +- .../analyser/deploy/MavenDeployTest.java | 131 ++++++++++++++++++ .../VerifyBuiltArtifactsCommandTest.java | 9 +- java-components/pom.xml | 5 + .../crds/jvmbuildservice.io_jbsconfigs.yaml | 7 + .../v1alpha1/jbsconfig_types.go | 20 ++- .../dependencybuild/buildrecipeyaml.go | 17 ++- 19 files changed, 423 insertions(+), 143 deletions(-) create mode 100644 java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java create mode 100644 java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/MavenDeployTest.java diff --git a/deploy/base-development.sh b/deploy/base-development.sh index 6191ec3aa..dbd5f2778 100755 --- a/deploy/base-development.sh +++ b/deploy/base-development.sh @@ -17,7 +17,7 @@ fi DIR=`dirname $0` kubectl apply -f $DIR/namespace.yaml kubectl config set-context --current --namespace=test-jvm-namespace -kubectl delete --ignore-not-found secret jvm-build-image-secrets jvm-build-git-secrets +kubectl delete --ignore-not-found secret jvm-build-image-secrets jvm-build-git-secrets jvm-build-maven-repo-secrets if [ -n "$QUAY_ORG" ] && [ -n "$QUAY_TOKEN" ]; then kubectl delete --ignore-not-found secret -n image-controller quaytoken @@ -28,10 +28,12 @@ kubectl create secret generic jvm-build-git-secrets --from-literal .git-credenti https://$GITHUB_E2E_ORGANIZATION:$GITHUB_TOKEN@github.com https://test:test@gitlab.com " +if [ -n "$MAVEN_PASSWORD" ]; then + kubectl create secret generic jvm-build-maven-repo-secrets --from-literal mavenpassword="$MAVEN_PASSWORD" +fi JVM_BUILD_SERVICE_IMAGE=quay.io/$QUAY_USERNAME/hacbs-jvm-controller \ JVM_BUILD_SERVICE_CACHE_IMAGE=quay.io/$QUAY_USERNAME/hacbs-jvm-cache \ JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE=quay.io/$QUAY_USERNAME/hacbs-jvm-build-request-processor:dev \ $DIR/patch-yaml.sh kubectl apply -k $DIR/overlays/development - diff --git a/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml b/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml index 237604b3f..38213e683 100644 --- a/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml +++ b/deploy/crds/base/jvmbuildservice.io_jbsconfigs.yaml @@ -97,6 +97,13 @@ spec: additionalProperties: type: string type: object + mavenDeployment: + properties: + repository: + type: string + username: + type: string + type: object owner: type: string port: diff --git a/deploy/overlays/ci-template/config.yaml b/deploy/overlays/ci-template/config.yaml index 7ed387985..57cedc971 100644 --- a/deploy/overlays/ci-template/config.yaml +++ b/deploy/overlays/ci-template/config.yaml @@ -4,6 +4,7 @@ kind: JBSConfig metadata: name: jvm-build-config spec: - owner: QUAY_USERNAME - host: quay.io - repository: test-images + registry: + owner: QUAY_USERNAME + host: quay.io + repository: test-images diff --git a/deploy/overlays/dev-template/config.yaml b/deploy/overlays/dev-template/config.yaml index c184ae5b1..4efabfd63 100644 --- a/deploy/overlays/dev-template/config.yaml +++ b/deploy/overlays/dev-template/config.yaml @@ -4,7 +4,11 @@ kind: JBSConfig metadata: name: jvm-build-config spec: - owner: QUAY_USERNAME + registry: + owner: QUAY_USERNAME + mavenDeployment: + username: MAVEN_USERNAME + repository: MAVEN_REPOSITORY relocationPatterns: - relocationPattern: buildPolicy: "default" diff --git a/deploy/patch-yaml.sh b/deploy/patch-yaml.sh index d55109a06..8cfc3c08e 100755 --- a/deploy/patch-yaml.sh +++ b/deploy/patch-yaml.sh @@ -25,6 +25,14 @@ find $DIR -path \*development\*.yaml -exec $SED -i s%jvm-build-service-cache-ima find $DIR -path \*development\*.yaml -exec $SED -i s%jvm-build-service-reqprocessor-image%${JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE}% {} \; find $DIR -path \*development\*.yaml -exec $SED -i s/dev-template/development/ {} \; find $DIR -path \*development\*.yaml -exec $SED -i s/QUAY_TOKEN/${QUAY_TOKEN}/ {} \; +if [ -z "${MAVEN_USERNAME}" ]; then + MAVEN_USERNAME="" +fi +if [ -z "${MAVEN_REPOSITORY}" ]; then + MAVEN_REPOSITORY="" +fi +find $DIR -path \*development\*.yaml -exec $SED -i s/MAVEN_USERNAME/${MAVEN_USERNAME}/ {} \; +find $DIR -path \*development\*.yaml -exec $SED -i s%MAVEN_REPOSITORY%${MAVEN_REPOSITORY}% {} \; if [ -n "$QUAY_TOKEN" ]; then $SED -i '/owner: QUAY_USERNAME/d' $DIR/overlays/development/config.yaml fi diff --git a/java-components/build-request-processor/pom.xml b/java-components/build-request-processor/pom.xml index 94abcb781..b3f7df4b7 100644 --- a/java-components/build-request-processor/pom.xml +++ b/java-components/build-request-processor/pom.xml @@ -9,30 +9,19 @@ hacbs-build-request-processor - - org.apache.httpcomponents - httpclient - - - io.quarkiverse.amazonservices - quarkus-amazon-s3 - - - software.amazon.awssdk - url-connection-client - io.github.redhat-appstudio.jvmbuild hacbs-classfile-tracker - org.cyclonedx - cyclonedx-core-java + io.github.redhat-appstudio.jvmbuild + hacbs-build-recipies-database - org.apache.commons - commons-compress + io.github.redhat-appstudio.jvmbuild + hacbs-resource-model + io.quarkus quarkus-rest-client-reactive-jackson @@ -45,10 +34,6 @@ io.quarkus quarkus-container-image-jib - - io.github.redhat-appstudio.jvmbuild - hacbs-resource-model - io.quarkus quarkus-picocli @@ -58,29 +43,40 @@ quarkus-jgit - io.github.redhat-appstudio.jvmbuild - hacbs-build-recipies-database + io.quarkiverse.mavenresolver + quarkus-maven-resolver + com.google.cloud.tools jib-core - org.apache.commons - commons-text + org.cyclonedx + cyclonedx-core-java + + org.ow2.asm + asm-tree + + org.gradle gradle-tooling-api - org.apache.maven - maven-embedder + org.apache.maven.indexer + indexer-core + + + org.apache.ant + ant - org.apache.maven - maven-model + org.apache.ivy + ivy + io.quarkus quarkus-junit5 @@ -91,54 +87,16 @@ quarkus-junit5-internal test - - io.rest-assured - rest-assured - test - org.testcontainers localstack test - - com.amazonaws - aws-java-sdk-s3 - test - org.assertj assertj-core test - - org.apache.maven.indexer - indexer-core - - - org.apache.maven.resolver - maven-resolver-connector-basic - - - org.apache.maven.resolver - maven-resolver-transport-file - - - org.apache.maven.resolver - maven-resolver-transport-http - - - org.ow2.asm - asm-tree - - - org.apache.ant - ant - - - org.apache.ivy - ivy - diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java index 1cd5afe10..577e00ee3 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java @@ -1,5 +1,7 @@ package com.redhat.hacbs.container.analyser.deploy; +import static org.apache.commons.lang3.ObjectUtils.isNotEmpty; + import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -19,6 +21,7 @@ import java.util.stream.Stream; import jakarta.enterprise.inject.spi.BeanManager; +import jakarta.inject.Inject; import org.cyclonedx.BomGeneratorFactory; import org.cyclonedx.CycloneDxSchema; @@ -28,14 +31,17 @@ import com.redhat.hacbs.classfile.tracker.TrackingData; import com.redhat.hacbs.container.analyser.dependencies.SBomGenerator; import com.redhat.hacbs.container.analyser.deploy.containerregistry.ContainerRegistryDeployer; +import com.redhat.hacbs.container.analyser.deploy.mavenrepository.MavenRepositoryDeployer; import com.redhat.hacbs.container.results.ResultsUpdater; import com.redhat.hacbs.recipies.util.FileUtil; import com.redhat.hacbs.resources.model.v1alpha1.dependencybuildstatus.Contaminates; import com.redhat.hacbs.resources.util.HashUtil; +import io.quarkus.bootstrap.resolver.maven.BootstrapMavenContext; import io.quarkus.logging.Log; import picocli.CommandLine; +@SuppressWarnings("OptionalUsedAsFieldOrParameterType") @CommandLine.Command(name = "deploy") public class DeployCommand implements Runnable { @@ -47,22 +53,22 @@ public class DeployCommand implements Runnable { final BeanManager beanManager; final ResultsUpdater resultsUpdater; - @CommandLine.Option(required = false, names = "--allowed-sources", defaultValue = "redhat,rebuilt", split = ",") + @CommandLine.Option(names = "--allowed-sources", defaultValue = "redhat,rebuilt", split = ",") Set allowedSources; @CommandLine.Option(required = true, names = "--path") Path deploymentPath; - @CommandLine.Option(required = false, names = "--task-run-name") + @CommandLine.Option(names = "--task-run-name") String taskRun; @CommandLine.Option(required = true, names = "--source-path") Path sourcePath; - @CommandLine.Option(required = false, names = "--logs-path") + @CommandLine.Option(names = "--logs-path") Path logsPath; - @CommandLine.Option(required = false, names = "--build-info-path") + @CommandLine.Option(names = "--build-info-path") Path buildInfoPath; @CommandLine.Option(required = true, names = "--scm-uri") @@ -78,7 +84,7 @@ public class DeployCommand implements Runnable { @CommandLine.Option(names = "--registry-owner", defaultValue = "hacbs") String owner; @ConfigProperty(name = "registry.token") - Optional token; + Optional token = Optional.empty(); @CommandLine.Option(names = "--registry-repository", defaultValue = "artifact-deployments") String repository; @CommandLine.Option(names = "--registry-insecure", defaultValue = "false") @@ -92,9 +98,26 @@ public class DeployCommand implements Runnable { @CommandLine.Option(names = "--registry-prepend-tag", defaultValue = "") String prependTag; + + // Maven Repo Deployment specification + @CommandLine.Option(names = "--mvn-username") + String mvnUser; + + @ConfigProperty(name = "maven.password") + Optional mvnPassword = Optional.empty(); + + @CommandLine.Option(names = "--mvn-repo") + String mvnRepo; + + // Testing only ; used to disable image deployment + protected boolean imageDeployment = true; + protected String imageName; protected String imageDigest; + @Inject + BootstrapMavenContext mvnCtx; + public DeployCommand(BeanManager beanManager, ResultsUpdater resultsUpdater) { this.beanManager = beanManager; @@ -198,7 +221,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO Files.walkFileTree(deploymentPath, new SimpleFileVisitor<>() { @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { Log.errorf("Contents: %s", file); return FileVisitResult.CONTINUE; } @@ -206,7 +229,6 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO throw new RuntimeException("deploy failed"); } //we still deploy, but without the contaminates - java.nio.file.Path deployFile = deploymentPath; // This means the build failed to produce any deployable output. // If everything is contaminated we still need the task to succeed so we can resolve the contamination. for (var i : contaminatedGavs.entrySet()) { @@ -217,7 +239,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO if (!gavs.isEmpty()) { try { cleanBrokenSymlinks(sourcePath); - doDeployment(deployFile, sourcePath, logsPath, gavs); + doDeployment(sourcePath, logsPath, gavs); } catch (Throwable t) { Log.error("Deployment failed", t); flushLogs(); @@ -301,18 +323,26 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th } - protected void doDeployment(Path deployDir, Path sourcePath, Path logsPath, Set gavs) throws Exception { - - ContainerRegistryDeployer deployer = new ContainerRegistryDeployer(host, port, owner, token.orElse(""), repository, - insecure, - prependTag, imageId); - deployer.deployArchive(deployDir, sourcePath, logsPath, gavs, new BiConsumer() { - @Override - public void accept(String s, String hash) { - imageName = s; - imageDigest = hash; - } - }); + protected void doDeployment(Path sourcePath, Path logsPath, Set gavs) throws Exception { + // TODO: Putting mvn deployment first as image deployment deletes the file tree + if (isNotEmpty(mvnRepo)) { + // Maven Repo Deployment + MavenRepositoryDeployer deployer = new MavenRepositoryDeployer(mvnCtx, mvnUser, mvnPassword.orElse(""), mvnRepo, + deploymentPath); + deployer.deploy(); + } + if (imageDeployment) { + ContainerRegistryDeployer deployer = new ContainerRegistryDeployer(host, port, owner, token.orElse(""), repository, + insecure, prependTag, + imageId); + deployer.deployArchive(deploymentPath, sourcePath, logsPath, gavs, new BiConsumer() { + @Override + public void accept(String s, String hash) { + imageName = s; + imageDigest = hash; + } + }); + } } private void flushLogs() { diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/containerregistry/ContainerRegistryDeployer.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/containerregistry/ContainerRegistryDeployer.java index 5be1b0b47..209ec28e8 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/containerregistry/ContainerRegistryDeployer.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/containerregistry/ContainerRegistryDeployer.java @@ -33,7 +33,6 @@ import com.google.cloud.tools.jib.api.buildplan.ImageFormat; import com.redhat.hacbs.container.analyser.deploy.DeployData; import com.redhat.hacbs.container.analyser.deploy.Gav; -import com.redhat.hacbs.recipies.util.FileUtil; import io.quarkus.logging.Log; @@ -93,13 +92,8 @@ public void deployArchive(Path deployDir, Path sourcePath, Path logsPath, Set gavNames) throws Exception { @@ -131,8 +125,7 @@ public void tagArchive(List gavNames) throws Exception { containerizer = containerizer.withAdditionalTag(gav.getTag()); } containerBuilder.addLabel("io.jvmbuildservice.gavs", String.join(",", gavNames)); - - var result = containerBuilder.containerize(containerizer); + containerBuilder.containerize(containerizer); } public void deployPreBuildImage(String baseImage, Path sourcePath, String imageSourcePath, String tag) @@ -172,7 +165,7 @@ public FilePermissions get(Path sourcePath, AbsoluteUnixPath destinationPath) { containerBuilder.addFileEntriesLayer(layerConfigurationBuilder.build()); Log.debugf("Image %s created", imageName); - var result = containerBuilder.containerize(containerizer); + containerBuilder.containerize(containerizer); } } @@ -211,7 +204,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO }); containerBuilder.addFileEntriesLayer(layerConfigurationBuilder.build()); Log.debugf("Image %s created", imageName); - var result = containerBuilder.containerize(containerizer); + containerBuilder.containerize(containerizer); } private void createImages(DeployData imageData, Path sourcePath, Path logsPath, diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java new file mode 100644 index 000000000..08d87c127 --- /dev/null +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/mavenrepository/MavenRepositoryDeployer.java @@ -0,0 +1,120 @@ +package com.redhat.hacbs.container.analyser.deploy.mavenrepository; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.maven.repository.internal.MavenRepositorySystemUtils; +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositorySystem; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.deployment.DeployRequest; +import org.eclipse.aether.deployment.DeploymentException; +import org.eclipse.aether.repository.LocalRepository; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.util.repository.AuthenticationBuilder; + +import io.quarkus.bootstrap.resolver.maven.BootstrapMavenContext; +import io.quarkus.bootstrap.resolver.maven.BootstrapMavenException; +import io.quarkus.logging.Log; + +public class MavenRepositoryDeployer { + private final String username; + + private final String password; + + private final String repository; + + private final Path artifacts; + + private final RepositorySystem system; + + private final DefaultRepositorySystemSession session; + + public MavenRepositoryDeployer(BootstrapMavenContext mvnCtx, String username, String password, String repository, + Path artifacts) + throws BootstrapMavenException { + this.username = username; + this.password = password; + this.repository = repository; + this.artifacts = artifacts; + + this.system = mvnCtx.getRepositorySystem(); + this.session = MavenRepositorySystemUtils.newSession(); + + // https://maven.apache.org/resolver/third-party-integrations.html states a local repository manager should be added. + session.setLocalRepositoryManager(system.newLocalRepositoryManager(session, new LocalRepository(artifacts.toFile()))); + } + + public void deploy() + throws IOException { + RemoteRepository distRepo = new RemoteRepository.Builder("repo", + "default", + repository) + .setAuthentication(new AuthenticationBuilder().addUsername(username) + .addPassword(password).build()) + .build(); + + Files.walkFileTree(artifacts, + new SimpleFileVisitor<>() { + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + try (var stream = Files.list(dir)) { + List files = stream.sorted().toList(); + boolean hasPom = files.stream().anyMatch(s -> s.toString().endsWith(".pom")); + if (hasPom) { + + Path relative = artifacts.relativize(dir); + String group = relative.getParent().getParent().toString().replace("/", + "."); + String artifact = relative.getParent().getFileName().toString(); + String version = dir.getFileName().toString(); + Log.info( + "GROUP: " + group + " , ARTIFACT:" + artifact + " , VERSION: " + + version); + Pattern p = Pattern + .compile(artifact + "-" + version + "(-(\\w+))?\\.(\\w+)"); + + DeployRequest deployRequest = new DeployRequest(); + deployRequest.setRepository(distRepo); + for (var i : files) { + Matcher matcher = p.matcher(i.getFileName().toString()); + if (matcher.matches()) { + Artifact jarArtifact = new DefaultArtifact(group, artifact, + matcher.group(2), + matcher.group(3), + version); + jarArtifact = jarArtifact.setFile(i.toFile()); + deployRequest.addArtifact(jarArtifact); + } + } + + try { + Log.infof("Deploying %s", deployRequest); + system.deploy(session, deployRequest); + } catch (DeploymentException e) { + throw new RuntimeException(e); + } + } else { + if (files.stream().anyMatch(p -> !p.toFile().isDirectory())) { + Log.warnf("For directory %s, no pom file found with files %s", dir, + files); + } + } + + return FileVisitResult.CONTINUE; + } + } + + }); + } +} diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java index 7c4ff3690..d381957e1 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/MavenUtils.java @@ -3,7 +3,6 @@ import static org.apache.commons.io.FilenameUtils.normalize; import static org.apache.maven.cli.configuration.SettingsXmlConfigurationProcessor.DEFAULT_GLOBAL_SETTINGS_FILE; import static org.apache.maven.cli.configuration.SettingsXmlConfigurationProcessor.DEFAULT_USER_SETTINGS_FILE; -import static org.apache.maven.repository.internal.MavenRepositorySystemUtils.*; import static org.apache.maven.settings.MavenSettingsBuilder.ALT_GLOBAL_SETTINGS_XML_LOCATION; import static org.apache.maven.settings.MavenSettingsBuilder.ALT_USER_SETTINGS_XML_LOCATION; @@ -21,16 +20,11 @@ import org.eclipse.aether.RepositorySystem; import org.eclipse.aether.RepositorySystemSession; import org.eclipse.aether.artifact.DefaultArtifact; -import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory; import org.eclipse.aether.repository.AuthenticationSelector; import org.eclipse.aether.repository.MirrorSelector; import org.eclipse.aether.repository.RemoteRepository; import org.eclipse.aether.resolution.ArtifactRequest; import org.eclipse.aether.resolution.ArtifactResolutionException; -import org.eclipse.aether.spi.connector.RepositoryConnectorFactory; -import org.eclipse.aether.spi.connector.transport.TransporterFactory; -import org.eclipse.aether.transport.file.FileTransporterFactory; -import org.eclipse.aether.transport.http.HttpTransporterFactory; import org.eclipse.aether.util.repository.AuthenticationBuilder; import org.eclipse.aether.util.repository.DefaultAuthenticationSelector; import org.eclipse.aether.util.repository.DefaultMirrorSelector; @@ -39,14 +33,6 @@ public class MavenUtils { private static final Logger Log = Logger.getLogger(MavenUtils.class); - public static RepositorySystem newRepositorySystem() { - var locator = newServiceLocator(); - locator.addService(RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class); - locator.addService(TransporterFactory.class, FileTransporterFactory.class); - locator.addService(TransporterFactory.class, HttpTransporterFactory.class); - return locator.getService(RepositorySystem.class); - } - public static Settings newSettings(Path globalSettingsFile, Path settingsFile) { try { var builderFactory = new DefaultSettingsBuilderFactory(); diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java index e7e5bb33f..e8dec461b 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java @@ -2,7 +2,6 @@ import static com.redhat.hacbs.container.verifier.MavenUtils.newAuthenticationSelector; import static com.redhat.hacbs.container.verifier.MavenUtils.newMirrorSelector; -import static com.redhat.hacbs.container.verifier.MavenUtils.newRepositorySystem; import static com.redhat.hacbs.container.verifier.MavenUtils.newSettings; import static com.redhat.hacbs.container.verifier.MavenUtils.pathToCoords; import static com.redhat.hacbs.container.verifier.MavenUtils.resolveArtifact; @@ -42,6 +41,8 @@ import com.redhat.hacbs.container.results.ResultsUpdater; import com.redhat.hacbs.container.verifier.asm.JarInfo; +import io.quarkus.bootstrap.resolver.maven.BootstrapMavenContext; +import io.quarkus.bootstrap.resolver.maven.BootstrapMavenException; import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -112,6 +113,9 @@ static class Options { @Inject Instance resultsUpdater; + @Inject + BootstrapMavenContext mvnCtx; + public VerifyBuiltArtifactsCommand() { } @@ -232,14 +236,15 @@ private List getExcludes() throws IOException { return newExcludes; } - private void initMaven(Path globalSettingsFile, Path settingsFile) throws IOException { + private void initMaven(Path globalSettingsFile, Path settingsFile) + throws IOException, BootstrapMavenException { session = newSession(); var settings = newSettings(globalSettingsFile, settingsFile); var mirrorSelector = newMirrorSelector(settings); session.setMirrorSelector(mirrorSelector); var authenticationSelector = newAuthenticationSelector(settings); session.setAuthenticationSelector(authenticationSelector); - system = newRepositorySystem(); + system = mvnCtx.getRepositorySystem(); var localRepositoryDirectory = Files.createTempDirectory("verify-built-artifacts-"); var localRepository = new LocalRepository(localRepositoryDirectory.toFile()); var manager = system.newLocalRepositoryManager(session, localRepository); diff --git a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/ContainerRegistryDeployerTest.java b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/ContainerRegistryDeployerTest.java index c814e6497..fb6c74612 100644 --- a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/ContainerRegistryDeployerTest.java +++ b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/ContainerRegistryDeployerTest.java @@ -60,9 +60,7 @@ private static int startTestRegistry() { container.start(); - Integer port = container.getMappedPort(5000); - - return port; + return container.getMappedPort(5000); } @AfterAll @@ -113,7 +111,8 @@ public void testDeployArchive(QuarkusMainLauncher launcher) throws IOException { } private Path createDeploymentRepo() throws IOException { - Path artifacts = Paths.get("target/test-data/artifacts").toAbsolutePath(); + Path testData = Files.createTempDirectory("test-data"); + Path artifacts = Paths.get(testData.toString(), "artifacts").toAbsolutePath(); Files.createDirectories(artifacts); // Add data to artifacts folder @@ -180,7 +179,7 @@ private String readHTTPData(URL url) throws IOException { } private URL getRegistryURL(String path) throws IOException { - return new URL("http://" + this.container.getHost() + ":" + port + "/v2/" + path); + return new URL("http://" + container.getHost() + ":" + port + "/v2/" + path); } class ContainerRegistryDetails { diff --git a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/DeployContaminateTest.java b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/DeployContaminateTest.java index cc14af5e4..eb18eb79b 100644 --- a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/DeployContaminateTest.java +++ b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/DeployContaminateTest.java @@ -20,7 +20,6 @@ import jakarta.enterprise.inject.spi.BeanManager; import jakarta.inject.Inject; -import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -57,7 +56,8 @@ public void clearLogs() { @Test public void testDeployOnlyContaminated() throws IOException, URISyntaxException { - Path onDiskRepo = Paths.get("target/test-data/artifacts").toAbsolutePath(); + Path testData = Files.createTempDirectory("test-data"); + Path onDiskRepo = Paths.get(testData.toString(), "artifacts").toAbsolutePath(); onDiskRepo.toFile().mkdirs(); Path source = Files.createTempDirectory("hacbs"); Files.writeString(source.resolve("pom.xml"), ""); @@ -110,8 +110,8 @@ public void testDeployWithContaminated() private Path createDeploymentRepo() throws IOException, URISyntaxException { - Path artifacts = Paths.get("target/test-data/artifacts").toAbsolutePath(); - FileUtils.deleteDirectory(artifacts.toFile()); + Path testData = Files.createTempDirectory("test-data"); + Path artifacts = Paths.get(testData.toString(), "artifacts").toAbsolutePath(); Files.createDirectories(artifacts); // Add data to artifacts folder @@ -152,9 +152,9 @@ public TestDeployment(BeanManager beanManager, ResultsUpdater resultsUpdater) { } @Override - protected void doDeployment(Path deployDir, Path sourcePath, Path logsPath, Set gavs) + protected void doDeployment(Path sourcePath, Path logsPath, Set gavs) throws Exception { - System.out.println("Skipping doDeployment for " + deployDir + " from " + sourcePath); + System.out.println("Skipping doDeployment for " + deploymentPath + " from " + sourcePath); } } } diff --git a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/MavenDeployTest.java b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/MavenDeployTest.java new file mode 100644 index 000000000..a0d10520a --- /dev/null +++ b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/analyser/deploy/MavenDeployTest.java @@ -0,0 +1,131 @@ +package com.redhat.hacbs.container.analyser.deploy; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.LogRecord; + +import jakarta.inject.Inject; + +import org.apache.maven.model.Model; +import org.apache.maven.model.io.xpp3.MavenXpp3Writer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.redhat.hacbs.container.results.ResultsUpdater; +import com.redhat.hacbs.resources.util.HashUtil; + +import io.quarkus.bootstrap.resolver.maven.BootstrapMavenContext; +import io.quarkus.test.LogCollectingTestResource; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.common.ResourceArg; +import io.quarkus.test.junit.QuarkusTest; + +@QuarkusTest +@QuarkusTestResource(value = LogCollectingTestResource.class, restrictToAnnotatedClass = true, initArgs = @ResourceArg(name = LogCollectingTestResource.LEVEL, value = "FINE")) +public class MavenDeployTest { + private static final String GROUP = "com.company.foo"; + private static final String VERSION = "3.25.8"; + public static final String FOO_BAR = "foo-bar"; + public static final String FOO_BAZ = "foo-baz"; + private static final String DOT = "."; + private static final String SHA_1 = "sha1"; + public static final String COMMIT = "3cf2d99b47f0a05466d1d0a2e09d8740faeda149"; + public static final String REPO = "https://github.com/foo/bar"; + + // ArtifactName -> [ collection of Artifacts ] + private final Map> ARTIFACT_FILE_MAP = Map.of( + FOO_BAR, Set.of("foo-bar-" + VERSION + ".jar", "foo-bar-" + VERSION + "-tests.jar"), + FOO_BAZ, Set.of("foo-baz-" + VERSION + ".pom")); + + @Inject + ResultsUpdater resultsUpdater; + + @Inject + BootstrapMavenContext mvnContext; + + @BeforeEach + public void clearLogs() { + LogCollectingTestResource.current().clear(); + } + + @Test + public void testDeploy() throws IOException { + Path onDiskRepo = createDeploymentRepo(); + Path source = Files.createTempDirectory("hacbs"); + Path deployment = Files.createTempDirectory("deployment"); + Files.writeString(source.resolve("pom.xml"), ""); + + DeployCommand deployCommand = new DeployCommand(null, resultsUpdater); + deployCommand.deploymentPath = onDiskRepo.toAbsolutePath(); + deployCommand.imageId = "test-image"; + deployCommand.scmUri = REPO; + deployCommand.commit = COMMIT; + deployCommand.sourcePath = source.toAbsolutePath(); + deployCommand.allowedSources = Set.of("redhat", "rebuilt"); // Default value + deployCommand.imageDeployment = false; + deployCommand.mvnRepo = "file://" + deployment.toAbsolutePath(); + deployCommand.mvnCtx = mvnContext; + + deployCommand.run(); + List logRecords = LogCollectingTestResource.current().getRecords(); + assertTrue(logRecords.stream().anyMatch(r -> LogCollectingTestResource.format(r) + .contains("GAVs to deploy: [com.company.foo:foo-bar:3.25.8, com.company.foo:foo-baz:3.25.8]"))); + assertTrue(logRecords.stream().anyMatch(r -> LogCollectingTestResource.format(r) + .contains("Deploying [com.company.foo:foo-baz:pom:3.25.8]"))); + assertTrue(logRecords.stream().anyMatch(r -> LogCollectingTestResource.format(r) + .contains( + "Deploying [com.company.foo:foo-bar:jar:tests:3.25.8, com.company.foo:foo-bar:jar:3.25.8, com.company.foo:foo-bar:pom:3.25.8]"))); + + File[] files = Paths.get(deployment.toString(), "com/company/foo/foo-bar/3.25.8").toFile().listFiles(); + assertNotNull(files); + assertEquals(9, files.length); + } + + private Path createDeploymentRepo() + throws IOException { + Path testData = Files.createTempDirectory("test-data"); + Path artifacts = Paths.get(testData.toString(), "artifacts").toAbsolutePath(); + Files.createDirectories(artifacts); + + // Add data to artifacts folder + for (Map.Entry> artifactFiles : ARTIFACT_FILE_MAP.entrySet()) { + String groupPath = GROUP.replace(DOT, File.separator); + Path testDir = Paths.get(artifacts.toString(), groupPath, artifactFiles.getKey(), + VERSION); + Files.createDirectories(testDir); + for (String value : artifactFiles.getValue()) { + Path testFile = Paths.get(testDir.toString(), value); + Files.createFile(testFile); + String testContent = "Just some data for " + artifactFiles.getKey(); + Files.writeString(testFile, testContent); + Path shaFile = Paths.get(testFile + DOT + SHA_1); + Files.createFile(shaFile); + String sha1 = HashUtil.sha1(Files.readAllBytes(testFile)); + Files.writeString(shaFile, sha1); + } + Optional jarFile = artifactFiles.getValue().stream().filter(s -> s.endsWith(VERSION + ".jar")).findAny(); + if (jarFile.isPresent()) { + System.err.println("### [TEST] Generating pom file for " + jarFile.get()); + Path pomFile = Paths.get(testDir.toString(), jarFile.get().replace(".jar", ".pom")); + Model model = new Model(); + model.setArtifactId(artifactFiles.getKey()); + model.setGroupId(GROUP); + model.setVersion(VERSION); + new MavenXpp3Writer().write(new FileWriter(pomFile.toFile()), model); + } + } + return artifacts; + } +} diff --git a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommandTest.java b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommandTest.java index f103505ac..cafb8d4ca 100644 --- a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommandTest.java +++ b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommandTest.java @@ -17,6 +17,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.parallel.ResourceLock; +import io.quarkus.bootstrap.resolver.maven.BootstrapMavenContext; import picocli.CommandLine; public class VerifyBuiltArtifactsCommandTest { @@ -57,7 +58,9 @@ void testCommand() throws Exception { var file = Path.of(uri); System.setProperty(ALT_USER_SETTINGS_XML_LOCATION, file.toAbsolutePath().toString()); var args = new String[] { "-r", "https://repo1.maven.org/maven2", "--deploy-path", tempMavenRepo.toString() }; - var exitCode = new CommandLine(new VerifyBuiltArtifactsCommand()).execute(args); + VerifyBuiltArtifactsCommand verifyBuiltArtifactsCommand = new VerifyBuiltArtifactsCommand(); + verifyBuiltArtifactsCommand.mvnCtx = new BootstrapMavenContext(); + var exitCode = new CommandLine(verifyBuiltArtifactsCommand).execute(args); assertThat(exitCode).isZero(); //now modify junit and add an extra class @@ -80,14 +83,14 @@ void testCommand() throws Exception { } args = new String[] { "-r", "https://repo1.maven.org/maven2", "--deploy-path", tempMavenRepo.toString() }; - exitCode = new CommandLine(new VerifyBuiltArtifactsCommand()).execute(args); + exitCode = new CommandLine(verifyBuiltArtifactsCommand).execute(args); assertThat(exitCode).isNotZero(); //now try with excludes args = new String[] { "-r", "https://repo1.maven.org/maven2", "--deploy-path", tempMavenRepo.toString(), "--excludes", "+:.*[^:]:class:" + VerifyBuiltArtifactsCommandTest.class.getName().replace(".", "/") }; - exitCode = new CommandLine(new VerifyBuiltArtifactsCommand()).execute(args); + exitCode = new CommandLine(verifyBuiltArtifactsCommand).execute(args); assertThat(exitCode).isZero(); } diff --git a/java-components/pom.xml b/java-components/pom.xml index 90389e197..d55a9ce67 100644 --- a/java-components/pom.xml +++ b/java-components/pom.xml @@ -151,6 +151,11 @@ quarkus-openapi-generator 2.2.11 + + io.quarkiverse.mavenresolver + quarkus-maven-resolver + 0.0.4 + org.cyclonedx diff --git a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml index 237604b3f..38213e683 100644 --- a/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml +++ b/java-components/resource-model/src/main/resources/crds/jvmbuildservice.io_jbsconfigs.yaml @@ -97,6 +97,13 @@ spec: additionalProperties: type: string type: object + mavenDeployment: + properties: + repository: + type: string + username: + type: string + type: object owner: type: string port: diff --git a/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go b/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go index 92718a3a0..e46c6c5c8 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/jbsconfig_types.go @@ -8,12 +8,14 @@ type HermeticBuildType string const ( JBSConfigName = "jvm-build-config" - DefaultImageSecretName = "jvm-build-image-secrets" //#nosec - GitSecretName = "jvm-build-git-secrets" //#nosec - TlsSecretName = "jvm-build-tls-secrets" //#nosec - TlsConfigMapName = "jvm-build-tls-ca" //#nosec - ImageSecretTokenKey = ".dockerconfigjson" //#nosec - GitSecretTokenKey = ".git-credentials" //#nosec + DefaultImageSecretName = "jvm-build-image-secrets" //#nosec + GitSecretName = "jvm-build-git-secrets" //#nosec + TlsSecretName = "jvm-build-tls-secrets" //#nosec + TlsConfigMapName = "jvm-build-tls-ca" //#nosec + ImageSecretTokenKey = ".dockerconfigjson" //#nosec + GitSecretTokenKey = ".git-credentials" //#nosec + MavenSecretKey = "mavenpassword" //#nosec + MavenSecretName = "jvm-build-maven-repo-secrets" //#nosec CacheDeploymentName = "jvm-build-workspace-artifact-cache" ConfigArtifactCacheRequestMemoryDefault = "512Mi" ConfigArtifactCacheRequestCPUDefault = "1" @@ -41,6 +43,7 @@ type JBSConfigSpec struct { SharedRegistries []ImageRegistry `json:"sharedRegistries,omitempty"` Registry ImageRegistry `json:"registry,omitempty"` + MavenDeployment MavenDeployment `json:"mavenDeployment,omitempty"` // Deprecated: Replaced by explicit declaration of Registry above. ImageRegistry `json:",inline,omitempty"` CacheSettings CacheSettings `json:"cacheSettings,omitempty"` @@ -89,6 +92,11 @@ type ImageRegistry struct { SecretName string `json:"secretName,omitempty"` } +type MavenDeployment struct { + Username string `json:"username,omitempty"` + Repository string `json:"repository,omitempty"` +} + type RelocationPatternElement struct { RelocationPattern RelocationPattern `json:"relocationPattern"` } diff --git a/pkg/reconciler/dependencybuild/buildrecipeyaml.go b/pkg/reconciler/dependencybuild/buildrecipeyaml.go index 663681eb2..6d170a4f6 100644 --- a/pkg/reconciler/dependencybuild/buildrecipeyaml.go +++ b/pkg/reconciler/dependencybuild/buildrecipeyaml.go @@ -147,12 +147,16 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC {Name: PipelineParamEnforceVersion, Type: pipelinev1beta1.ParamTypeString}, {Name: PipelineParamCacheUrl, Type: pipelinev1beta1.ParamTypeString, Default: &pipelinev1beta1.ResultValue{Type: pipelinev1beta1.ParamTypeString, StringVal: cacheUrl + buildRepos + "/" + strconv.FormatInt(commitTime, 10)}}, } - registryToken := []v1.EnvVar{} + registryToken := make([]v1.EnvVar, 0) if jbsConfig.ImageRegistry().SecretName != "" { registryToken = []v1.EnvVar{ {Name: "REGISTRY_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: jbsConfig.ImageRegistry().SecretName}, Key: v1alpha12.ImageSecretTokenKey, Optional: &trueBool}}}, } } + if jbsConfig.Spec.MavenDeployment.Repository != "" { + registryToken = append(registryToken, v1.EnvVar{Name: "MAVEN_PASSWORD", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: v1alpha12.MavenSecretName}, Key: v1alpha12.MavenSecretKey, Optional: &trueBool}}}) + } + buildSetup := pipelinev1beta1.TaskSpec{ Workspaces: []pipelinev1beta1.WorkspaceDeclaration{{Name: WorkspaceBuildSettings}, {Name: WorkspaceSource}, {Name: WorkspaceTls}}, Params: pipelineParams, @@ -597,7 +601,7 @@ func imageRegistryCommands(imageId string, recipe *v1alpha12.BuildRecipe, db *v1 "--image-id=" + imageIdToTag, } imageRegistry := jbsConfig.ImageRegistry() - registryArgs := []string{} + registryArgs := make([]string, 0) if imageRegistry.Host != "" { registryArgs = append(registryArgs, "--registry-host="+imageRegistry.Host) preBuildImageName += imageRegistry.Host @@ -635,6 +639,15 @@ func imageRegistryCommands(imageId string, recipe *v1alpha12.BuildRecipe, db *v1 tagArgs = append(tagArgs, registryArgs...) tagArgs = append(tagArgs, "$(params.GAVS)") + mavenArgs := make([]string, 0) + if jbsConfig.Spec.MavenDeployment.Repository != "" { + mavenArgs = append(mavenArgs, "--mvn-repo="+jbsConfig.Spec.MavenDeployment.Repository) + } + if jbsConfig.Spec.MavenDeployment.Username != "" { + mavenArgs = append(mavenArgs, "--mvn-username="+jbsConfig.Spec.MavenDeployment.Username) + } + deployArgs = append(deployArgs, mavenArgs...) + hermeticPreBuildImageArgs := []string{ "deploy-hermetic-pre-build-image", "--source-image=" + preBuildImageName,