From 033e4ad6758746415955b41b9cdebb28bacef432 Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Thu, 19 Dec 2024 11:30:52 +0000 Subject: [PATCH 1/4] Use separate konflux-tooling --- deploy/pipeline/mw-pipeline-v0.1.yaml | 2 +- deploy/tasks/pre-build.yaml | 59 +-------------------------- 2 files changed, 2 insertions(+), 59 deletions(-) diff --git a/deploy/pipeline/mw-pipeline-v0.1.yaml b/deploy/pipeline/mw-pipeline-v0.1.yaml index 8edd09daa..c9303658a 100644 --- a/deploy/pipeline/mw-pipeline-v0.1.yaml +++ b/deploy/pipeline/mw-pipeline-v0.1.yaml @@ -108,7 +108,7 @@ spec: resolver: http params: - name: url - value: https://raw.githubusercontent.com/rnc/jvm-build-service/refs/heads/BR3/deploy/tasks/pre-build.yaml + value: https://raw.githubusercontent.com/rnc/jvm-build-service/refs/heads/BR1/deploy/tasks/pre-build.yaml workspaces: - name: source workspace: source diff --git a/deploy/tasks/pre-build.yaml b/deploy/tasks/pre-build.yaml index ca98b22b3..01ab7a988 100644 --- a/deploy/tasks/pre-build.yaml +++ b/deploy/tasks/pre-build.yaml @@ -19,28 +19,6 @@ spec: - name: NAME description: Name of the pipeline run (i.e. unique dependency build name) type: string - - name: GIT_IDENTITY - description: Git username. If empty, deploy-pre-build-source step will be skipped. - type: string - default: "" - - name: GIT_URL - description: String to determine whether we're using gitlab or github - type: string - default: "github" - - name: GIT_SSL_VERIFICATION - description: Whether to disable ssl verification - type: string - default: "false" - - name: GIT_REUSE_REPOSITORY - description: Whether to reuse existing git repository or create new one - type: string - default: "false" - - name: SCM_URL - description: Reference to the git repository - type: string - - name: SCM_HASH - description: Git hash - type: string - name: RECIPE_IMAGE description: The image from the build recipe to use - name: BUILD_TOOL @@ -49,9 +27,6 @@ spec: description: The build tool version to use (e.g. 3.9.5) - name: JAVA_VERSION description: Java version to use (7, 8, 9, 11, 17, 21, 22, 23) - - name: BUILD_PLUGINS - description: Optional comma separated list of build plugins that should be disabled. - default: "" - name: BUILD_SCRIPT description: The build script to embed with the Containerfile - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE @@ -62,8 +37,6 @@ spec: results: - name: PRE_BUILD_IMAGE_DIGEST description: Digest of the image just built - - name: GIT_ARCHIVE - description: Git archive information workspaces: - description: The git repo will be cloned onto the volume backing this Workspace. name: source @@ -81,40 +54,10 @@ spec: cpu: 10m memory: 512Mi script: | - /opt/jboss/container/java/run/run-java.sh $(params.BUILD_TOOL)-prepare --java-version=$(params.JAVA_VERSION) --build-tool-version=$(params.BUILD_TOOL_VERSION) --recipe-image=$(params.RECIPE_IMAGE) --request-processor-image=$(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) --disabled-plugins=$(params.BUILD_PLUGINS) $(workspaces.source.path)/source + /opt/jboss/container/java/run/run-java.sh prepare --type=$(params.BUILD_TOOL) --java-version=$(params.JAVA_VERSION) --build-tool-version=$(params.BUILD_TOOL_VERSION) --recipe-image=$(params.RECIPE_IMAGE) --request-processor-image=$(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) $(workspaces.source.path)/source env: - name: BUILD_SCRIPT value: $(params.BUILD_SCRIPT) - # TODO: Look at making this optional until we know whether we need to store source - - name: create-pre-build-source - image: $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) - securityContext: - runAsUser: 0 - computeResources: - limits: - cpu: 300m - memory: 512Mi - requests: - cpu: 10m - memory: 512Mi - env: - - name: GIT_DEPLOY_TOKEN - valueFrom: - secretKeyRef: - optional: true - name: jvm-build-git-repo-secrets - key: gitdeploytoken - args: - - deploy-pre-build-source - - --source-path=$(workspaces.source.path)/source - - --task-run-name=$(context.taskRun.name) - - --scm-uri=$(params.SCM_URL) - - --scm-commit=$(params.SCM_HASH) - - --image-id=$(params.NAME) - - --git-identity=$(params.GIT_IDENTITY) - - --git-url=$(params.GIT_URL) - - --git-disable-ssl-verification=$(params.GIT_SSL_VERIFICATION) - - --git-reuse-repository=$(params.GIT_REUSE_REPOSITORY) - name: create-pre-build-image image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:52f1391e6f1c472fd10bb838f64fae2ed3320c636f536014978a5ddbdfc6b3af script: | From 7d78120cc8b9fc0ed096208854cc37105381f02b Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Thu, 19 Dec 2024 14:45:14 +0000 Subject: [PATCH 2/4] Split prebuild for jbs. Simplify for PNC. --- deploy/pipeline/mw-pipeline-v0.1.yaml | 1 + deploy/tasks/maven-deployment.yaml | 1 - deploy/tasks/pre-build-jbs.yaml | 133 ++++++++++++++++++ deploy/tasks/pre-build.yaml | 15 +- .../preprocessor/AbstractPreprocessor.java | 12 +- .../notification/NotificationTest.java | 3 - .../v1alpha1/systemconfig_types.go | 1 + 7 files changed, 149 insertions(+), 17 deletions(-) create mode 100644 deploy/tasks/pre-build-jbs.yaml diff --git a/deploy/pipeline/mw-pipeline-v0.1.yaml b/deploy/pipeline/mw-pipeline-v0.1.yaml index c9303658a..473abef38 100644 --- a/deploy/pipeline/mw-pipeline-v0.1.yaml +++ b/deploy/pipeline/mw-pipeline-v0.1.yaml @@ -47,6 +47,7 @@ spec: type: string - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE type: string + default: "quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/konflux-tooling:latest" - name: NOTIFICATION_CONTEXT type: string default: "" diff --git a/deploy/tasks/maven-deployment.yaml b/deploy/tasks/maven-deployment.yaml index b25d2a92f..939d8fff0 100644 --- a/deploy/tasks/maven-deployment.yaml +++ b/deploy/tasks/maven-deployment.yaml @@ -42,7 +42,6 @@ spec: - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE description: Name of the processor image. Useful to override for development. type: string - default: "quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/jvm-build-service/build-request-processor:latest" - name: caTrustConfigMapKey description: The name of the key in the ConfigMap that contains the CA bundle data. diff --git a/deploy/tasks/pre-build-jbs.yaml b/deploy/tasks/pre-build-jbs.yaml new file mode 100644 index 000000000..611d3d9bf --- /dev/null +++ b/deploy/tasks/pre-build-jbs.yaml @@ -0,0 +1,133 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: pre-build + annotations: + tekton.dev/pipelines.minVersion: 0.12.1 + tekton.dev/tags: image-build, konflux + labels: + app.kubernetes.io/version: "0.1" + build.appstudio.redhat.com/build_type: docker +spec: + description: |- + Sets up pre-build running the preprocessor, pushing the source and creating the OCI image. + params: + - name: IMAGE_URL + description: URL of the OCI image to use. + type: string + - name: NAME + description: Name of the pipeline run (i.e. unique dependency build name) + type: string + - name: GIT_IDENTITY + description: Git username. If empty, deploy-pre-build-source step will be skipped. + type: string + default: "" + - name: GIT_URL + description: String to determine whether we're using gitlab or github + type: string + default: "github" + - name: GIT_SSL_VERIFICATION + description: Whether to disable ssl verification + type: string + default: "false" + - name: GIT_REUSE_REPOSITORY + description: Whether to reuse existing git repository or create new one + type: string + default: "false" + - name: SCM_URL + description: Reference to the git repository + type: string + - name: SCM_HASH + description: Git hash + type: string + - name: RECIPE_IMAGE + description: The image from the build recipe to use + - name: BUILD_TOOL + description: The build tool to use (ant, gradle, maven, sbt). + - name: BUILD_TOOL_VERSION + description: The build tool version to use (e.g. 3.9.5) + - name: JAVA_VERSION + description: Java version to use (7, 8, 9, 11, 17, 21, 22, 23) + - name: BUILD_PLUGINS + description: Optional comma separated list of build plugins that should be disabled. + default: "" + - name: BUILD_SCRIPT + description: The build script to embed with the Containerfile + - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE + description: Name of the processor image. Useful to override for development. + type: string + default: "quay.io/ncross/hacbs-jvm-build-request-processor:latest" + # default: "quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/jvm-build-service/build-request-processor:latest" + results: + - name: PRE_BUILD_IMAGE_DIGEST + description: Digest of the image just built + - name: GIT_ARCHIVE + description: Git archive information + workspaces: + - description: The git repo will be cloned onto the volume backing this Workspace. + name: source + mountPath: /var/workdir + steps: + - name: preprocessor + image: $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) + securityContext: + runAsUser: 0 + computeResources: + limits: + cpu: 300m + memory: 512Mi + requests: + cpu: 10m + memory: 512Mi + script: | + # Complete temporary hack for backwards compatibility with JBS until tasks are ported to build-definitions + if [[ $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) == *"konflux-tooling"* ]]; then + /opt/jboss/container/java/run/run-java.sh prepare --type=$(params.BUILD_TOOL) --java-version=$(params.JAVA_VERSION) --build-tool-version=$(params.BUILD_TOOL_VERSION) --recipe-image=$(params.RECIPE_IMAGE) --request-processor-image=$(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) $(workspaces.source.path)/source + else + /opt/jboss/container/java/run/run-java.sh $(params.BUILD_TOOL)-prepare --java-version=$(params.JAVA_VERSION) --build-tool-version=$(params.BUILD_TOOL_VERSION) --recipe-image=$(params.RECIPE_IMAGE) --request-processor-image=$(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) --disabled-plugins=$(params.BUILD_PLUGINS) $(workspaces.source.path)/source + fi + env: + - name: BUILD_SCRIPT + value: $(params.BUILD_SCRIPT) + # TODO: Look at making this optional until we know whether we need to store source + - name: create-pre-build-source + image: $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) + securityContext: + runAsUser: 0 + computeResources: + limits: + cpu: 300m + memory: 512Mi + requests: + cpu: 10m + memory: 512Mi + env: + - name: GIT_DEPLOY_TOKEN + valueFrom: + secretKeyRef: + optional: true + name: jvm-build-git-repo-secrets + key: gitdeploytoken + args: + - deploy-pre-build-source + - --source-path=$(workspaces.source.path)/source + - --task-run-name=$(context.taskRun.name) + - --scm-uri=$(params.SCM_URL) + - --scm-commit=$(params.SCM_HASH) + - --image-id=$(params.NAME) + - --git-identity=$(params.GIT_IDENTITY) + - --git-url=$(params.GIT_URL) + - --git-disable-ssl-verification=$(params.GIT_SSL_VERIFICATION) + - --git-reuse-repository=$(params.GIT_REUSE_REPOSITORY) + - name: create-pre-build-image + image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:52f1391e6f1c472fd10bb838f64fae2ed3320c636f536014978a5ddbdfc6b3af + script: | + set -x + echo "IMAGE is $(params.IMAGE_URL)" + cat $HOME/.docker/config.json || true + echo "Creating pre-build-image archive" + create-archive --store $(params.IMAGE_URL) $(results.PRE_BUILD_IMAGE_DIGEST.path)=$(workspaces.source.path)/source + env: + - name: IMAGE_URL + value: $(params.IMAGE_URL) diff --git a/deploy/tasks/pre-build.yaml b/deploy/tasks/pre-build.yaml index 01ab7a988..42b74c8d5 100644 --- a/deploy/tasks/pre-build.yaml +++ b/deploy/tasks/pre-build.yaml @@ -32,8 +32,6 @@ spec: - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE description: Name of the processor image. Useful to override for development. type: string - default: "quay.io/ncross/hacbs-jvm-build-request-processor:latest" - # default: "quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/jvm-build-service/build-request-processor:latest" results: - name: PRE_BUILD_IMAGE_DIGEST description: Digest of the image just built @@ -53,17 +51,20 @@ spec: requests: cpu: 10m memory: 512Mi - script: | - /opt/jboss/container/java/run/run-java.sh prepare --type=$(params.BUILD_TOOL) --java-version=$(params.JAVA_VERSION) --build-tool-version=$(params.BUILD_TOOL_VERSION) --recipe-image=$(params.RECIPE_IMAGE) --request-processor-image=$(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) $(workspaces.source.path)/source + args: + - prepare + - --build-tool-version=$(params.BUILD_TOOL_VERSION) + - --java-version=$(params.JAVA_VERSION) + - --recipe-image=$(params.RECIPE_IMAGE) + - --request-processor-image=$(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) + - --type=$(params.BUILD_TOOL) + - $(workspaces.source.path)/source env: - name: BUILD_SCRIPT value: $(params.BUILD_SCRIPT) - name: create-pre-build-image image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:52f1391e6f1c472fd10bb838f64fae2ed3320c636f536014978a5ddbdfc6b3af script: | - set -x - echo "IMAGE is $(params.IMAGE_URL)" - cat $HOME/.docker/config.json || true echo "Creating pre-build-image archive" create-archive --store $(params.IMAGE_URL) $(results.PRE_BUILD_IMAGE_DIGEST.path)=$(workspaces.source.path)/source env: diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/build/preprocessor/AbstractPreprocessor.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/build/preprocessor/AbstractPreprocessor.java index 5fc037608..8ae3e870a 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/build/preprocessor/AbstractPreprocessor.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/build/preprocessor/AbstractPreprocessor.java @@ -29,16 +29,18 @@ public abstract class AbstractPreprocessor implements Runnable { protected List disabledPlugins; @CommandLine.Option(names = "--recipe-image", required = true) - String recipeImage; + protected String recipeImage; @CommandLine.Option(names = "--request-processor-image", required = true) - String buildRequestProcessorImage; + protected String buildRequestProcessorImage; @CommandLine.Option(names = "--java-version", required = true) - String javaVersion; + protected String javaVersion; @CommandLine.Option(names = "--build-tool-version", required = true) - String buildToolVersion; + protected String buildToolVersion; + + protected ToolType type; protected enum ToolType { ANT, @@ -52,8 +54,6 @@ public String toString() { } } - protected ToolType type; - /** * This section creates two files within a .jbs subdirectory. The Containerfile is used * by Konflux to initiate a build and the run-build.sh contains generic setup diff --git a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/notification/NotificationTest.java b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/notification/NotificationTest.java index 90723f93f..0d213c818 100644 --- a/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/notification/NotificationTest.java +++ b/java-components/build-request-processor/src/test/java/com/redhat/hacbs/container/notification/NotificationTest.java @@ -56,9 +56,6 @@ public void testNotify() throws IOException, URISyntaxException { .uri(new URI(wireMockServer.baseUrl() + "/internal/completed")) .build(); - System.err.println("### wiremock uri: " + wireMockServer.baseUrl()); - // {"method":"PUT","uri":"http://localhost:8081/internal/completed","headers":[{"name":"Content-Type","value":"application/json"}],"attachment":null} - NotifyCommand notifyCommand = new NotifyCommand(); notifyCommand.status = "Succeeded"; notifyCommand.buildId = "1234"; diff --git a/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go b/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go index cb21075a7..48dcffaa8 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go @@ -48,6 +48,7 @@ type SystemConfigList struct { const ( KonfluxGitDefinition = "https://raw.githubusercontent.com/konflux-ci/build-definitions/refs/heads/main/task/git-clone/0.1/git-clone.yaml" KonfluxPreBuildDefinitions = "https://raw.githubusercontent.com/redhat-appstudio/jvm-build-service/main/deploy/tasks/pre-build.yaml" + KonfluxPreBuildGitDefinitions = "https://raw.githubusercontent.com/redhat-appstudio/jvm-build-service/main/deploy/tasks/pre-build-jbs.yaml" KonfluxBuildDefinitions = "https://raw.githubusercontent.com/redhat-appstudio/jvm-build-service/main/deploy/tasks/buildah-oci-ta.yaml" KonfluxMavenDeployDefinitions = "https://raw.githubusercontent.com/redhat-appstudio/jvm-build-service/main/deploy/tasks/maven-deployment.yaml" ) From ab9366a3b6fbad1dbcf2df6009c0abfd9cbb52ff Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Thu, 19 Dec 2024 15:00:48 +0000 Subject: [PATCH 3/4] Split prebuild source committing for JBS --- deploy/tasks/pre-build-git.yaml | 82 +++++++++++ deploy/tasks/pre-build-jbs.yaml | 133 ------------------ deploy/tasks/pre-build.yaml | 19 ++- .../preprocessor/AbstractPreprocessor.java | 4 +- .../v1alpha1/systemconfig_types.go | 4 +- .../dependencybuild/buildrecipeyaml.go | 112 +++++++++++---- 6 files changed, 176 insertions(+), 178 deletions(-) create mode 100644 deploy/tasks/pre-build-git.yaml delete mode 100644 deploy/tasks/pre-build-jbs.yaml diff --git a/deploy/tasks/pre-build-git.yaml b/deploy/tasks/pre-build-git.yaml new file mode 100644 index 000000000..ee38332c9 --- /dev/null +++ b/deploy/tasks/pre-build-git.yaml @@ -0,0 +1,82 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: pre-build-git + annotations: + tekton.dev/pipelines.minVersion: 0.12.1 + tekton.dev/tags: image-build, konflux + labels: + app.kubernetes.io/version: "0.1" + build.appstudio.redhat.com/build_type: docker +spec: + description: |- + Sets up pre-build for pushing the source. + params: + - name: NAME + description: Name of the pipeline run (i.e. unique dependency build name) + type: string + - name: GIT_IDENTITY + description: Git username. If empty, deploy-pre-build-source step will be skipped. + type: string + default: "" + - name: GIT_URL + description: String to determine whether we're using gitlab or github + type: string + default: "github" + - name: GIT_SSL_VERIFICATION + description: Whether to disable ssl verification + type: string + default: "false" + - name: GIT_REUSE_REPOSITORY + description: Whether to reuse existing git repository or create new one + type: string + default: "false" + - name: SCM_URL + description: Reference to the git repository + type: string + - name: SCM_HASH + description: Git hash + type: string + - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE + description: Name of the processor image. Useful to override for development. + type: string + default: "quay.io/ncross/hacbs-jvm-build-request-processor:latest" + # default: "quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/jvm-build-service/build-request-processor:latest" + results: + - name: GIT_ARCHIVE + description: Git archive information + workspaces: + - description: The git repo will be cloned onto the volume backing this Workspace. + name: source + mountPath: /var/workdir + steps: + - name: create-pre-build-source + image: $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) + securityContext: + runAsUser: 0 + computeResources: + limits: + cpu: 300m + memory: 512Mi + requests: + cpu: 10m + memory: 512Mi + env: + - name: GIT_DEPLOY_TOKEN + valueFrom: + secretKeyRef: + optional: true + name: jvm-build-git-repo-secrets + key: gitdeploytoken + args: + - deploy-pre-build-source + - --source-path=$(workspaces.source.path)/source + - --task-run-name=$(context.taskRun.name) + - --scm-uri=$(params.SCM_URL) + - --scm-commit=$(params.SCM_HASH) + - --image-id=$(params.NAME) + - --git-identity=$(params.GIT_IDENTITY) + - --git-url=$(params.GIT_URL) + - --git-disable-ssl-verification=$(params.GIT_SSL_VERIFICATION) + - --git-reuse-repository=$(params.GIT_REUSE_REPOSITORY) diff --git a/deploy/tasks/pre-build-jbs.yaml b/deploy/tasks/pre-build-jbs.yaml deleted file mode 100644 index 611d3d9bf..000000000 --- a/deploy/tasks/pre-build-jbs.yaml +++ /dev/null @@ -1,133 +0,0 @@ ---- -apiVersion: tekton.dev/v1 -kind: Task -metadata: - name: pre-build - annotations: - tekton.dev/pipelines.minVersion: 0.12.1 - tekton.dev/tags: image-build, konflux - labels: - app.kubernetes.io/version: "0.1" - build.appstudio.redhat.com/build_type: docker -spec: - description: |- - Sets up pre-build running the preprocessor, pushing the source and creating the OCI image. - params: - - name: IMAGE_URL - description: URL of the OCI image to use. - type: string - - name: NAME - description: Name of the pipeline run (i.e. unique dependency build name) - type: string - - name: GIT_IDENTITY - description: Git username. If empty, deploy-pre-build-source step will be skipped. - type: string - default: "" - - name: GIT_URL - description: String to determine whether we're using gitlab or github - type: string - default: "github" - - name: GIT_SSL_VERIFICATION - description: Whether to disable ssl verification - type: string - default: "false" - - name: GIT_REUSE_REPOSITORY - description: Whether to reuse existing git repository or create new one - type: string - default: "false" - - name: SCM_URL - description: Reference to the git repository - type: string - - name: SCM_HASH - description: Git hash - type: string - - name: RECIPE_IMAGE - description: The image from the build recipe to use - - name: BUILD_TOOL - description: The build tool to use (ant, gradle, maven, sbt). - - name: BUILD_TOOL_VERSION - description: The build tool version to use (e.g. 3.9.5) - - name: JAVA_VERSION - description: Java version to use (7, 8, 9, 11, 17, 21, 22, 23) - - name: BUILD_PLUGINS - description: Optional comma separated list of build plugins that should be disabled. - default: "" - - name: BUILD_SCRIPT - description: The build script to embed with the Containerfile - - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE - description: Name of the processor image. Useful to override for development. - type: string - default: "quay.io/ncross/hacbs-jvm-build-request-processor:latest" - # default: "quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/jvm-build-service/build-request-processor:latest" - results: - - name: PRE_BUILD_IMAGE_DIGEST - description: Digest of the image just built - - name: GIT_ARCHIVE - description: Git archive information - workspaces: - - description: The git repo will be cloned onto the volume backing this Workspace. - name: source - mountPath: /var/workdir - steps: - - name: preprocessor - image: $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) - securityContext: - runAsUser: 0 - computeResources: - limits: - cpu: 300m - memory: 512Mi - requests: - cpu: 10m - memory: 512Mi - script: | - # Complete temporary hack for backwards compatibility with JBS until tasks are ported to build-definitions - if [[ $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) == *"konflux-tooling"* ]]; then - /opt/jboss/container/java/run/run-java.sh prepare --type=$(params.BUILD_TOOL) --java-version=$(params.JAVA_VERSION) --build-tool-version=$(params.BUILD_TOOL_VERSION) --recipe-image=$(params.RECIPE_IMAGE) --request-processor-image=$(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) $(workspaces.source.path)/source - else - /opt/jboss/container/java/run/run-java.sh $(params.BUILD_TOOL)-prepare --java-version=$(params.JAVA_VERSION) --build-tool-version=$(params.BUILD_TOOL_VERSION) --recipe-image=$(params.RECIPE_IMAGE) --request-processor-image=$(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) --disabled-plugins=$(params.BUILD_PLUGINS) $(workspaces.source.path)/source - fi - env: - - name: BUILD_SCRIPT - value: $(params.BUILD_SCRIPT) - # TODO: Look at making this optional until we know whether we need to store source - - name: create-pre-build-source - image: $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) - securityContext: - runAsUser: 0 - computeResources: - limits: - cpu: 300m - memory: 512Mi - requests: - cpu: 10m - memory: 512Mi - env: - - name: GIT_DEPLOY_TOKEN - valueFrom: - secretKeyRef: - optional: true - name: jvm-build-git-repo-secrets - key: gitdeploytoken - args: - - deploy-pre-build-source - - --source-path=$(workspaces.source.path)/source - - --task-run-name=$(context.taskRun.name) - - --scm-uri=$(params.SCM_URL) - - --scm-commit=$(params.SCM_HASH) - - --image-id=$(params.NAME) - - --git-identity=$(params.GIT_IDENTITY) - - --git-url=$(params.GIT_URL) - - --git-disable-ssl-verification=$(params.GIT_SSL_VERIFICATION) - - --git-reuse-repository=$(params.GIT_REUSE_REPOSITORY) - - name: create-pre-build-image - image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:52f1391e6f1c472fd10bb838f64fae2ed3320c636f536014978a5ddbdfc6b3af - script: | - set -x - echo "IMAGE is $(params.IMAGE_URL)" - cat $HOME/.docker/config.json || true - echo "Creating pre-build-image archive" - create-archive --store $(params.IMAGE_URL) $(results.PRE_BUILD_IMAGE_DIGEST.path)=$(workspaces.source.path)/source - env: - - name: IMAGE_URL - value: $(params.IMAGE_URL) diff --git a/deploy/tasks/pre-build.yaml b/deploy/tasks/pre-build.yaml index 42b74c8d5..8f537ed90 100644 --- a/deploy/tasks/pre-build.yaml +++ b/deploy/tasks/pre-build.yaml @@ -11,27 +11,24 @@ metadata: build.appstudio.redhat.com/build_type: docker spec: description: |- - Sets up pre-build running the preprocessor, pushing the source and creating the OCI image. + Sets up pre-build running the preprocessor creating the OCI image. params: - - name: IMAGE_URL - description: URL of the OCI image to use. - type: string - - name: NAME - description: Name of the pipeline run (i.e. unique dependency build name) - type: string - - name: RECIPE_IMAGE - description: The image from the build recipe to use - name: BUILD_TOOL description: The build tool to use (ant, gradle, maven, sbt). - name: BUILD_TOOL_VERSION description: The build tool version to use (e.g. 3.9.5) - - name: JAVA_VERSION - description: Java version to use (7, 8, 9, 11, 17, 21, 22, 23) - name: BUILD_SCRIPT description: The build script to embed with the Containerfile + - name: JAVA_VERSION + description: Java version to use (7, 8, 9, 11, 17, 21, 22, 23) + - name: IMAGE_URL + description: URL of the OCI image to use. + type: string - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE description: Name of the processor image. Useful to override for development. type: string + - name: RECIPE_IMAGE + description: The image from the build recipe to use results: - name: PRE_BUILD_IMAGE_DIGEST description: Digest of the image just built diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/build/preprocessor/AbstractPreprocessor.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/build/preprocessor/AbstractPreprocessor.java index 8ae3e870a..5068fd7b5 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/build/preprocessor/AbstractPreprocessor.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/build/preprocessor/AbstractPreprocessor.java @@ -16,7 +16,8 @@ * We keep all the options the same between maven, gradle, sbt and ant for now to keep the pipeline setup simpler. * Some of these may be ignored by different processors */ -public abstract class AbstractPreprocessor implements Runnable { +@CommandLine.Command(name = "prepare") +public class AbstractPreprocessor implements Runnable { /** * Equivalent to $(workspaces.source.path)/source @@ -40,6 +41,7 @@ public abstract class AbstractPreprocessor implements Runnable { @CommandLine.Option(names = "--build-tool-version", required = true) protected String buildToolVersion; + @CommandLine.Option(names = "--type") protected ToolType type; protected enum ToolType { diff --git a/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go b/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go index 48dcffaa8..7eeed6fc4 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go @@ -47,8 +47,8 @@ type SystemConfigList struct { const ( KonfluxGitDefinition = "https://raw.githubusercontent.com/konflux-ci/build-definitions/refs/heads/main/task/git-clone/0.1/git-clone.yaml" - KonfluxPreBuildDefinitions = "https://raw.githubusercontent.com/redhat-appstudio/jvm-build-service/main/deploy/tasks/pre-build.yaml" - KonfluxPreBuildGitDefinitions = "https://raw.githubusercontent.com/redhat-appstudio/jvm-build-service/main/deploy/tasks/pre-build-jbs.yaml" + KonfluxPreBuildDefinitions = "https://raw.githubusercontent.com/rnc/jvm-build-service/BR1/deploy/tasks/pre-build.yaml" + KonfluxPreBuildGitDefinitions = "https://raw.githubusercontent.com/rnc/jvm-build-service/BR1/deploy/tasks/pre-build-git.yaml" KonfluxBuildDefinitions = "https://raw.githubusercontent.com/redhat-appstudio/jvm-build-service/main/deploy/tasks/buildah-oci-ta.yaml" KonfluxMavenDeployDefinitions = "https://raw.githubusercontent.com/redhat-appstudio/jvm-build-service/main/deploy/tasks/maven-deployment.yaml" ) diff --git a/pkg/reconciler/dependencybuild/buildrecipeyaml.go b/pkg/reconciler/dependencybuild/buildrecipeyaml.go index 002a55a84..b17d8fdce 100644 --- a/pkg/reconciler/dependencybuild/buildrecipeyaml.go +++ b/pkg/reconciler/dependencybuild/buildrecipeyaml.go @@ -25,11 +25,12 @@ const ( WorkspaceSource = "source" WorkspaceTls = "tls" - GitTaskName = "git-clone" - PreBuildTaskName = "pre-build" - BuildTaskName = "build" - PostBuildTaskName = "post-build" - DeployTaskName = "deploy" + GitTaskName = "git-clone" + PreBuildTaskName = "pre-build" + PreBuildGitTaskName = "pre-build-git" + BuildTaskName = "build" + PostBuildTaskName = "post-build" + DeployTaskName = "deploy" ) //go:embed scripts/maven-build.sh @@ -272,7 +273,7 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi preBuildImageRequired := preBuildImage == "" if preBuildImageRequired { preBuildImage = "$(tasks." + PreBuildTaskName + ".results." + PipelineResultPreBuildImageDigest + ")" - runAfter = []string{PreBuildTaskName} + runAfter = []string{PreBuildGitTaskName} } runAfterBuild = append(runAfter, BuildTaskName) @@ -307,6 +308,19 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi }, }, } + preBuildGitResolver := tektonpipeline.ResolverRef{ + // We can use either a http or git resolver. Using http as avoids cloning an entire repository. + Resolver: "http", + Params: []tektonpipeline.Param{ + { + Name: "url", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: v1alpha1.KonfluxPreBuildGitDefinitions, + }, + }, + }, + } pipelineGitTask := []tektonpipeline.PipelineTask{{ Name: GitTaskName, TaskRef: &tektonpipeline.TaskRef{ @@ -366,87 +380,118 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi }, }, { - Name: "GIT_IDENTITY", + Name: "RECIPE_IMAGE", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: jbsConfig.Spec.GitSourceArchive.Identity, + StringVal: recipe.Image, }, }, { - Name: "GIT_URL", + Name: "BUILD_TOOL", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: jbsConfig.Spec.GitSourceArchive.URL, + StringVal: tool, }, }, { - Name: "GIT_SSL_VERIFICATION", + Name: "BUILD_TOOL_VERSION", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: strconv.FormatBool(jbsConfig.Spec.GitSourceArchive.DisableSSLVerification), + StringVal: recipe.ToolVersion, }, }, { - Name: "GIT_REUSE_REPOSITORY", + Name: "JAVA_VERSION", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: strconv.FormatBool(db.Annotations[artifactbuild.DependencyScmAnnotation] == "true"), + StringVal: recipe.JavaVersion, }, }, { - Name: "SCM_URL", + Name: "BUILD_SCRIPT", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: db.Spec.ScmInfo.SCMURL, + StringVal: konfluxScript, }, }, + // This will break various parts of JBS as the PNC tooling does NOT support disabling plugins on + // a per build basis. + // + //{ + // Name: "BUILD_PLUGINS", + // Value: tektonpipeline.ParamValue{ + // Type: tektonpipeline.ParamTypeString, + // StringVal: strings.Join(recipe.DisabledPlugins, ","), + // }, + //}, { - Name: "SCM_HASH", + Name: "JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: db.Spec.ScmInfo.CommitHash, + StringVal: "quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/konflux-tooling:latest", + }, + }, + }, + }} + fmt.Printf("### RECIPE : DISABLEDPLUGINS %#v \n", recipe.DisabledPlugins) + pipelinePreBuildGitTask := []tektonpipeline.PipelineTask{{ + Name: PreBuildGitTaskName, + RunAfter: []string{PreBuildTaskName}, + TaskRef: &tektonpipeline.TaskRef{ + // Can't specify name and resolver as they clash. + ResolverRef: preBuildGitResolver, + }, + Workspaces: []tektonpipeline.WorkspacePipelineTaskBinding{ + {Name: WorkspaceSource, Workspace: WorkspaceSource}, + }, + Params: []tektonpipeline.Param{ + { + Name: "NAME", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: imageId, }, }, { - Name: "RECIPE_IMAGE", + Name: "GIT_IDENTITY", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: recipe.Image, + StringVal: jbsConfig.Spec.GitSourceArchive.Identity, }, }, { - Name: "BUILD_TOOL", + Name: "GIT_URL", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: tool, + StringVal: jbsConfig.Spec.GitSourceArchive.URL, }, }, { - Name: "BUILD_TOOL_VERSION", + Name: "GIT_SSL_VERIFICATION", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: recipe.ToolVersion, + StringVal: strconv.FormatBool(jbsConfig.Spec.GitSourceArchive.DisableSSLVerification), }, }, { - Name: "JAVA_VERSION", + Name: "GIT_REUSE_REPOSITORY", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: recipe.JavaVersion, + StringVal: strconv.FormatBool(db.Annotations[artifactbuild.DependencyScmAnnotation] == "true"), }, }, { - Name: "BUILD_SCRIPT", + Name: "SCM_URL", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: konfluxScript, + StringVal: db.Spec.ScmInfo.SCMURL, }, }, { - Name: "BUILD_PLUGINS", + Name: "SCM_HASH", Value: tektonpipeline.ParamValue{ Type: tektonpipeline.ParamTypeString, - StringVal: strings.Join(recipe.DisabledPlugins, ","), + StringVal: db.Spec.ScmInfo.CommitHash, }, }, { @@ -460,9 +505,10 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi }} ps.Tasks = append(pipelineGitTask, ps.Tasks...) ps.Tasks = append(pipelinePreBuildTask, ps.Tasks...) + ps.Tasks = append(pipelinePreBuildGitTask, ps.Tasks...) ps.Results = []tektonpipeline.PipelineResult{ {Name: PipelineResultPreBuildImageDigest, Value: tektonpipeline.ResultValue{Type: tektonpipeline.ParamTypeString, StringVal: "$(tasks." + PreBuildTaskName + ".results." + PipelineResultPreBuildImageDigest + ")"}}, - {Name: PipelineResultGitArchive, Value: tektonpipeline.ResultValue{Type: tektonpipeline.ParamTypeString, StringVal: "$(tasks." + PreBuildTaskName + ".results." + PipelineResultGitArchive + ")"}}, + {Name: PipelineResultGitArchive, Value: tektonpipeline.ResultValue{Type: tektonpipeline.ParamTypeString, StringVal: "$(tasks." + PreBuildGitTaskName + ".results." + PipelineResultGitArchive + ")"}}, } } @@ -755,6 +801,10 @@ use-archive oci:$URL@$AARCHIVE=%s`, orasOptions, registryArgsWithDefaults(jbsCon ps.Tasks[index].Params = append(ps.Tasks[index].Params, tektonpipeline.Param{ Name: i.Name, Value: value}) + index += 1 + ps.Tasks[index].Params = append(ps.Tasks[index].Params, tektonpipeline.Param{ + Name: i.Name, + Value: value}) } index += 1 ps.Tasks[index].Params = append(ps.Tasks[index].Params, tektonpipeline.Param{ From 12221ec127575a6aa78da6c1ba9b17cea66768c2 Mon Sep 17 00:00:00 2001 From: Nick Cross Date: Tue, 24 Dec 2024 10:37:49 +0000 Subject: [PATCH 4/4] Remove workspace from prebuildgit --- deploy/tasks/pre-build-git.yaml | 26 ++++++++++++++----- .../hacbs/container/deploy/git/Git.java | 1 + .../v1alpha1/systemconfig_types.go | 2 +- .../dependencybuild/buildrecipeyaml.go | 11 +++++--- .../dependencybuild/dependencybuild.go | 2 +- 5 files changed, 29 insertions(+), 13 deletions(-) diff --git a/deploy/tasks/pre-build-git.yaml b/deploy/tasks/pre-build-git.yaml index ee38332c9..5348c9ed3 100644 --- a/deploy/tasks/pre-build-git.yaml +++ b/deploy/tasks/pre-build-git.yaml @@ -13,6 +13,9 @@ spec: description: |- Sets up pre-build for pushing the source. params: + - name: PRE_BUILD_IMAGE_DIGEST + description: Digest to use + type: string - name: NAME description: Name of the pipeline run (i.e. unique dependency build name) type: string @@ -41,16 +44,25 @@ spec: - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE description: Name of the processor image. Useful to override for development. type: string - default: "quay.io/ncross/hacbs-jvm-build-request-processor:latest" - # default: "quay.io/redhat-user-workloads/konflux-jbs-pnc-tenant/jvm-build-service/build-request-processor:latest" results: - name: GIT_ARCHIVE description: Git archive information - workspaces: - - description: The git repo will be cloned onto the volume backing this Workspace. - name: source - mountPath: /var/workdir + volumes: + - name: workdir + emptyDir: {} + stepTemplate: + volumeMounts: + - mountPath: /var/workdir + name: workdir steps: + - name: restore-trusted-artifact + image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:52f1391e6f1c472fd10bb838f64fae2ed3320c636f536014978a5ddbdfc6b3af + script: | + echo "Restoring source to workspace" + use-archive $PRE_BUILD_IMAGE_DIGEST=/var/workdir/ + env: + - name: PRE_BUILD_IMAGE_DIGEST + value: $(params.PRE_BUILD_IMAGE_DIGEST) - name: create-pre-build-source image: $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) securityContext: @@ -71,7 +83,7 @@ spec: key: gitdeploytoken args: - deploy-pre-build-source - - --source-path=$(workspaces.source.path)/source + - --source-path=/var/workdir - --task-run-name=$(context.taskRun.name) - --scm-uri=$(params.SCM_URL) - --scm-commit=$(params.SCM_HASH) diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/deploy/git/Git.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/deploy/git/Git.java index d733410d5..b13742c82 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/deploy/git/Git.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/deploy/git/Git.java @@ -146,6 +146,7 @@ protected GitStatus pushRepository(Path path, String httpTransportUrl, String co } var branchName = tagName + "-jbs-branch"; var createBranch = jGit.branchList().call().stream().map(Ref::getName).noneMatch(("refs/heads/" + branchName)::equals); + Log.infof("Will create branch %s for tag %s and branch name %s", createBranch, tagName, branchName); var ref = jGit.checkout().setStartPoint(tagName).setName(branchName).setCreateBranch(createBranch).call(); StoredConfig jConfig = jRepo.getConfig(); Log.infof("Updating current origin of %s to %s", jConfig.getString("remote", "origin", "url"), diff --git a/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go b/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go index 7eeed6fc4..07780e0cd 100644 --- a/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go +++ b/pkg/apis/jvmbuildservice/v1alpha1/systemconfig_types.go @@ -50,5 +50,5 @@ const ( KonfluxPreBuildDefinitions = "https://raw.githubusercontent.com/rnc/jvm-build-service/BR1/deploy/tasks/pre-build.yaml" KonfluxPreBuildGitDefinitions = "https://raw.githubusercontent.com/rnc/jvm-build-service/BR1/deploy/tasks/pre-build-git.yaml" KonfluxBuildDefinitions = "https://raw.githubusercontent.com/redhat-appstudio/jvm-build-service/main/deploy/tasks/buildah-oci-ta.yaml" - KonfluxMavenDeployDefinitions = "https://raw.githubusercontent.com/redhat-appstudio/jvm-build-service/main/deploy/tasks/maven-deployment.yaml" + KonfluxMavenDeployDefinitions = "https://raw.githubusercontent.com/rnc/jvm-build-service/BR1/deploy/tasks/maven-deployment.yaml" ) diff --git a/pkg/reconciler/dependencybuild/buildrecipeyaml.go b/pkg/reconciler/dependencybuild/buildrecipeyaml.go index b17d8fdce..0c56a3b3b 100644 --- a/pkg/reconciler/dependencybuild/buildrecipeyaml.go +++ b/pkg/reconciler/dependencybuild/buildrecipeyaml.go @@ -433,7 +433,6 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi }, }, }} - fmt.Printf("### RECIPE : DISABLEDPLUGINS %#v \n", recipe.DisabledPlugins) pipelinePreBuildGitTask := []tektonpipeline.PipelineTask{{ Name: PreBuildGitTaskName, RunAfter: []string{PreBuildTaskName}, @@ -441,9 +440,6 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi // Can't specify name and resolver as they clash. ResolverRef: preBuildGitResolver, }, - Workspaces: []tektonpipeline.WorkspacePipelineTaskBinding{ - {Name: WorkspaceSource, Workspace: WorkspaceSource}, - }, Params: []tektonpipeline.Param{ { Name: "NAME", @@ -452,6 +448,13 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi StringVal: imageId, }, }, + { + Name: "PRE_BUILD_IMAGE_DIGEST", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: "$(tasks." + PreBuildTaskName + ".results." + PipelineResultPreBuildImageDigest + ")", + }, + }, { Name: "GIT_IDENTITY", Value: tektonpipeline.ParamValue{ diff --git a/pkg/reconciler/dependencybuild/dependencybuild.go b/pkg/reconciler/dependencybuild/dependencybuild.go index ce97587aa..ab5925d54 100644 --- a/pkg/reconciler/dependencybuild/dependencybuild.go +++ b/pkg/reconciler/dependencybuild/dependencybuild.go @@ -668,7 +668,7 @@ func (r *ReconcileDependencyBuild) handleStateBuilding(ctx context.Context, db * log.Info(fmt.Sprintf("handleStateBuilding: pipelinerun %s:%s already exists, not retrying", pr.Namespace, pr.Name)) return reconcile.Result{}, nil } - r.eventRecorder.Eventf(db, v1.EventTypeWarning, "PipelineRunCreationFailed", "The DependencyBuild %s/%s failed to create its build pipeline run", db.Namespace, db.Name) + r.eventRecorder.Eventf(db, v1.EventTypeWarning, "PipelineRunCreationFailed", "The DependencyBuild %s/%s failed to create its build pipeline run with %#v", db.Namespace, db.Name, err) return reconcile.Result{}, err } return reconcile.Result{}, r.client.Status().Update(ctx, db)