diff --git a/deploy/tasks/maven-deployment.yaml b/deploy/tasks/maven-deployment.yaml index b69bb92d51..a9c1c306dc 100644 --- a/deploy/tasks/maven-deployment.yaml +++ b/deploy/tasks/maven-deployment.yaml @@ -37,14 +37,10 @@ spec: type: string default: "quay.io/redhat-appstudio/hacbs-jvm-build-request-processor:dev" volumes: - - name: shared - emptyDir: {} - name: workdir emptyDir: {} stepTemplate: volumeMounts: - - mountPath: /shared - name: shared - mountPath: /var/workdir name: workdir steps: @@ -70,10 +66,10 @@ spec: runAsUser: 0 computeResources: limits: - cpu: "1" - memory: 2Gi + cpu: 300m + memory: 512Mi requests: - cpu: 50m + cpu: 10m memory: 512Mi env: - name: MVN_REPO diff --git a/deploy/tasks/pre-build.yaml b/deploy/tasks/pre-build.yaml new file mode 100644 index 0000000000..41c5dd3d3e --- /dev/null +++ b/deploy/tasks/pre-build.yaml @@ -0,0 +1,145 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: pre-build + annotations: + tekton.dev/pipelines.minVersion: 0.12.1 + tekton.dev/tags: image-build, konflux + labels: + app.kubernetes.io/version: "0.1" + build.appstudio.redhat.com/build_type: docker +spec: + description: |- + Sets up pre-build running the preprocessor, pushing the source and creating the OCI image. + params: + - name: IMAGE_URL + description: URL of the OCI image to use. + type: string + - name: NAME + description: Name of the pipeline run (i.e. unique dependency build name) + type: string + - name: GIT_SCRIPT + description: Git clone commands + type: string + - name: GIT_IDENTITY + description: Git username + type: string + - name: GIT_URL + description: URL to determine whether we're using gitlab or github + type: string + - name: GIT_DEPLOY_TOKEN + description: Name of jvm-build-git-repo-secrets secret containing git password/token to use. + type: string + - name: GIT_SSL_VERIFICATION + description: Whether to disable ssl verification + type: string + default: "false" + - name: GIT_REUSE_REPOSITORY + description: Whether to reuse existing git repository or create new one + type: string + - name: SCM_URL + description: Reference to the git repository + type: string + - name: SCM_HASH + description: Git hash + type: string + - name: RECIPE_IMAGE + description: The image from the build recipe to use + - name: BUILD_SCRIPT + description: The build script to embed with the Containerfile + - name: PREPROCESSOR_ARGS + description: The arguments for the build preprocessor + - name: ORAS_OPTIONS + type: string + description: Optional environment variable string for build-trusted-artifacts + default: "" + - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE + description: Name of the processor image. Useful to override for development. + type: string + default: "quay.io/redhat-appstudio/hacbs-jvm-build-request-processor:dev" + results: + - name: PRE_BUILD_IMAGE_DIGEST + description: Digest of the image just built + - name: GIT_ARCHIVE + description: Git archive information + workspaces: + - description: The git repo will be cloned onto the volume backing this Workspace. + name: source + mountPath: /var/workdir + - name: tls + steps: + # Should we use our own git clone task? Or embed (somehow) Konflux's version? + - name: git-clone + image: $(params.RECIPE_IMAGE) + computeResources: + limits: + cpu: 300m + memory: 512Mi + requests: + cpu: 10m + memory: 512Mi + securityContext: + runAsUser: 0 + env: + - name: GIT_TOKEN + valueFrom: + secretKeyRef: + name: jvm-build-git-secrets + key: .git-credentials + script: | + $(params.GIT_SCRIPT) + - name: preprocessor + image: $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) + securityContext: + runAsUser: 0 + computeResources: + limits: + cpu: 300m + memory: 512Mi + requests: + cpu: 10m + memory: 512Mi + script: | + $(params.BUILD_SCRIPT) + /opt/jboss/container/java/run/run-java.sh $(params.PREPROCESSOR_ARGS) + - name: create-pre-build-source + image: $(params.JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE) + securityContext: + runAsUser: 0 + computeResources: + limits: + cpu: 300m + memory: 512Mi + requests: + cpu: 10m + memory: 512Mi + env: + - name: GIT_DEPLOY_TOKEN + valueFrom: + secretKeyRef: + name: $(params.GIT_DEPLOY_TOKEN) + key: gitdeploytoken + args: + - deploy-pre-build-source + - --source-path=$(workspaces.source.path)/source + - --task-run-name=$(context.taskRun.name) + - --scm-uri=$(params.SCM_URL) + - --scm-commit=$(params.SCM_HASH) + - --image-id=$(params.NAME) + - --git-identity=$(params.GIT_IDENTITY) + - --git-url=$(params.GIT_URL) + - --git-disable-ssl-verification=$(params.GIT_SSL_VERIFICATION) + - --git-reuse-repository=$(params.GIT_REUSE_REPOSITORY) + - name: create-pre-build-image + image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:d6f57d97d19008437680190908fe5444cda380f9c77d0e9efde7153720412e05 + script: | + echo "Creating pre-build-image archive" + export ORAS_OPTIONS="$ORAS_OPTIONS --image-spec=v1.0 --artifact-type application/vnd.oci.image.config.v1+json" + create-archive --store $(params.IMAGE_URL) $(results.PRE_BUILD_IMAGE_DIGEST.path)=$(workspaces.source.path)/source + env: + - name: ORAS_OPTIONS + value: $(params.ORAS_OPTIONS) + - name: IMAGE_URL + value: $(params.IMAGE_URL) + diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/deploy/git/GitHub.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/deploy/git/GitHub.java index 464bf589af..3ecfeb1ef4 100644 --- a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/deploy/git/GitHub.java +++ b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/deploy/git/GitHub.java @@ -32,11 +32,11 @@ enum Type { public GitHub(String endpoint, String identity, String token, boolean ssl) throws IOException { if (isNotEmpty(token)) { - github = new GitHubBuilder().withEndpoint(endpoint == null ? GITHUB_URL : endpoint) + github = new GitHubBuilder().withEndpoint(isNotEmpty(endpoint) ? endpoint : GITHUB_URL) .withOAuthToken(token) .build(); } else { - github = new GitHubBuilder().withEndpoint(endpoint == null ? GITHUB_URL : endpoint) + github = new GitHubBuilder().withEndpoint(isNotEmpty(endpoint) ? endpoint : GITHUB_URL) .build(); } owner = identity; diff --git a/pkg/reconciler/dependencybuild/buildrecipeyaml.go b/pkg/reconciler/dependencybuild/buildrecipeyaml.go index d0fe5f7706..290e515a6c 100644 --- a/pkg/reconciler/dependencybuild/buildrecipeyaml.go +++ b/pkg/reconciler/dependencybuild/buildrecipeyaml.go @@ -155,6 +155,7 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi verifyBuiltArtifactsArgs := verifyParameters(jbsConfig, recipe) preBuildImageArgs, copyArtifactsArgs, deployArgs, konfluxArgs := pipelineBuildCommands(imageId, db, jbsConfig, buildId) + fmt.Printf("### Was using preBuildImageArgs %#v and konfluxArgs %#v ", preBuildImageArgs, konfluxArgs) gitScript := gitScript(db, recipe) install := additionalPackages(recipe) orasOptions := "" @@ -204,7 +205,6 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi additionalMemory = systemConfig.Spec.MaxAdditionalMemory } var buildToolSection string - trueBool := true if tool == "maven" { buildToolSection = mavenSettings + "\n" + mavenBuild } else if tool == "gradle" { @@ -335,9 +335,6 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi pipelineParams := []tektonpipeline.ParamSpec{ {Name: PipelineBuildId, Type: tektonpipeline.ParamTypeString}, - {Name: PipelineParamScmUrl, Type: tektonpipeline.ParamTypeString}, - {Name: PipelineParamScmTag, Type: tektonpipeline.ParamTypeString}, - {Name: PipelineParamScmHash, Type: tektonpipeline.ParamTypeString}, {Name: PipelineParamChainsGitUrl, Type: tektonpipeline.ParamTypeString}, {Name: PipelineParamChainsGitCommit, Type: tektonpipeline.ParamTypeString}, {Name: PipelineParamGoals, Type: tektonpipeline.ParamTypeArray}, @@ -362,87 +359,145 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi runAfterBuild = append(runAfter, BuildTaskName) ps := &tektonpipeline.PipelineSpec{ - Workspaces: []tektonpipeline.PipelineWorkspaceDeclaration{{Name: WorkspaceBuildSettings}, {Name: WorkspaceSource}, {Name: WorkspaceTls}}, + Workspaces: []tektonpipeline.PipelineWorkspaceDeclaration{{Name: WorkspaceSource}, {Name: WorkspaceTls}}, } if preBuildImageRequired { - buildSetup := tektonpipeline.TaskSpec{ - Workspaces: []tektonpipeline.WorkspaceDeclaration{{Name: WorkspaceBuildSettings}, {Name: WorkspaceSource, MountPath: WorkspaceMount}, {Name: WorkspaceTls}}, - Params: pipelineParams, - Results: []tektonpipeline.TaskResult{ - {Name: PipelineResultPreBuildImageDigest, Type: tektonpipeline.ResultsTypeString}, - {Name: PipelineResultGitArchive, Type: tektonpipeline.ResultsTypeString}, + resolver := tektonpipeline.ResolverRef{ + // We can use either a http or git resolver. Using http as avoids cloning an entire repository. + Resolver: "http", + Params: []tektonpipeline.Param{ + { + Name: "url", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: "https://raw.githubusercontent.com/rnc/jvm-build-service/KJB11/deploy/tasks/pre-build.yaml", + }, + }, }, - Steps: []tektonpipeline.Step{ + } + pipelineTask := []tektonpipeline.PipelineTask{{ + Name: PreBuildTaskName, + TaskRef: &tektonpipeline.TaskRef{ + // Can't specify name and resolver as they clash. + ResolverRef: resolver, + }, + Workspaces: []tektonpipeline.WorkspacePipelineTaskBinding{ + {Name: WorkspaceSource, Workspace: WorkspaceSource}, + {Name: WorkspaceTls, Workspace: WorkspaceTls}, + }, + Params: []tektonpipeline.Param{ { - Name: "git-clone-and-settings", - Image: recipe.Image, - SecurityContext: &v1.SecurityContext{RunAsUser: &zero}, - ComputeResources: v1.ResourceRequirements{ - Requests: v1.ResourceList{"memory": limits.defaultRequestMemory, "cpu": limits.defaultRequestCPU}, - Limits: v1.ResourceList{"memory": limits.defaultRequestMemory, "cpu": limits.defaultLimitCPU}, + Name: "IMAGE_URL", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: registryArgsWithDefaults(jbsConfig, imageId+"-pre-build-image"), }, - Script: gitScript, - Env: []v1.EnvVar{ - {Name: PipelineParamCacheUrl, Value: "$(params." + PipelineParamCacheUrl + ")"}, - {Name: "GIT_TOKEN", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: v1alpha1.GitSecretName}, Key: v1alpha1.GitSecretTokenKey, Optional: &trueBool}}}, + }, + { + Name: "NAME", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: imageId, }, }, { - Name: "preprocessor", - Image: buildRequestProcessorImage, - ImagePullPolicy: pullPolicy, - SecurityContext: &v1.SecurityContext{RunAsUser: &zero}, - Env: []v1.EnvVar{ - {Name: PipelineParamCacheUrl, Value: "$(params." + PipelineParamCacheUrl + ")"}, + Name: "GIT_SCRIPT", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: gitScript, }, - ComputeResources: v1.ResourceRequirements{ - Requests: v1.ResourceList{"memory": limits.defaultRequestMemory, "cpu": limits.defaultRequestCPU}, - Limits: v1.ResourceList{"memory": limits.defaultRequestMemory, "cpu": limits.defaultLimitCPU}, + }, + { + Name: "GIT_IDENTITY", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: jbsConfig.Spec.GitSourceArchive.Identity, }, - Script: artifactbuild.InstallKeystoreIntoBuildRequestProcessor(preprocessorArgs), }, { - Name: "create-pre-build-source", - Image: buildRequestProcessorImage, - ImagePullPolicy: pullPolicy, - SecurityContext: &v1.SecurityContext{RunAsUser: &zero}, - Env: secretVariables, - ComputeResources: v1.ResourceRequirements{ - Requests: v1.ResourceList{"memory": limits.defaultBuildRequestMemory, "cpu": limits.defaultRequestCPU}, - Limits: v1.ResourceList{"memory": limits.defaultBuildRequestMemory, "cpu": limits.defaultLimitCPU}, + Name: "GIT_URL", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: jbsConfig.Spec.GitSourceArchive.URL, }, - Script: createKonfluxScripts(kf, konfluxScript) + "\n" + artifactbuild.InstallKeystoreIntoBuildRequestProcessor(konfluxArgs), }, { - Name: "create-pre-build-image", - Image: strings.TrimSpace(strings.Split(buildTrustedArtifacts, "FROM")[1]), - ImagePullPolicy: v1.PullIfNotPresent, - SecurityContext: &v1.SecurityContext{RunAsUser: &zero}, - Env: secretVariables, - ComputeResources: v1.ResourceRequirements{ - Requests: v1.ResourceList{"memory": limits.defaultBuildRequestMemory, "cpu": limits.defaultRequestCPU}, - Limits: v1.ResourceList{"memory": limits.defaultBuildRequestMemory, "cpu": limits.defaultLimitCPU}, + Name: "GIT_DEPLOY_TOKEN", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: v1alpha1.GitRepoSecretName, + }, + }, + { + Name: "GIT_SSL_VERIFICATION", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: strconv.FormatBool(jbsConfig.Spec.GitSourceArchive.DisableSSLVerification), + }, + }, + { + Name: "GIT_REUSE_REPOSITORY", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: strconv.FormatBool(db.Annotations[artifactbuild.DependencyScmAnnotation] == "true"), + }, + }, + { + Name: "SCM_URL", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: db.Spec.ScmInfo.SCMURL, + }, + }, + { + Name: "SCM_HASH", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: db.Spec.ScmInfo.CommitHash, + }, + }, + { + Name: "RECIPE_IMAGE", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: recipe.Image, + }, + }, + { + Name: "BUILD_SCRIPT", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: createKonfluxScripts(kf, konfluxScript), + }, + }, + { + Name: "PREPROCESSOR_ARGS", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: strings.Join(preprocessorArgs, " "), + }, + }, + { + Name: "ORAS_OPTIONS", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: orasOptions, + }, + }, + { + Name: "JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE", + Value: tektonpipeline.ParamValue{ + Type: tektonpipeline.ParamTypeString, + StringVal: buildRequestProcessorImage, }, - Script: preBuildImageArgs, }, - }, - } - pipelineTask := []tektonpipeline.PipelineTask{{ - Name: PreBuildTaskName, - TaskSpec: &tektonpipeline.EmbeddedTask{ - TaskSpec: buildSetup, - }, - Params: []tektonpipeline.Param{}, Workspaces: []tektonpipeline.WorkspacePipelineTaskBinding{ - {Name: WorkspaceBuildSettings, Workspace: WorkspaceBuildSettings}, - {Name: WorkspaceSource, Workspace: WorkspaceSource}, - {Name: WorkspaceTls, Workspace: WorkspaceTls}, }, }} ps.Tasks = append(pipelineTask, ps.Tasks...) - - for _, i := range buildSetup.Results { - ps.Results = append(ps.Results, tektonpipeline.PipelineResult{Name: i.Name, Description: i.Description, Value: tektonpipeline.ResultValue{Type: tektonpipeline.ParamTypeString, StringVal: "$(tasks." + PreBuildTaskName + ".results." + i.Name + ")"}}) + ps.Results = []tektonpipeline.PipelineResult{ + {Name: PipelineResultPreBuildImageDigest, Value: tektonpipeline.ResultValue{Type: tektonpipeline.ParamTypeString, StringVal: "$(tasks." + PreBuildTaskName + ".results." + PipelineResultPreBuildImageDigest + ")"}}, + {Name: PipelineResultGitArchive, Value: tektonpipeline.ResultValue{Type: tektonpipeline.ParamTypeString, StringVal: "$(tasks." + PreBuildTaskName + ".results." + PipelineResultGitArchive + ")"}}, } } @@ -522,7 +577,7 @@ func createPipelineSpec(log logr.Logger, tool string, commitTime int64, jbsConfi ps.Results = append(ps.Results, tektonpipeline.PipelineResult{Name: PipelineResultImageDigest, Value: tektonpipeline.ResultValue{Type: tektonpipeline.ParamTypeString, StringVal: "$(tasks." + BuildTaskName + ".results." + PipelineResultImageDigest + ")"}}) postBuildTask := tektonpipeline.TaskSpec{ - Workspaces: []tektonpipeline.WorkspaceDeclaration{{Name: WorkspaceBuildSettings}, {Name: WorkspaceSource, MountPath: WorkspaceMount}, {Name: WorkspaceTls}}, + Workspaces: []tektonpipeline.WorkspaceDeclaration{{Name: WorkspaceSource, MountPath: WorkspaceMount}, {Name: WorkspaceTls}}, Params: append(pipelineParams, tektonpipeline.ParamSpec{Name: PipelineResultPreBuildImageDigest, Type: tektonpipeline.ParamTypeString}), Results: []tektonpipeline.TaskResult{ {Name: PipelineResultContaminants}, @@ -570,7 +625,6 @@ use-archive oci:$URL@$AARCHIVE=$(workspaces.source.path)/artifacts`, orasOptions Timeout: &v12.Duration{Duration: time.Hour * v1alpha1.DefaultTimeout}, Params: []tektonpipeline.Param{{Name: PipelineResultPreBuildImageDigest, Value: tektonpipeline.ParamValue{Type: tektonpipeline.ParamTypeString, StringVal: preBuildImage}}}, Workspaces: []tektonpipeline.WorkspacePipelineTaskBinding{ - {Name: WorkspaceBuildSettings, Workspace: WorkspaceBuildSettings}, {Name: WorkspaceSource, Workspace: WorkspaceSource}, {Name: WorkspaceTls, Workspace: WorkspaceTls}, }, diff --git a/pkg/reconciler/dependencybuild/dependencybuild.go b/pkg/reconciler/dependencybuild/dependencybuild.go index 19b1eeb585..3d7a744ac0 100644 --- a/pkg/reconciler/dependencybuild/dependencybuild.go +++ b/pkg/reconciler/dependencybuild/dependencybuild.go @@ -41,9 +41,9 @@ const ( //TODO eventually we'll need to decide if we want to make this tuneable contextTimeout = 300 * time.Second PipelineBuildId = "DEPENDENCY_BUILD" - PipelineParamScmUrl = "URL" + PipelineParamScmUrl = "SCM_URL" PipelineParamScmTag = "TAG" - PipelineParamScmHash = "HASH" + PipelineParamScmHash = "SCM_HASH" PipelineParamPath = "CONTEXT_DIR" PipelineParamChainsGitUrl = "CHAINS-GIT_URL" PipelineParamChainsGitCommit = "CHAINS-GIT_COMMIT" @@ -1403,9 +1403,7 @@ func (r *ReconcileDependencyBuild) handleStateDeploying(ctx context.Context, db } pr.Spec.Params = paramValues - pr.Spec.Workspaces = []tektonpipeline.WorkspaceBinding{ - {Name: WorkspaceSource, EmptyDir: &v1.EmptyDirVolumeSource{}}, - } + pr.Spec.Workspaces = []tektonpipeline.WorkspaceBinding{} if !jbsConfig.Spec.CacheSettings.DisableTLS { pr.Spec.Workspaces = append(pr.Spec.Workspaces, tektonpipeline.WorkspaceBinding{Name: "tls", ConfigMap: &v1.ConfigMapVolumeSource{LocalObjectReference: v1.LocalObjectReference{Name: v1alpha1.TlsConfigMapName}}})