Skip to content

Commit

Permalink
Reference pre-build images by hash
Browse files Browse the repository at this point in the history
  • Loading branch information
@stuartwdouglas
stuartwdouglas committed Nov 9, 2023
1 parent fd24877 commit 9497a0f
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 37 deletions.
18 changes: 17 additions & 1 deletion ...n/java/com/redhat/hacbs/container/analyser/deploy/DeployHermeticPreBuildImageCommand.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
package com.redhat.hacbs.container.analyser.deploy;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
import java.util.function.BiConsumer;

import org.eclipse.microprofile.config.inject.ConfigProperty;

Expand All @@ -17,6 +20,9 @@ public class DeployHermeticPreBuildImageCommand implements Runnable {
String builderImage;
@CommandLine.Option(names = "--image-name")
String imageName;

@CommandLine.Option(names = "--image-hash", required = true)
Path imageHash;
@CommandLine.Option(names = "--build-artifact-path", required = true)
Path buildArtifactsPath;

Expand Down Expand Up @@ -45,7 +51,17 @@ public void run() {
insecure,
prependTag, "");
try {
deployer.deployHermeticPreBuildImage(builderImage, buildArtifactsPath, repositoryPath, imageSourcePath, imageName);
deployer.deployHermeticPreBuildImage(builderImage, buildArtifactsPath, repositoryPath, imageSourcePath, imageName,
new BiConsumer<String, String>() {
@Override
public void accept(String name, String hash) {
try {
Files.writeString(imageHash, name.substring(0, name.lastIndexOf(":")) + "@sha256:" + hash);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
});
} catch (Exception e) {
throw new RuntimeException(e);
}
Expand Down
17 changes: 16 additions & 1 deletion .../src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployPreBuildImageCommand.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
package com.redhat.hacbs.container.analyser.deploy;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
import java.util.function.BiConsumer;

import org.eclipse.microprofile.config.inject.ConfigProperty;

Expand All @@ -18,6 +21,8 @@ public class DeployPreBuildImageCommand implements Runnable {
String builderImage;
@CommandLine.Option(names = "--source-path", required = true)
Path sourcePath;
@CommandLine.Option(names = "--image-hash", required = true)
Path imageHash;

@CommandLine.Option(names = "--image-source-path", required = true)
String imageSourcePath;
Expand Down Expand Up @@ -45,7 +50,17 @@ public void run() {
insecure,
prependTag, "");
try {
deployer.deployPreBuildImage(builderImage, sourcePath, imageSourcePath, imageName);
deployer.deployPreBuildImage(builderImage, sourcePath, imageSourcePath, imageName,
new BiConsumer<String, String>() {
@Override
public void accept(String name, String hash) {
try {
Files.writeString(imageHash, name.substring(0, name.lastIndexOf(":")) + "@sha256:" + hash);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
});
} catch (Exception e) {
throw new RuntimeException(e);
}
Expand Down
18 changes: 14 additions & 4 deletions ...m/redhat/hacbs/container/analyser/deploy/containerregistry/ContainerRegistryDeployer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ public void tagArchive(List<String> gavNames) throws Exception {
containerBuilder.containerize(containerizer);
}

public void deployPreBuildImage(String baseImage, Path sourcePath, String imageSourcePath, String tag)
public void deployPreBuildImage(String baseImage, Path sourcePath, String imageSourcePath, String tag,
BiConsumer<String, String> imageNameHashCallback)
throws Exception {
Log.debugf("Using Container registry %s:%d/%s/%s", host, port, owner, repository);
String imageName = createImageName(tag);
Expand Down Expand Up @@ -167,12 +168,17 @@ public FilePermissions get(Path sourcePath, AbsoluteUnixPath destinationPath) {

containerBuilder.addFileEntriesLayer(layerConfigurationBuilder.build());
Log.debugf("Image %s created", imageName);
containerBuilder.containerize(containerizer);
var result = containerBuilder.containerize(containerizer);

if (imageNameHashCallback != null) {
imageNameHashCallback.accept(imageName, result.getDigest().getHash());
}
}
}

public void deployHermeticPreBuildImage(String baseImage, Path buildArtifactsPath, Path repositoryPath,
String imageSourcePath, String tag) throws Exception {
String imageSourcePath, String tag,
BiConsumer<String, String> imageNameHashCallback) throws Exception {
Log.debugf("Using Container registry %s:%d/%s/%s", host, port, owner, repository);
String imageName = createImageName(tag);
RegistryImage registryImage = RegistryImage.named(imageName);
Expand Down Expand Up @@ -207,7 +213,11 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
});
containerBuilder.addFileEntriesLayer(layerConfigurationBuilder.build());
Log.debugf("Image %s created", imageName);
containerBuilder.containerize(containerizer);
var result = containerBuilder.containerize(containerizer);

if (imageNameHashCallback != null) {
imageNameHashCallback.accept(imageName, result.getDigest().getHash());
}
}

private void createImages(DeployData imageData, Path sourcePath, Path logsPath,
Expand Down
59 changes: 28 additions & 31 deletions pkg/reconciler/dependencybuild/buildrecipeyaml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@ import (
)

const (
WorkspaceBuildSettings = "build-settings"
WorkspaceSource = "source"
WorkspaceTls = "tls"
OriginalContentPath = "/original-content"
MavenArtifactsPath = "/maven-artifacts"
WorkspaceBuildSettings = "build-settings"
WorkspaceSource = "source"
WorkspaceTls = "tls"
OriginalContentPath = "/original-content"
MavenArtifactsPath = "/maven-artifacts"
PreBuildImageDigest = "PRE_BUILD_IMAGE_DIGEST"
HermeticPreBuildImageDigest = "HERMETIC_PRE_BUILD_IMAGE_DIGEST"
)

//go:embed scripts/maven-build.sh
Expand Down Expand Up @@ -60,7 +62,7 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC
hermeticBuildRequired := jbsConfig.Spec.HermeticBuilds == v1alpha12.HermeticBuildTypeRequired
verifyBuiltArtifactsArgs := verifyParameters(jbsConfig, recipe)

preBuildImageName, hermeticPreBuildImageName, preBuildImageArgs, deployArgs, hermeticDeployArgs, tagArgs, createHermeticImageArgs := imageRegistryCommands(imageId, recipe, db, jbsConfig, hermeticBuildRequired)
preBuildImageArgs, deployArgs, hermeticDeployArgs, tagArgs, createHermeticImageArgs := imageRegistryCommands(imageId, recipe, db, jbsConfig, hermeticBuildRequired)
gitArgs := gitArgs(db, recipe)
install := additionalPackages(recipe)

Expand Down Expand Up @@ -178,6 +180,7 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC
buildSetup := pipelinev1beta1.TaskSpec{
Workspaces: []pipelinev1beta1.WorkspaceDeclaration{{Name: WorkspaceBuildSettings}, {Name: WorkspaceSource}, {Name: WorkspaceTls}},
Params: pipelineParams,
Results: []pipelinev1beta1.TaskResult{{Name: PreBuildImageDigest, Type: pipelinev1beta1.ResultsTypeString}},
Steps: []pipelinev1beta1.Step{
{
Name: "git-clone-and-settings",
Expand Down Expand Up @@ -232,7 +235,7 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC
}
buildTask := pipelinev1beta1.TaskSpec{
Workspaces: []pipelinev1beta1.WorkspaceDeclaration{{Name: WorkspaceBuildSettings}, {Name: WorkspaceSource}, {Name: WorkspaceTls}},
Params: pipelineParams,
Params: append(pipelineParams, pipelinev1beta1.ParamSpec{Name: PreBuildImageDigest, Type: pipelinev1beta1.ParamTypeString}),
Results: []pipelinev1beta1.TaskResult{
{Name: artifactbuild.PipelineResultContaminants},
{Name: artifactbuild.PipelineResultDeployedResources},
Expand All @@ -244,7 +247,7 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC
Steps: []pipelinev1beta1.Step{
{
Name: "build",
Image: preBuildImageName,
Image: "$(params." + PreBuildImageDigest + ")",
ImagePullPolicy: v1.PullAlways,
WorkingDir: "$(workspaces." + WorkspaceSource + ".path)/workspace",
SecurityContext: &v1.SecurityContext{RunAsUser: &zero},
Expand Down Expand Up @@ -276,10 +279,13 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC
},
},
}
if hermeticBuildRequired {
buildTask.Results = append(buildTask.Results, pipelinev1beta1.TaskResult{Name: HermeticPreBuildImageDigest})
}

hermeticBuildTask := pipelinev1beta1.TaskSpec{
Workspaces: []pipelinev1beta1.WorkspaceDeclaration{{Name: WorkspaceBuildSettings}, {Name: WorkspaceSource}, {Name: WorkspaceTls}},
Params: pipelineParams,
Params: append(pipelineParams, pipelinev1beta1.ParamSpec{Name: HermeticPreBuildImageDigest, Type: pipelinev1beta1.ParamTypeString}),
Results: []pipelinev1beta1.TaskResult{
{Name: artifactbuild.PipelineResultContaminants},
{Name: artifactbuild.PipelineResultDeployedResources},
Expand All @@ -291,7 +297,7 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC
Steps: []pipelinev1beta1.Step{
{
Name: "hermetic-build",
Image: hermeticPreBuildImageName,
Image: "$(params." + HermeticPreBuildImageDigest + ")",
ImagePullPolicy: v1.PullAlways,
WorkingDir: "$(workspaces." + WorkspaceSource + ".path)",
SecurityContext: &v1.SecurityContext{RunAsUser: &zero, Capabilities: &v1.Capabilities{Add: []v1.Capability{"SETFCAP"}}},
Expand Down Expand Up @@ -355,7 +361,8 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC
TaskSpec: hermeticBuildTask,
},

Params: []pipelinev1beta1.Param{}, Workspaces: []pipelinev1beta1.WorkspacePipelineTaskBinding{
Params: []pipelinev1beta1.Param{{Name: HermeticPreBuildImageDigest, Value: pipelinev1beta1.ParamValue{Type: pipelinev1beta1.ParamTypeString, StringVal: "$(tasks." + artifactbuild.BuildTaskName + ".results." + HermeticPreBuildImageDigest + ")"}}},
Workspaces: []pipelinev1beta1.WorkspacePipelineTaskBinding{
{Name: WorkspaceBuildSettings, Workspace: WorkspaceBuildSettings},
{Name: WorkspaceSource, Workspace: WorkspaceSource},
{Name: WorkspaceTls, Workspace: WorkspaceTls},
Expand Down Expand Up @@ -393,7 +400,8 @@ func createPipelineSpec(tool string, commitTime int64, jbsConfig *v1alpha12.JBSC
TaskSpec: &pipelinev1beta1.EmbeddedTask{
TaskSpec: buildTask,
},
Params: []pipelinev1beta1.Param{}, Workspaces: []pipelinev1beta1.WorkspacePipelineTaskBinding{
Params: []pipelinev1beta1.Param{{Name: PreBuildImageDigest, Value: pipelinev1beta1.ParamValue{Type: pipelinev1beta1.ParamTypeString, StringVal: "$(tasks." + artifactbuild.PreBuildTaskName + ".results." + PreBuildImageDigest + ")"}}},
Workspaces: []pipelinev1beta1.WorkspacePipelineTaskBinding{
{Name: WorkspaceBuildSettings, Workspace: WorkspaceBuildSettings},
{Name: WorkspaceSource, Workspace: WorkspaceSource},
{Name: WorkspaceTls, Workspace: WorkspaceTls},
Expand Down Expand Up @@ -579,15 +587,16 @@ func gitArgs(db *v1alpha12.DependencyBuild, recipe *v1alpha12.BuildRecipe) strin
return gitArgs
}

func imageRegistryCommands(imageId string, recipe *v1alpha12.BuildRecipe, db *v1alpha12.DependencyBuild, jbsConfig *v1alpha12.JBSConfig, hermeticBuild bool) (string, string, []string, []string, []string, []string, []string) {
preBuildImageName := ""
func imageRegistryCommands(imageId string, recipe *v1alpha12.BuildRecipe, db *v1alpha12.DependencyBuild, jbsConfig *v1alpha12.JBSConfig, hermeticBuild bool) ([]string, []string, []string, []string, []string) {

preBuildImageTag := imageId + "-pre-build-image"
preBuildImageArgs := []string{
"deploy-pre-build-image",
"--builder-image=" + recipe.Image,
"--source-path=$(workspaces.source.path)",
"--image-source-path=" + OriginalContentPath,
"--image-name=" + preBuildImageTag,
"--image-hash=$(results." + PreBuildImageDigest + ".path)",
}
hermeticPreBuildImageTag := imageId + "-hermetic-pre-build-image"
hermeticImageId := imageId + "-hermetic"
Expand All @@ -603,9 +612,8 @@ func imageRegistryCommands(imageId string, recipe *v1alpha12.BuildRecipe, db *v1
"--scm-commit=" + db.Spec.ScmInfo.CommitHash,
}
hermeticDeployArgs := append([]string{}, deployArgs...)

deployArgs = append(deployArgs, "--image-id="+imageId)
hermeticDeployArgs = append(hermeticDeployArgs, "--image-id="+hermeticImageId)
deployArgs = append(deployArgs, "--image-id="+imageId)

var imageIdToTag string
if hermeticBuild {
Expand All @@ -616,40 +624,28 @@ func imageRegistryCommands(imageId string, recipe *v1alpha12.BuildRecipe, db *v1

tagArgs := []string{
"tag-container",
"--image-id=" + imageIdToTag,
"--image-id=" + imageIdToTag, //TODO: hash
}
imageRegistry := jbsConfig.ImageRegistry()
registryArgs := make([]string, 0)
if imageRegistry.Host != "" {
registryArgs = append(registryArgs, "--registry-host="+imageRegistry.Host)
preBuildImageName += imageRegistry.Host
} else {
preBuildImageName += "quay.io"
}
if imageRegistry.Port != "" && imageRegistry.Port != "443" {
registryArgs = append(registryArgs, "--registry-port="+imageRegistry.Port)
preBuildImageName += ":" + imageRegistry.Port
}
if imageRegistry.Owner != "" {
registryArgs = append(registryArgs, "--registry-owner="+imageRegistry.Owner)
preBuildImageName += "/" + imageRegistry.Owner
}
if imageRegistry.Repository != "" {
registryArgs = append(registryArgs, "--registry-repository="+imageRegistry.Repository)
preBuildImageName += "/" + imageRegistry.Repository
} else {
preBuildImageName += "/artifact-deployments"
}

hermeticPreBuildImageName := preBuildImageName + ":" + hermeticPreBuildImageTag
preBuildImageName += ":" + preBuildImageTag
if imageRegistry.Insecure {
registryArgs = append(registryArgs, "--registry-insecure")
}
if imageRegistry.PrependTag != "" {
registryArgs = append(registryArgs, "--registry-prepend-tag="+imageRegistry.PrependTag)
preBuildImageName = prependTagToImage(preBuildImageName, imageRegistry.PrependTag)
hermeticPreBuildImageName = prependTagToImage(hermeticPreBuildImageName, imageRegistry.PrependTag)
}
deployArgs = append(deployArgs, registryArgs...)
hermeticDeployArgs = append(hermeticDeployArgs, registryArgs...)
Expand All @@ -674,15 +670,16 @@ func imageRegistryCommands(imageId string, recipe *v1alpha12.BuildRecipe, db *v1

hermeticPreBuildImageArgs := []string{
"deploy-hermetic-pre-build-image",
"--source-image=" + preBuildImageName,
"--source-image=$(params." + PreBuildImageDigest + ")",
"--image-name=" + hermeticPreBuildImageTag,
"--build-artifact-path=$(workspaces.source.path)/artifacts",
"--image-source-path=" + MavenArtifactsPath,
"--repository-path=$(workspaces.source.path)/build-info/",
"--image-hash=$(results." + HermeticPreBuildImageDigest + ".path)",
}
hermeticPreBuildImageArgs = append(hermeticPreBuildImageArgs, registryArgs...)

return preBuildImageName, hermeticPreBuildImageName, preBuildImageArgs, deployArgs, hermeticDeployArgs, tagArgs, hermeticPreBuildImageArgs
return preBuildImageArgs, deployArgs, hermeticDeployArgs, tagArgs, hermeticPreBuildImageArgs
}

// This is equivalent to ContainerRegistryDeployer.java::createImageName with the same image tag length restriction.
Expand Down

0 comments on commit 9497a0f

Please sign in to comment.