Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add GitArchive information to BuildAttempt #944

Merged
merged 2 commits into from
Nov 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions deploy/crds/base/jvmbuildservice.io_dependencybuilds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,16 @@ spec:
items:
type: string
type: array
gitArchive:
description: The git archive source information
properties:
sha:
type: string
tag:
type: string
url:
type: string
type: object
hermeticBuildImage:
description: The hermetic build image produced by the
build
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,17 +157,19 @@

public void run() {
try {
Set<String> gavs = new HashSet<>();
Map<String, Set<String>> contaminatedPaths = new HashMap<>();
Map<String, Contaminates> contaminatedGavs = new HashMap<>();
Git.GitStatus archivedSourceTags = new Git.GitStatus();

// Save the source first regardless of deployment checks
if (isNotEmpty(gitIdentity) && gitToken.isPresent()) {
Log.infof("Git credentials are identity '%s' and URL '%s'", gitIdentity, gitURL);
var git = Git.builder(gitURL, gitIdentity, gitToken.get(), gitDisableSSLVerification);
git.create(scmUri);
git.add(sourcePath, commit, imageId);
archivedSourceTags = git.add(sourcePath, commit, imageId);

Check warning on line 170 in java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java

View check run for this annotation

Codecov / codecov/patch

java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java#L170

Added line #L170 was not covered by tests
}

Set<String> gavs = new HashSet<>();
Map<String, Set<String>> contaminatedPaths = new HashMap<>();
Map<String, Contaminates> contaminatedGavs = new HashMap<>();
// Represents directories that should not be deployed i.e. if a single artifact (barring test jars) is
// contaminated then none of the artifacts will be deployed.
Set<Path> toRemove = new HashSet<>();
Expand Down Expand Up @@ -337,13 +339,15 @@
newContaminates.add(i.getValue());
}
String serialisedContaminants = ResultsUpdater.MAPPER.writeValueAsString(newContaminates);
String serialisedGitArchive = ResultsUpdater.MAPPER.writeValueAsString(archivedSourceTags);

Check warning on line 342 in java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java

View check run for this annotation

Codecov / codecov/patch

java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/DeployCommand.java#L342

Added line #L342 was not covered by tests
Log.infof("Updating results %s with contaminants %s and deployed resources %s",
taskRun, serialisedContaminants, gavs);
resultsUpdater.updateResults(taskRun, Map.of(
"CONTAMINANTS", serialisedContaminants,
"DEPLOYED_RESOURCES", String.join(",", gavs),
"IMAGE_URL", imageName == null ? "" : imageName,
"IMAGE_DIGEST", imageDigest == null ? "" : "sha256:" + imageDigest));
"IMAGE_DIGEST", imageDigest == null ? "" : "sha256:" + imageDigest,
"GIT_ARCHIVE", serialisedGitArchive));
}
} catch (Exception e) {
Log.error("Deployment failed", e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.PushResult;
Expand All @@ -24,7 +25,7 @@
public abstract void create(String name)
throws IOException, URISyntaxException;

public abstract void add(Path path, String commit, String imageId)
public abstract GitStatus add(Path path, String commit, String imageId)
throws IOException;

/**
Expand Down Expand Up @@ -62,7 +63,7 @@
}
}

protected void pushRepository(Path path, String httpTransportUrl, String commit, String imageId) {
protected GitStatus pushRepository(Path path, String httpTransportUrl, String commit, String imageId) {
try (var jGit = org.eclipse.jgit.api.Git.init().setDirectory(path.toFile()).call()) {
// Find the tag name associated with the commit. Then append the unique imageId. This is from the Go code
// and is a hash of abr.Status.SCMInfo.SCMURL + abr.Status.SCMInfo.Tag + abr.Status.SCMInfo.Path
Expand All @@ -87,6 +88,7 @@
Ref tagRefStable = jGit.tag().setAnnotated(true).setName(tagName + "-" + imageId).setForceUpdate(true).call();
Ref tagRefUnique = jGit.tag().setAnnotated(true).setName(tagName + "-" + UUID.randomUUID()).setForceUpdate(true)
.call();

Iterable<PushResult> results = jGit.push().setForce(true).setRemote("origin")
.add(jRepo.getBranch()) // Push the default branch else GitHub doesn't show the code.
.add(tagRefStable)
Expand All @@ -95,14 +97,18 @@

for (PushResult result : results) {
result.getRemoteUpdates().forEach(r -> {
if (!r.getStatus().equals(RemoteRefUpdate.Status.OK)) {
if (!r.getStatus().equals(RemoteRefUpdate.Status.OK)
&& !r.getStatus().equals(RemoteRefUpdate.Status.UP_TO_DATE)) {
Log.errorf("Push failure " + r);
throw new RuntimeException("Failed to push updates due to " + r.getMessage());
}
});
Log.debugf("Pushed " + result.getMessages() + " " + result.getURI() + " updates: "
+ result.getRemoteUpdates());
}

return new GitStatus(httpTransportUrl, Repository.shortenRefName(tagRefUnique.getName()),
jRepo.getRefDatabase().peel(tagRefUnique).getPeeledObjectId().getName());
} catch (GitAPIException | IOException e) {
throw new RuntimeException(e);
}
Expand All @@ -125,4 +131,24 @@
}

abstract String groupSplit();

public static class GitStatus {
public String url;
public String tag;
public String sha;

public GitStatus() {
}

public GitStatus(String url, String tag, String sha) {
this.url = url;
this.tag = tag;
this.sha = sha;
}

@Override
public String toString() {
return "GitStatus{url='" + url + "', tag='" + tag + "', sha='" + sha + "'}";

Check warning on line 151 in java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java

View check run for this annotation

Codecov / codecov/patch

java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/Git.java#L151

Added line #L151 was not covered by tests
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
if (type == Type.USER) {
repository = github.getUser(owner).getRepository(name);
if (repository == null) {
Log.infof("Creating repository with name %s", name);

Check warning on line 65 in java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java

View check run for this annotation

Codecov / codecov/patch

java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java#L65

Added line #L65 was not covered by tests
repository = github.createRepository(name)
.wiki(false)
.defaultBranch("main")
Expand All @@ -73,6 +74,7 @@
} else {
repository = github.getOrganization(owner).getRepository(name);
if (repository == null) {
Log.infof("Creating repository with name %s", name);

Check warning on line 77 in java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java

View check run for this annotation

Codecov / codecov/patch

java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java#L77

Added line #L77 was not covered by tests
repository = github.getOrganization(owner).createRepository(name)
.wiki(false)
.defaultBranch("main")
Expand All @@ -85,11 +87,11 @@
}

@Override
public void add(Path path, String commit, String imageId) {
public GitStatus add(Path path, String commit, String imageId) {
if (repository == null) {
throw new RuntimeException("Call create first");
}
pushRepository(path, repository.getHttpTransportUrl(), commit, imageId);
return pushRepository(path, repository.getHttpTransportUrl(), commit, imageId);

Check warning on line 94 in java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java

View check run for this annotation

Codecov / codecov/patch

java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitHub.java#L94

Added line #L94 was not covered by tests
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
public void create(String scmUri)
throws URISyntaxException {
String name = parseScmURI(scmUri);
Log.infof("Creating repository with name %s", name);
try {
project = gitLabApi.getProjectApi().getUserProjectsStream(owner, new ProjectFilter().withSearch(name))
.filter(p -> p.getName().equals(name))
Expand All @@ -49,6 +48,7 @@
Log.warnf("Repository %s already exists", name);
} else {
// Can't set public visibility after creation for some reason with this API.
Log.infof("Creating repository with name %s", name);

Check warning on line 51 in java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java

View check run for this annotation

Codecov / codecov/patch

java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java#L51

Added line #L51 was not covered by tests
project = gitLabApi.getProjectApi().createProject(name,
null,
null,
Expand All @@ -66,12 +66,12 @@
}

@Override
public void add(Path path, String commit, String imageId)
public GitStatus add(Path path, String commit, String imageId)
throws IOException {
if (project == null) {
throw new RuntimeException("Call create first");
}
pushRepository(path, project.getHttpUrlToRepo(), commit, imageId);
return pushRepository(path, project.getHttpUrlToRepo(), commit, imageId);

Check warning on line 74 in java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java

View check run for this annotation

Codecov / codecov/patch

java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/analyser/deploy/git/GitLab.java#L74

Added line #L74 was not covered by tests
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,10 @@
import java.util.stream.Collectors;

import org.apache.commons.lang3.builder.DiffResult;
import org.jboss.logging.Logger;

import com.redhat.hacbs.container.verifier.asm.AsmDiffable;

public class DiffUtils {
private static final Logger Log = Logger.getLogger(DiffUtils.class);

public record DiffResults(Set<String> shared, Set<String> added, Set<String> deleted,
Map<String, DiffResult<?>> diffResults,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@
import org.eclipse.aether.util.repository.AuthenticationBuilder;
import org.eclipse.aether.util.repository.DefaultAuthenticationSelector;
import org.eclipse.aether.util.repository.DefaultMirrorSelector;
import org.jboss.logging.Logger;

import io.quarkus.logging.Log;

public class MavenUtils {
private static final Logger Log = Logger.getLogger(MavenUtils.class);

public static Settings newSettings(Path globalSettingsFile, Path settingsFile) {
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,20 +36,19 @@
import org.eclipse.aether.repository.LocalRepository;
import org.eclipse.aether.repository.RemoteRepository;
import org.eclipse.aether.repository.RemoteRepository.Builder;
import org.jboss.logging.Logger;

import com.redhat.hacbs.container.results.ResultsUpdater;
import com.redhat.hacbs.container.verifier.asm.JarInfo;

import io.quarkus.bootstrap.resolver.maven.BootstrapMavenContext;
import io.quarkus.bootstrap.resolver.maven.BootstrapMavenException;
import io.quarkus.logging.Log;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;

@Command(name = "verify-built-artifacts")
public class VerifyBuiltArtifactsCommand implements Callable<Integer> {
private static final Logger Log = Logger.getLogger(VerifyBuiltArtifactsCommand.class);

static class LocalOptions {
@Option(required = true, names = { "-of", "--original-file" })
Expand Down Expand Up @@ -139,7 +138,6 @@
session.setReadOnly();
}

Log.debugf("Deploy path: %s", options.mavenOptions.deployPath);
var futureResults = new HashMap<String, Future<List<String>>>();

Files.walkFileTree(options.mavenOptions.deployPath, new SimpleFileVisitor<>() {
Expand Down Expand Up @@ -191,7 +189,7 @@
}
if (taskRunName != null) {
var json = ResultsUpdater.MAPPER.writeValueAsString(verificationResults);
io.quarkus.logging.Log.infof("Writing verification results %s", json);
Log.infof("Writing verification results %s", json);

Check warning on line 192 in java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java

View check run for this annotation

Codecov / codecov/patch

java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/verifier/VerifyBuiltArtifactsCommand.java#L192

Added line #L192 was not covered by tests
resultsUpdater.get().updateResults(taskRunName, Map.of("VERIFICATION_RESULTS", json));
}
return (failed && !reportOnly ? 1 : 0);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,14 @@
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.jboss.logging.Logger;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.tree.ClassNode;

import com.redhat.hacbs.container.verifier.DiffUtils;

public record JarInfo(String name, Map<String, ClassInfo> classes) implements AsmDiffable<JarInfo> {
import io.quarkus.logging.Log;

private static final Logger Log = Logger.getLogger(JarInfo.class);
public record JarInfo(String name, Map<String, ClassInfo> classes) implements AsmDiffable<JarInfo> {

// diffClass excluding name

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.TagOpt;
import org.eclipse.jgit.transport.URIish;
import org.junit.jupiter.api.BeforeEach;
Expand Down Expand Up @@ -54,6 +56,7 @@ public void testPush()
Path initialRepo = Files.createTempDirectory("initial-repo");
Path testRepo = Files.createTempDirectory("test-repo");
String testRepoURI = "file://" + testRepo;
String imageID = "75ecd81c7a2b384151c990975eb1dd10";
try (var testRepository = org.eclipse.jgit.api.Git.init().setDirectory(testRepo.toFile()).call();
var initialRepository = org.eclipse.jgit.api.Git.init().setDirectory(initialRepo.toFile()).call()) {
Path repoRoot = Paths.get(Objects.requireNonNull(getClass().getResource("/")).toURI()).getParent().getParent()
Expand All @@ -74,19 +77,20 @@ public void create(String name) {
}

@Override
public void add(Path path, String commit, String imageId) {
public GitStatus add(Path path, String commit, String imageId) {
return null;
}

@Override
public String groupSplit() {
return null;
}
};
test.pushRepository(
Git.GitStatus tagResults = test.pushRepository(
initialRepo,
testRepoURI,
"c396268fb90335bde5c9272b9a194c3d4302bf24",
"75ecd81c7a2b384151c990975eb1dd10");
imageID);

List<LogRecord> logRecords = LogCollectingTestResource.current().getRecords();

Expand All @@ -98,9 +102,18 @@ public String groupSplit() {
.anyMatch(
r -> LogCollectingTestResource.format(r).matches("Updating current origin of.*to " + testRepoURI)));

assertEquals(2, testRepository.tagList().call().size());
assertTrue(testRepository.tagList().call().stream()
.anyMatch(r -> r.getName().equals("refs/tags/0.1-75ecd81c7a2b384151c990975eb1dd10")));
List<Ref> tags = testRepository.tagList().call();
assertEquals(2, tags.size());
assertTrue(tags.stream().anyMatch(r -> r.getName().equals("refs/tags/0.1-75ecd81c7a2b384151c990975eb1dd10")));

var found = tags.stream().filter(t -> Repository.shortenRefName(t.getName()).matches(tagResults.tag)).findFirst();
assertTrue(found.isPresent());
assertTrue(tagResults.url.contains(testRepoURI));
assertTrue(tagResults.sha.matches(testRepository.getRepository()
.getRefDatabase()
.peel(found.get())
.getPeeledObjectId()
.getName()));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@

import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
import org.jboss.logging.Logger;

import com.redhat.hacbs.recipies.scm.AbstractPomScmLocator;

import io.quarkus.logging.Log;

@ApplicationScoped
public class CachePomScmLocator extends AbstractPomScmLocator {

private static final Logger log = Logger.getLogger(CachePomScmLocator.class);
@Inject
CacheFacade cache;

Expand All @@ -27,7 +27,7 @@
public Optional<Model> getPom(String group, String artifact, String version) {
var response = cache.getArtifactFile("default", group.replace(".", "/"), artifact, version, artifact
+ "-" + version + ".pom", false);
if (!response.isPresent()) {
if (response.isEmpty()) {
return Optional.empty();
}
try {
Expand All @@ -38,7 +38,7 @@
}

} catch (Exception e) {
log.errorf(e, "Failed to get pom for %s:%s:%s", group, artifact, version);
Log.errorf(e, "Failed to get pom for %s:%s:%s", group, artifact, version);

Check warning on line 41 in java-components/cache/src/main/java/com/redhat/hacbs/artifactcache/services/CachePomScmLocator.java

View check run for this annotation

Codecov / codecov/patch

java-components/cache/src/main/java/com/redhat/hacbs/artifactcache/services/CachePomScmLocator.java#L41

Added line #L41 was not covered by tests
return Optional.empty();
} finally {
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,16 @@ spec:
items:
type: string
type: array
gitArchive:
description: The git archive source information
properties:
sha:
type: string
tag:
type: string
url:
type: string
type: object
hermeticBuildImage:
description: The hermetic build image produced by the
build
Expand Down
Loading