Merge pull request #933 from stuartwdouglas/more-improvements

More improvements

java-components/management-console/src/main/java/com/redhat/hacbs/management/dto/BuildDTO.java

@@ -2,11 +2,15 @@
 
 import java.util.List;
 
+import jakarta.persistence.EntityManager;
+
 import org.eclipse.microprofile.openapi.annotations.media.Schema;
 
 import com.redhat.hacbs.management.model.MavenArtifact;
 import com.redhat.hacbs.management.model.StoredDependencyBuild;
 
+import io.quarkus.arc.Arc;
+
 public record BuildDTO(
         @Schema(required = true) long id,
         @Schema(required = true) String name,
@@ -18,8 +22,19 @@ public record BuildDTO(
         boolean contaminated,
         List<String> artifacts,
         BuildAttemptDTO successfulBuild,
-        List<BuildAttemptDTO> buildAttempts) {
+        List<BuildAttemptDTO> buildAttempts,
+
+        boolean inQueue) {
     public static BuildDTO of(StoredDependencyBuild build) {
+        EntityManager entityManager = Arc.container().instance(EntityManager.class).get();
+        var inQueue = false;
+        Long n = (Long) entityManager.createQuery(
+                "select count(*) from StoredArtifactBuild a inner join BuildQueue b on b.mavenArtifact=a.mavenArtifact where a.buildIdentifier=:b")
+                .setParameter("b", build.buildIdentifier).getSingleResult();
+        if (n > 0) {
+            inQueue = true;
+        }
+
         BuildAttemptDTO success = build.buildAttempts.stream().filter(s -> s.successful).findFirst().map(BuildAttemptDTO::of)
                 .orElse(null);
         List<BuildAttemptDTO> others = build.buildAttempts.stream().filter(s -> success == null || s.id != success.id())
@@ -35,7 +50,8 @@ public static BuildDTO of(StoredDependencyBuild build) {
                 build.contaminated,
                 build.producedArtifacts.stream().map(MavenArtifact::gav).toList(),
                 success,
-                others);
+                others,
+                inQueue);
 
     }
 }

java-components/management-console/src/main/java/com/redhat/hacbs/management/dto/DeploymentDTO.java

@@ -31,12 +31,26 @@ public long getUntrustedDependencies() {
             return dependencies.stream()
                     .filter(s -> !(Objects.equals(s.source, "rebuild") || Objects.equals(s.source, "redhat"))).count();
         }
+
+        @Schema(required = true)
+        public long getTrustedDependencies() {
+            return dependencies.stream()
+                    .filter(s -> Objects.equals(s.source, "rebuild") || Objects.equals(s.source, "redhat")).count();
+        }
+
+        @Schema(required = true)
+        public long getAvailableBuilds() {
+            return dependencies.stream()
+                    .filter(s -> !(Objects.equals(s.source, "rebuild") || Objects.equals(s.source, "redhat")))
+                    .filter(s -> s.buildSuccess).count();
+        }
     }
 
     public record Dependency(
             @Schema(required = true) String gav,
-            @Schema(required = true) String source, String buildId,
-            Map<String, String> attributes) implements Comparable<Dependency> {
+            @Schema(required = true) String source, Long build, @Schema(required = true) boolean inQueue,
+            @Schema(required = true) boolean buildSuccess,
+            @Schema(required = true) Map<String, String> attributes) implements Comparable<Dependency> {
 
         @Override
         public int compareTo(Dependency o) {

java-components/management-console/src/main/java/com/redhat/hacbs/management/importer/DependencyBuildImporter.java

@@ -84,6 +84,12 @@ public void doImport(DependencyBuild dependencyBuild) {
         }
         storedBuild.buildAttempts = new ArrayList<>();
 
+        if (s3Bucket != null) {
+            //todo we just assume the logs are present
+            storedBuild.buildDiscoveryUrl = "s3://" + s3Bucket + "/build-logs/" + dependencyBuild.getMetadata().getName() + "/"
+                    + dependencyBuild.getMetadata().getUid()
+                    + "/build-discovery.log";
+        }
         if (dependencyBuild.getStatus().getBuildAttempts() != null) {
             for (var i : dependencyBuild.getStatus().getBuildAttempts()) {
                 BuildAttempt attempt = new BuildAttempt();
@@ -94,6 +100,7 @@ public void doImport(DependencyBuild dependencyBuild) {
                 attempt.gradleVersion = i.getBuildRecipe().getToolVersions().get("gradle");
                 attempt.sbtVersion = i.getBuildRecipe().getToolVersions().get("sbt");
                 attempt.antVersion = i.getBuildRecipe().getToolVersions().get("ant");
+                attempt.buildId = i.getBuildId();
                 attempt.tool = i.getBuildRecipe().getTool();
                 attempt.builderImage = i.getBuildRecipe().getImage();
                 attempt.commandLine(i.getBuildRecipe().getCommandLine());

java-components/management-console/src/main/java/com/redhat/hacbs/management/model/ArtifactIdentifier.java

@@ -21,8 +21,7 @@ public static ArtifactIdentifier findORCreate(String group, String artifact) {
             ret = new ArtifactIdentifier();
             ret.artifact = artifact;
             ret.group = group;
-            ret.persist();
-
+            ret.persistAndFlush();
         }
         return ret;
     }

java-components/management-console/src/main/java/com/redhat/hacbs/management/model/BuildAttempt.java

@@ -23,6 +23,8 @@ public class BuildAttempt extends PanacheEntity {
     public String sbtVersion;
     public String antVersion;
     public String tool;
+
+    public String buildId;
     @Column(length = -1)
     public String builderImage;
     @Column(length = -1)

java-components/management-console/src/main/java/com/redhat/hacbs/management/model/ContainerImage.java

@@ -66,7 +66,7 @@ public static ContainerImage getOrCreate(String image) {
             containerImage.tag = tag;
             containerImage.image = imagePart;
             containerImage.digest = digest;
-            containerImage.persist();
+            containerImage.persistAndFlush();
         }
         return containerImage;
     }

java-components/management-console/src/main/java/com/redhat/hacbs/management/model/StoredDependencyBuild.java

@@ -39,6 +39,7 @@ public class StoredDependencyBuild extends PanacheEntity {
 
     public boolean contaminated;
 
+    public String buildDiscoveryUrl;
     @OneToMany
     public List<ShadingDetails> shadingDetails;
 

java-components/management-console/src/main/java/com/redhat/hacbs/management/resources/BuildAttemptResource.java

@@ -39,7 +39,6 @@ public Response logs(@PathParam("id") int id) {
                     .key(path);
         });
         return Response.ok(stream, MediaType.TEXT_PLAIN_TYPE).build();
-
     }
 
 }

java-components/management-console/src/main/java/com/redhat/hacbs/management/resources/BuildHistoryResource.java

@@ -1,5 +1,7 @@
 package com.redhat.hacbs.management.resources;
 
+import java.io.InputStream;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -11,6 +13,8 @@
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.PathParam;
 import jakarta.ws.rs.QueryParam;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 
 import org.eclipse.microprofile.openapi.annotations.Operation;
 
@@ -20,12 +24,17 @@
 import com.redhat.hacbs.management.model.StoredArtifactBuild;
 import com.redhat.hacbs.management.model.StoredDependencyBuild;
 
+import io.quarkus.logging.Log;
 import io.quarkus.panache.common.Page;
 import io.quarkus.panache.common.Sort;
+import software.amazon.awssdk.services.s3.S3Client;
 
 @Path("/builds/history")
 public class BuildHistoryResource {
 
+    @Inject
+    S3Client s3Client;
+
     @Inject
     EntityManager entityManager;
 
@@ -66,4 +75,22 @@ public BuildDTO get(@PathParam("id") long id) {
         return BuildDTO.of(build);
     }
 
+    @GET
+    @Path("/discover-logs/{id}")
+    public Response logs(@PathParam("id") int id) {
+        StoredDependencyBuild attempt = StoredDependencyBuild.findById(id);
+        if (attempt == null) {
+            throw new NotFoundException();
+        }
+        URI uri = URI.create(attempt.buildDiscoveryUrl);
+
+        InputStream stream = s3Client.getObject(b -> {
+            String path = uri.getPath().substring(1);
+            String bucket = uri.getHost();
+            Log.infof("requesting logs %s from bucket %s", path, bucket);
+            b.bucket(bucket)
+                    .key(path);
+        });
+        return Response.ok(stream, MediaType.TEXT_PLAIN_TYPE).build();
+    }
 }

java-components/management-console/src/main/java/com/redhat/hacbs/management/resources/DeploymentResource.java

@@ -7,11 +7,15 @@
 import java.util.Map;
 
 import jakarta.inject.Inject;
+import jakarta.persistence.EntityManager;
+import jakarta.persistence.NoResultException;
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.Path;
 
 import com.redhat.hacbs.management.dto.DeploymentDTO;
+import com.redhat.hacbs.management.model.BuildAttempt;
 import com.redhat.hacbs.management.model.ContainerImage;
+import com.redhat.hacbs.management.model.StoredDependencyBuild;
 import com.redhat.hacbs.management.watcher.DeploymentWatcher;
 
 @Path("/deployment")
@@ -20,6 +24,9 @@ public class DeploymentResource {
     @Inject
     DeploymentWatcher deploymentWatcher;
 
+    @Inject
+    EntityManager entityManager;
+
     @GET
     public List<DeploymentDTO> getDeployments() {
         List<DeploymentDTO> ret = new ArrayList<>();
@@ -34,6 +41,8 @@ public List<DeploymentDTO> getDeployments() {
                 } else {
                     List<DeploymentDTO.Dependency> depList = new ArrayList<>();
                     info.analysisComplete = existing.analysisComplete;
+
+                    //TODO: this is slow as hell
                     for (var dep : existing.imageDependencies) {
                         Map<String, String> attributes = new HashMap<>();
                         if (dep.attributes != null) {
@@ -42,8 +51,36 @@ public List<DeploymentDTO> getDeployments() {
                                 attributes.put(parts[0], parts[1]);
                             }
                         }
+                        Long buildId = null;
+                        boolean buildSuccess = false;
+                        if (dep.buildId == null) {
+                            try {
+                                StoredDependencyBuild db = (StoredDependencyBuild) entityManager.createQuery(
+                                        "select b from StoredArtifactBuild s inner join StoredDependencyBuild b on b.buildIdentifier=s.buildIdentifier where s.mavenArtifact = :artifact order by b.creationTime desc")
+                                        .setParameter("artifact", dep.mavenArtifact)
+                                        .setMaxResults(1)
+                                        .getSingleResult();
+                                buildId = db.id;
+                                buildSuccess = db.succeeded;
+                            } catch (NoResultException ignore) {
+                            }
+                        } else {
+                            BuildAttempt db = BuildAttempt.find("buildId", dep.buildId).singleResult();
+                            if (db != null) {
+                                buildId = db.dependencyBuild.id;
+                                buildSuccess = db.dependencyBuild.succeeded;
+                            }
+                        }
+                        var inQueue = false;
+                        Long n = (Long) entityManager.createQuery(
+                                "select count(*) from BuildQueue b where b.mavenArtifact=:artifact")
+                                .setParameter("artifact", dep.mavenArtifact)
+                                .getSingleResult();
+                        if (n > 0) {
+                            inQueue = true;
+                        }
                         DeploymentDTO.Dependency d = new DeploymentDTO.Dependency(dep.mavenArtifact.gav(), dep.source,
-                                dep.buildId, attributes);
+                                buildId, inQueue, buildSuccess, attributes);
                         depList.add(d);
                     }
                     Collections.sort(depList);

java-components/management-console/src/main/java/com/redhat/hacbs/management/watcher/JvmImageScanWatcher.java

@@ -10,6 +10,8 @@
 import jakarta.inject.Inject;
 import jakarta.transaction.Transactional;
 
+import org.hibernate.exception.ConstraintViolationException;
+
 import com.redhat.hacbs.management.model.BuildQueue;
 import com.redhat.hacbs.management.model.ContainerImage;
 import com.redhat.hacbs.management.model.ImageDependency;
@@ -43,7 +45,20 @@ public void setup() {
         client.resources(JvmImageScan.class).inAnyNamespace().inform(new ResourceEventHandler<JvmImageScan>() {
             @Override
             public void onAdd(JvmImageScan obj) {
-                ExecutorRecorder.getCurrent().execute(() -> handleImage(obj));
+                ContainerImage image = ensureImageExists(obj);
+                if (image != null) {
+                    ExecutorRecorder.getCurrent().execute(() -> {
+                        for (var i = 0; i < 3; ++i) {
+                            try {
+                                //TODO: this can be a bit racey, multiple things happening at once can add the same artifact to the DB
+                                handleImage(obj, image.id);
+                                return;
+                            } catch (ConstraintViolationException e) {
+                                Log.errorf(e, "Failed to import");
+                            }
+                        }
+                    });
+                }
             }
 
             @Override
@@ -60,35 +75,41 @@ public void onDelete(JvmImageScan obj, boolean deletedFinalStateUnknown) {
     }
 
     @Transactional
-    void handleImage(JvmImageScan resource) {
+    ContainerImage ensureImageExists(JvmImageScan resource) {
         var image = resource.getSpec().getImage();
         if (!image.contains("@")) {
             Log.errorf("image %s has no digest, not saving scan result", image);
             client.resource(resource).delete();
-            return;
+            return null;
         }
         Log.infof("Processing image scan %s", resource.getMetadata().getName());
         if (resource.getStatus() == null) {
-            return;
+            return null;
         }
         if (Objects.equals(resource.getStatus().getState(), "JvmImageScanFailed")) {
-
             ContainerImage containerImage = ContainerImage.getOrCreate(image);
             containerImage.analysisComplete = true;
             containerImage.analysisFailed = true;
             containerImage.persist();
-            return;
+            return null;
         }
         if (!Objects.equals(resource.getStatus().getState(), "JvmImageScanComplete")) {
-            return;
+            return null;
         }
-        ContainerImage containerImage = ContainerImage.getOrCreate(image);
+        return ContainerImage.getOrCreate(image);
+    }
+
+    @Transactional
+    void handleImage(JvmImageScan resource, long imageId) {
+        ContainerImage containerImage = ContainerImage.findById(imageId);
         if (containerImage.analysisComplete) {
             return;
         }
         Map<String, ImageDependency> existing = new HashMap<>();
-        for (var i : containerImage.imageDependencies) {
-            existing.put(i.mavenArtifact.gav(), i);
+        if (containerImage.imageDependencies != null) {
+            for (var i : containerImage.imageDependencies) {
+                existing.put(i.mavenArtifact.gav(), i);
+            }
         }
         List<Results> results = resource.getStatus().getResults();
         if (results != null) {