adding missed conflict resolutions

ls1intum · Dec 12, 2024 · e3e12f2 · e3e12f2
1 parent 70b1328
commit e3e12f2
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 36 deletions.
diff --git a/src/main/java/de/tum/cit/aet/artemis/buildagent/service/SharedQueueProcessingService.java b/src/main/java/de/tum/cit/aet/artemis/buildagent/service/SharedQueueProcessingService.java
@@ -28,6 +28,7 @@
 import org.redisson.api.RMap;
 import org.redisson.api.RPriorityQueue;
 import org.redisson.api.RQueue;
+import org.redisson.api.RTopic;
 import org.redisson.api.RedissonClient;
 import org.redisson.api.listener.ListAddListener;
 import org.redisson.api.listener.ListRemoveListener;
@@ -101,8 +102,6 @@ public class SharedQueueProcessingService {
      */
     private final ReentrantLock pauseResumeLock = new ReentrantLock();
 
-    private UUID listenerId;
-
     /**
      * Scheduled future for checking availability and processing next build job.
      */
@@ -178,16 +177,16 @@ public void init() {
          */
         scheduledFuture = taskScheduler.scheduleAtFixedRate(this::checkAvailabilityAndProcessNextBuild, Duration.ofSeconds(10));
 
-        ITopic<String> pauseBuildAgentTopic = hazelcastInstance.getTopic("pauseBuildAgentTopic");
-        pauseBuildAgentTopic.addMessageListener(message -> {
-            if (buildAgentShortName.equals(message.getMessageObject())) {
+        RTopic pauseBuildAgentTopic = redissonClient.getTopic("pauseBuildAgentTopic");
+        pauseBuildAgentTopic.addListener(String.class, (channel, name) -> {
+            if (buildAgentShortName.equals(name)) {
                 pauseBuildAgent();
             }
         });
 
-        ITopic<String> resumeBuildAgentTopic = hazelcastInstance.getTopic("resumeBuildAgentTopic");
-        resumeBuildAgentTopic.addMessageListener(message -> {
-            if (buildAgentShortName.equals(message.getMessageObject())) {
+        RTopic resumeBuildAgentTopic = redissonClient.getTopic("resumeBuildAgentTopic");
+        resumeBuildAgentTopic.addListener(String.class, (channel, name) -> {
+            if (buildAgentShortName.equals(name)) {
                 resumeBuildAgent();
             }
         });
@@ -307,10 +306,6 @@ private void checkAvailabilityAndProcessNextBuild() {
         }
     }
 
-    private static boolean noDataMemberInClusterAvailable(HazelcastInstance hazelcastInstance) {
-        return hazelcastInstance.getCluster().getMembers().stream().allMatch(Member::isLiteMember);
-    }
-
     private BuildJobQueueItem addToProcessingJobs() {
         BuildJobQueueItem buildJob = buildJobQueue.poll();
         if (buildJob != null) {
@@ -490,7 +485,7 @@ private void pauseBuildAgent() {
 
         pauseResumeLock.lock();
         try {
-            log.info("Pausing build agent with address {}", hazelcastInstance.getCluster().getLocalMember().getAddress().toString());
+            log.info("Pausing build agent {}", getBuildAgentName());
 
             isPaused.set(true);
             removeListenerAndCancelScheduledFuture();
@@ -538,7 +533,7 @@ private void handleTimeoutAndCancelRunningJobs() {
         Set<String> runningBuildJobIdsAfterGracePeriod = buildJobManagementService.getRunningBuildJobIds();
         List<BuildJobQueueItem> runningBuildJobsAfterGracePeriod = processingJobs.getAll(runningBuildJobIdsAfterGracePeriod).values().stream().toList();
         runningBuildJobIdsAfterGracePeriod.forEach(buildJobManagementService::cancelBuildJob);
-        queue.addAll(runningBuildJobsAfterGracePeriod);
+        buildJobQueue.addAll(runningBuildJobsAfterGracePeriod);
         log.info("Cancelled running build jobs and added them back to the queue with Ids {}", runningBuildJobIdsAfterGracePeriod);
         log.debug("Cancelled running build jobs: {}", runningBuildJobsAfterGracePeriod);
     }
@@ -551,12 +546,16 @@ private void resumeBuildAgent() {
 
         pauseResumeLock.lock();
         try {
-            log.info("Resuming build agent with address {}", hazelcastInstance.getCluster().getLocalMember().getAddress().toString());
+            log.info("Resuming build agent {}", getBuildAgentName());
             isPaused.set(false);
             processResults.set(true);
             // We remove the listener and scheduledTask first to avoid having multiple listeners and scheduled tasks running
             removeListenerAndCancelScheduledFuture();
-            listenerId = queue.addItemListener(new QueuedBuildJobItemListener(), true);
+            listenerIdAdd = this.buildJobQueue.addListener((ListAddListener) name -> {
+                log.debug("CIBuildJobQueueItem added to queue: {}", name);
+                log.debug("Current queued items: {}", name);
+                checkAvailabilityAndProcessNextBuild();
+            });
             scheduledFuture = taskScheduler.scheduleAtFixedRate(this::checkAvailabilityAndProcessNextBuild, Duration.ofSeconds(10));
             checkAvailabilityAndProcessNextBuild();
             updateLocalBuildAgentInformation();

diff --git a/src/main/java/de/tum/cit/aet/artemis/programming/service/localci/LocalCITriggerService.java b/src/main/java/de/tum/cit/aet/artemis/programming/service/localci/LocalCITriggerService.java
@@ -20,6 +20,7 @@
 import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Service;
 
+import de.tum.cit.aet.artemis.buildagent.dto.BuildAgentDTO;
 import de.tum.cit.aet.artemis.buildagent.dto.BuildConfig;
 import de.tum.cit.aet.artemis.buildagent.dto.BuildJobQueueItem;
 import de.tum.cit.aet.artemis.buildagent.dto.DockerRunConfig;

diff --git a/...de/tum/cit/aet/artemis/buildagent/service/BuildAgentSshAuthenticationIntegrationTest.java b/...de/tum/cit/aet/artemis/buildagent/service/BuildAgentSshAuthenticationIntegrationTest.java
@@ -31,10 +31,10 @@ void testWriteSSHKey() {
     }
 
     @Test
-    void testSSHInHazelcast() {
+    void testSSHInRedis() {
         sharedQueueProcessingService.updateBuildAgentInformation();
         Map<String, BuildAgentInformation> buildAgentInformation = redissonClient.getMap("buildAgentInformation");
-        assertThat(buildAgentInformation.values()).as("SSH public key available in hazelcast.")
+        assertThat(buildAgentInformation.values()).as("SSH public key available in Redis.")
                 .anyMatch(agent -> agent.publicSshKey().equals(buildAgentSSHKeyService.getPublicKeyAsString()));
     }
 }
diff --git a/.../de/tum/cit/aet/artemis/programming/AbstractProgrammingIntegrationLocalCILocalVCTest.java b/.../de/tum/cit/aet/artemis/programming/AbstractProgrammingIntegrationLocalCILocalVCTest.java
@@ -1,11 +1,9 @@
 package de.tum.cit.aet.artemis.programming;
 
+import org.redisson.api.RedissonClient;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
 
-import com.hazelcast.core.HazelcastInstance;
-
 import de.tum.cit.aet.artemis.atlas.competency.util.CompetencyUtilService;
 import de.tum.cit.aet.artemis.buildagent.service.SharedQueueProcessingService;
 import de.tum.cit.aet.artemis.core.connector.AeolusRequestMockProvider;
@@ -30,8 +28,7 @@ public abstract class AbstractProgrammingIntegrationLocalCILocalVCTest extends A
 
     // Config
     @Autowired
-    @Qualifier("hazelcastInstance")
-    protected HazelcastInstance hazelcastInstance;
+    protected RedissonClient redissonClient;
 
     @Value("${artemis.user-management.internal-admin.username}")
     protected String localVCUsername;

diff --git a/src/test/java/de/tum/cit/aet/artemis/programming/icl/LocalCIIntegrationTest.java b/src/test/java/de/tum/cit/aet/artemis/programming/icl/LocalCIIntegrationTest.java
@@ -40,6 +40,8 @@
 import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.mockito.ArgumentMatcher;
 import org.mockito.Mockito;
+import org.redisson.api.RMap;
+import org.redisson.api.RQueue;
 import org.springframework.core.io.FileSystemResource;
 import org.springframework.http.HttpStatus;
 import org.springframework.security.test.context.support.WithMockUser;
@@ -49,8 +51,6 @@
 import com.github.dockerjava.api.command.ExecStartCmd;
 import com.github.dockerjava.api.exception.NotFoundException;
 import com.github.dockerjava.api.model.Frame;
-import com.hazelcast.collection.IQueue;
-import com.hazelcast.map.IMap;
 
 import de.tum.cit.aet.artemis.assessment.domain.Result;
 import de.tum.cit.aet.artemis.buildagent.dto.BuildJobQueueItem;
@@ -501,20 +501,20 @@ void testDisableNetworkAccessAndEnvVars() {
     @WithMockUser(username = TEST_PREFIX + "student1", roles = "USER")
     void testPauseAndResumeBuildAgent() {
         String buildAgentName = "artemis-build-agent-test";
-        hazelcastInstance.getTopic("pauseBuildAgentTopic").publish(buildAgentName);
+        redissonClient.getTopic("pauseBuildAgentTopic").publish(buildAgentName);
 
         ProgrammingExerciseStudentParticipation studentParticipation = localVCLocalCITestService.createParticipation(programmingExercise, student1Login);
 
         localVCServletService.processNewPush(commitHash, studentAssignmentRepository.originGit.getRepository());
         await().until(() -> {
-            IQueue<BuildJobQueueItem> buildQueue = hazelcastInstance.getQueue("buildJobQueue");
-            IMap<String, BuildJobQueueItem> buildJobMap = hazelcastInstance.getMap("processingJobs");
+            RQueue<BuildJobQueueItem> buildQueue = redissonClient.getQueue("buildJobQueue");
+            RMap<String, BuildJobQueueItem> buildJobMap = redissonClient.getMap("processingJobs");
             BuildJobQueueItem buildJobQueueItem = buildQueue.peek();
 
             return buildJobQueueItem != null && buildJobQueueItem.buildConfig().commitHashToBuild().equals(commitHash) && !buildJobMap.containsKey(buildJobQueueItem.id());
         });
 
-        hazelcastInstance.getTopic("resumeBuildAgentTopic").publish(buildAgentName);
+        redissonClient.getTopic("resumeBuildAgentTopic").publish(buildAgentName);
         localVCLocalCITestService.testLatestSubmission(studentParticipation.getId(), commitHash, 1, false);
     }
 }
diff --git a/src/test/java/de/tum/cit/aet/artemis/programming/icl/LocalCIResourceIntegrationTest.java b/src/test/java/de/tum/cit/aet/artemis/programming/icl/LocalCIResourceIntegrationTest.java
@@ -34,12 +34,14 @@
 import de.tum.cit.aet.artemis.buildagent.dto.FinishedBuildJobDTO;
 import de.tum.cit.aet.artemis.buildagent.dto.JobTimingInfo;
 import de.tum.cit.aet.artemis.buildagent.dto.RepositoryInfo;
+import de.tum.cit.aet.artemis.buildagent.service.SharedQueueProcessingService;
 import de.tum.cit.aet.artemis.core.dto.SortingOrder;
 import de.tum.cit.aet.artemis.core.dto.pageablesearch.PageableSearchDTO;
 import de.tum.cit.aet.artemis.programming.AbstractProgrammingIntegrationLocalCILocalVCTestBase;
 import de.tum.cit.aet.artemis.programming.domain.RepositoryType;
 import de.tum.cit.aet.artemis.programming.domain.build.BuildJob;
 import de.tum.cit.aet.artemis.programming.domain.build.BuildStatus;
+import de.tum.cit.aet.artemis.programming.service.BuildLogEntryService;
 
 class LocalCIResourceIntegrationTest extends AbstractProgrammingIntegrationLocalCILocalVCTestBase {
 
@@ -99,11 +101,13 @@ void createJobs() {
         BuildConfig buildConfig = new BuildConfig("echo 'test'", "test", "test", "test", "test", "test", null, null, false, false, false, null, 0, null, null, null, null);
         RepositoryInfo repositoryInfo = new RepositoryInfo("test", null, RepositoryType.USER, "test", "test", "test", null, null);
 
-        job1 = new BuildJobQueueItem("1", "job1", "address1", 1, course.getId(), 1, 1, 1, BuildStatus.SUCCESSFUL, repositoryInfo, jobTimingInfo1, buildConfig, null);
-        job2 = new BuildJobQueueItem("2", "job2", "address1", 2, course.getId(), 1, 1, 1, BuildStatus.SUCCESSFUL, repositoryInfo, jobTimingInfo2, buildConfig, null);
+        buildAgent = new BuildAgentDTO(buildAgentShortName, "address1", buildAgentDisplayName);
+
+        job1 = new BuildJobQueueItem("1", "job1", buildAgent, 1, course.getId(), 1, 1, 1, BuildStatus.SUCCESSFUL, repositoryInfo, jobTimingInfo1, buildConfig, null);
+        job2 = new BuildJobQueueItem("2", "job2", buildAgent, 2, course.getId(), 1, 1, 1, BuildStatus.SUCCESSFUL, repositoryInfo, jobTimingInfo2, buildConfig, null);
         String memberAddress = "build-agent-1";
         agent1 = new BuildAgentInformation(buildAgent, 1, 0, new ArrayList<>(List.of(job1)), BuildAgentInformation.BuildAgentStatus.IDLE, new ArrayList<>(List.of(job2)), null);
-        BuildJobQueueItem finishedJobQueueItem1 = new BuildJobQueueItem("3", "job3", "address1", 3, course.getId(), 1, 1, 1, BuildStatus.SUCCESSFUL, repositoryInfo, jobTimingInfo1,
+        BuildJobQueueItem finishedJobQueueItem1 = new BuildJobQueueItem("3", "job3", buildAgent, 3, course.getId(), 1, 1, 1, BuildStatus.SUCCESSFUL, repositoryInfo, jobTimingInfo1,
                 buildConfig, null);
         BuildJobQueueItem finishedJobQueueItem2 = new BuildJobQueueItem("4", "job4", buildAgent, 4, course.getId() + 1, 1, 1, 1, BuildStatus.FAILED, repositoryInfo, jobTimingInfo2,
                 buildConfig, null);

diff --git a/src/test/java/de/tum/cit/aet/artemis/programming/icl/LocalCIServiceTest.java b/src/test/java/de/tum/cit/aet/artemis/programming/icl/LocalCIServiceTest.java
@@ -27,16 +27,22 @@
 import de.tum.cit.aet.artemis.buildagent.dto.BuildJobQueueItem;
 import de.tum.cit.aet.artemis.buildagent.dto.JobTimingInfo;
 import de.tum.cit.aet.artemis.buildagent.dto.RepositoryInfo;
+import de.tum.cit.aet.artemis.buildagent.service.SharedQueueProcessingService;
 import de.tum.cit.aet.artemis.core.domain.Course;
+import de.tum.cit.aet.artemis.exercise.participation.util.ParticipationUtilService;
+import de.tum.cit.aet.artemis.exercise.util.ExerciseUtilService;
 import de.tum.cit.aet.artemis.programming.AbstractProgrammingIntegrationLocalCILocalVCTest;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingExercise;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingExerciseBuildConfig;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingExerciseStudentParticipation;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingLanguage;
 import de.tum.cit.aet.artemis.programming.domain.RepositoryType;
 import de.tum.cit.aet.artemis.programming.dto.CheckoutDirectoriesDTO;
+import de.tum.cit.aet.artemis.programming.service.BuildScriptProviderService;
+import de.tum.cit.aet.artemis.programming.service.aeolus.AeolusTemplateService;
 import de.tum.cit.aet.artemis.programming.service.aeolus.Windfile;
 import de.tum.cit.aet.artemis.programming.service.ci.ContinuousIntegrationService.BuildStatus;
+import de.tum.cit.aet.artemis.programming.util.ProgrammingExerciseUtilService;
 
 class LocalCIServiceTest extends AbstractProgrammingIntegrationLocalCILocalVCTest {
 

diff --git a/src/test/java/de/tum/cit/aet/artemis/programming/icl/LocalVCLocalCIIntegrationTest.java b/src/test/java/de/tum/cit/aet/artemis/programming/icl/LocalVCLocalCIIntegrationTest.java
@@ -36,13 +36,12 @@
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.parallel.Execution;
 import org.junit.jupiter.api.parallel.ExecutionMode;
+import org.redisson.api.RQueue;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.http.HttpStatus;
 import org.springframework.security.test.context.support.WithMockUser;
 
-import com.hazelcast.collection.IQueue;
-
 import de.tum.cit.aet.artemis.buildagent.dto.BuildJobQueueItem;
 import de.tum.cit.aet.artemis.core.service.ldap.LdapUserDto;
 import de.tum.cit.aet.artemis.exam.domain.Exam;
@@ -96,7 +95,7 @@ class LocalVCLocalCIIntegrationTest extends AbstractProgrammingIntegrationLocalC
 
     private String teamRepositorySlug;
 
-    protected IQueue<BuildJobQueueItem> queuedJobs;
+    protected RQueue<BuildJobQueueItem> queuedJobs;
 
     @Override
     protected String getTestPrefix() {
@@ -153,7 +152,7 @@ void initRepositories() throws GitAPIException, IOException, URISyntaxException,
 
         localVCLocalCITestService.mockInspectImage(dockerClient);
 
-        queuedJobs = hazelcastInstance.getQueue("buildJobQueue");
+        queuedJobs = redissonClient.getQueue("buildJobQueue");
     }
 
     @AfterEach