Merge branch 'develop' of https://github.com/ls1intum/Artemis into ch…

…ore/newResult-websocket-dto

src/main/java/de/tum/in/www1/artemis/service/RepositoryAccessService.java

@@ -147,20 +147,22 @@ else if (!isStudent && !isOwner) {
      * Checks if the user has access to the test repository of the given programming exercise.
      * Throws an {@link AccessForbiddenException} otherwise.
      *
-     * @param atLeastEditor if true, the user needs at least editor permissions, otherwise only teaching assistant permissions are required.
-     * @param exercise      the programming exercise the test repository belongs to.
-     * @param user          the user that wants to access the test repository.
+     * @param atLeastEditor  if true, the user needs at least editor permissions, otherwise only teaching assistant permissions are required.
+     * @param exercise       the programming exercise the test repository belongs to.
+     * @param user           the user that wants to access the test repository.
+     * @param repositoryType the type of the repository.
      */
-    public void checkAccessTestRepositoryElseThrow(boolean atLeastEditor, ProgrammingExercise exercise, User user) {
+    public void checkAccessTestOrAuxRepositoryElseThrow(boolean atLeastEditor, ProgrammingExercise exercise, User user, String repositoryType) {
         if (atLeastEditor) {
             if (!authorizationCheckService.isAtLeastEditorInCourse(exercise.getCourseViaExerciseGroupOrCourseMember(), user)) {
-                throw new AccessForbiddenException("You are not allowed to access the test repository of this programming exercise.");
+                throw new AccessForbiddenException("You are not allowed to push to the " + repositoryType + " repository of this programming exercise.");
             }
         }
         else {
             if (!authorizationCheckService.isAtLeastTeachingAssistantInCourse(exercise.getCourseViaExerciseGroupOrCourseMember(), user)) {
-                throw new AccessForbiddenException("You are not allowed to push to the test repository of this programming exercise.");
+                throw new AccessForbiddenException("You are not allowed to access the " + repositoryType + " repository of this programming exercise.");
             }
         }
     }
+
 }

src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIBuildJobExecutionService.java

@@ -17,6 +17,7 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.io.IOUtils;
+import org.hibernate.Hibernate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -28,9 +29,11 @@
 import com.github.dockerjava.api.model.HostConfig;
 
 import de.tum.in.www1.artemis.config.localvcci.LocalCIConfiguration;
+import de.tum.in.www1.artemis.domain.AuxiliaryRepository;
 import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseParticipation;
 import de.tum.in.www1.artemis.exception.LocalCIException;
 import de.tum.in.www1.artemis.exception.localvc.LocalVCInternalException;
+import de.tum.in.www1.artemis.repository.AuxiliaryRepositoryRepository;
 import de.tum.in.www1.artemis.service.connectors.ci.ContinuousIntegrationService;
 import de.tum.in.www1.artemis.service.connectors.localci.dto.LocalCIBuildResult;
 import de.tum.in.www1.artemis.service.connectors.localvc.LocalVCRepositoryUrl;
@@ -54,6 +57,8 @@ public class LocalCIBuildJobExecutionService {
 
     private final LocalCIContainerService localCIContainerService;
 
+    private final AuxiliaryRepositoryRepository auxiliaryRepositoryRepository;
+
     /**
      * Instead of creating a new XMLInputFactory for every build job, it is created once and provided as a Bean (see {@link LocalCIConfiguration#localCIXMLInputFactory()}).
      */
@@ -66,16 +71,17 @@ public class LocalCIBuildJobExecutionService {
     private String localVCBasePath;
 
     public LocalCIBuildJobExecutionService(LocalCIBuildPlanService localCIBuildPlanService, Optional<VersionControlService> versionControlService,
-            LocalCIContainerService localCIContainerService, XMLInputFactory localCIXMLInputFactory) {
+            LocalCIContainerService localCIContainerService, AuxiliaryRepositoryRepository auxiliaryRepositoryRepository, XMLInputFactory localCIXMLInputFactory) {
         this.localCIBuildPlanService = localCIBuildPlanService;
         this.versionControlService = versionControlService;
         this.localCIContainerService = localCIContainerService;
+        this.auxiliaryRepositoryRepository = auxiliaryRepositoryRepository;
         this.localCIXMLInputFactory = localCIXMLInputFactory;
     }
 
     public enum LocalCIBuildJobRepositoryType {
 
-        ASSIGNMENT("assignment"), TEST("test");
+        ASSIGNMENT("assignment"), TEST("test"), AUXILIARY("auxiliary");
 
         private final String name;
 
@@ -104,14 +110,47 @@ public LocalCIBuildResult runBuildJob(ProgrammingExerciseParticipation participa
         // Update the build plan status to "BUILDING".
         localCIBuildPlanService.updateBuildPlanStatus(participation, ContinuousIntegrationService.BuildStatus.BUILDING);
 
+        List<AuxiliaryRepository> auxiliaryRepositories;
+
+        // If the auxiliary repositories are not initialized, we need to fetch them from the database.
+        if (Hibernate.isInitialized(participation.getProgrammingExercise().getAuxiliaryRepositories())) {
+            auxiliaryRepositories = participation.getProgrammingExercise().getAuxiliaryRepositories();
+        }
+        else {
+            auxiliaryRepositories = auxiliaryRepositoryRepository.findByExerciseId(participation.getProgrammingExercise().getId());
+        }
+
+        // Prepare script
+        Path buildScriptPath = localCIContainerService.createBuildScript(participation.getProgrammingExercise(), auxiliaryRepositories);
+
         // Retrieve the paths to the repositories that the build job needs.
         // This includes the assignment repository (the one to be tested, e.g. the student's repository, or the template repository), and the tests repository which includes
         // the tests to be executed.
         LocalVCRepositoryUrl assignmentRepositoryUrl;
         LocalVCRepositoryUrl testsRepositoryUrl;
+        LocalVCRepositoryUrl[] auxiliaryRepositoriesUrls;
+        Path[] auxiliaryRepositoriesPaths;
+        String[] auxiliaryRepositoryNames;
+
         try {
             assignmentRepositoryUrl = new LocalVCRepositoryUrl(participation.getRepositoryUrl(), localVCBaseUrl);
             testsRepositoryUrl = new LocalVCRepositoryUrl(participation.getProgrammingExercise().getTestRepositoryUrl(), localVCBaseUrl);
+
+            if (!auxiliaryRepositories.isEmpty()) {
+                auxiliaryRepositoriesUrls = new LocalVCRepositoryUrl[auxiliaryRepositories.size()];
+                auxiliaryRepositoriesPaths = new Path[auxiliaryRepositories.size()];
+                auxiliaryRepositoryNames = new String[auxiliaryRepositories.size()];
+
+                for (int i = 0; i < auxiliaryRepositories.size(); i++) {
+                    auxiliaryRepositoriesUrls[i] = new LocalVCRepositoryUrl(auxiliaryRepositories.get(i).getRepositoryUrl(), localVCBaseUrl);
+                    auxiliaryRepositoriesPaths[i] = auxiliaryRepositoriesUrls[i].getLocalRepositoryPath(localVCBasePath).toAbsolutePath();
+                    auxiliaryRepositoryNames[i] = auxiliaryRepositories.get(i).getName();
+                }
+            }
+            else {
+                auxiliaryRepositoriesPaths = new Path[0];
+                auxiliaryRepositoryNames = new String[0];
+            }
         }
         catch (LocalVCInternalException e) {
             throw new LocalCIException("Error while creating LocalVCRepositoryUrl", e);
@@ -130,7 +169,8 @@ public LocalCIBuildResult runBuildJob(ProgrammingExerciseParticipation participa
 
         // Create the volume configuration for the container. The assignment repository, the tests repository, and the build script are bound into the container to be used by
         // the build job.
-        HostConfig volumeConfig = localCIContainerService.createVolumeConfig(assignmentRepositoryPath, testsRepositoryPath);
+        HostConfig volumeConfig = localCIContainerService.createVolumeConfig(assignmentRepositoryPath, testsRepositoryPath, auxiliaryRepositoriesPaths, auxiliaryRepositoryNames,
+                buildScriptPath);
 
         // Create the container from the "ls1tum/artemis-maven-template" image with the local paths to the Git repositories and the shell script bound to it. Also give the
         // container information about the branch and commit hash to be used.
@@ -182,6 +222,8 @@ private LocalCIBuildResult runScriptAndParseResults(ProgrammingExerciseParticipa
             // Could not read commit hash from .git folder. Stop the container and return a build result that indicates that the build failed (empty list for failed tests and
             // empty list for successful tests).
             localCIContainerService.stopContainer(containerName);
+            // Delete script file from host system
+            localCIContainerService.deleteScriptFile(participation.getProgrammingExercise().getId().toString());
             return constructFailedBuildResult(branch, assignmentRepoCommitHash, testRepoCommitHash, buildCompletedDate);
         }
 
@@ -198,11 +240,16 @@ private LocalCIBuildResult runScriptAndParseResults(ProgrammingExerciseParticipa
             // If the test results are not found, this means that something went wrong during the build and testing of the submission.
             // Stop the container and return a build results that indicates that the build failed.
             localCIContainerService.stopContainer(containerName);
+            // Delete script file from host system
+            localCIContainerService.deleteScriptFile(participation.getProgrammingExercise().getId().toString());
             return constructFailedBuildResult(branch, assignmentRepoCommitHash, testRepoCommitHash, buildCompletedDate);
         }
 
         localCIContainerService.stopContainer(containerName);
 
+        // Delete script file from host system
+        localCIContainerService.deleteScriptFile(participation.getProgrammingExercise().getId().toString());
+
         LocalCIBuildResult buildResult;
         try {
             buildResult = parseTestResults(testResultsTarInputStream, branch, assignmentRepoCommitHash, testRepoCommitHash, buildCompletedDate);

src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIBuildJobManagementService.java

@@ -73,8 +73,6 @@ public LocalCIBuildJobManagementService(LocalCIBuildJobExecutionService localCIB
      * @throws LocalCIException If the build job could not be submitted to the executor service.
      */
     public CompletableFuture<LocalCIBuildResult> addBuildJobToQueue(ProgrammingExerciseParticipation participation, String commitHash) {
-
-        // It should not be possible to create a programming exercise with a different project type than Gradle. This is just a sanity check.
         ProjectType projectType = participation.getProgrammingExercise().getProjectType();
         if (projectType == null || !projectType.isGradle()) {
             throw new LocalCIException("Project type must be Gradle.");

src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIContainerService.java

@@ -1,7 +1,10 @@
 package de.tum.in.www1.artemis.service.connectors.localci;
 
+import java.io.BufferedWriter;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Optional;
@@ -24,7 +27,8 @@
 import com.github.dockerjava.api.model.HostConfig;
 import com.github.dockerjava.api.model.Volume;
 
-import de.tum.in.www1.artemis.config.localvcci.LocalCIConfiguration;
+import de.tum.in.www1.artemis.domain.AuxiliaryRepository;
+import de.tum.in.www1.artemis.domain.ProgrammingExercise;
 import de.tum.in.www1.artemis.exception.LocalCIException;
 
 /**
@@ -39,33 +43,35 @@ public class LocalCIContainerService {
 
     private final DockerClient dockerClient;
 
-    /**
-     * The Path to the script file located in the resources folder. The script file contains the steps that run the tests on the Docker container.
-     * This path is provided as a Bean, because the retrieval is quite costly in the production environment (see {@link LocalCIConfiguration#buildScriptFilePath()}).
-     */
-    private final Path buildScriptFilePath;
-
     @Value("${artemis.continuous-integration.build.images.java.default}")
     String dockerImage;
 
-    public LocalCIContainerService(DockerClient dockerClient, Path buildScriptFilePath) {
+    public LocalCIContainerService(DockerClient dockerClient) {
         this.dockerClient = dockerClient;
-        this.buildScriptFilePath = buildScriptFilePath;
     }
 
     /**
      * Configure the volumes of the container such that it can access the assignment repository, the test repository, and the build script.
      *
-     * @param assignmentRepositoryPath the path to the assignment repository in the file system
-     * @param testRepositoryPath       the path to the test repository in the file system
+     * @param assignmentRepositoryPath   the path to the assignment repository in the file system
+     * @param testRepositoryPath         the path to the test repository in the file system
+     * @param auxiliaryRepositoriesPaths the paths to the auxiliary repositories in the file system
+     * @param auxiliaryRepositoryNames   the names of the auxiliary repositories
+     * @param buildScriptPath            the path to the build script in the file system
      * @return the host configuration for the container containing the binds to the assignment repository, the test repository, and the build script
      */
-    public HostConfig createVolumeConfig(Path assignmentRepositoryPath, Path testRepositoryPath) {
-        return HostConfig.newHostConfig().withAutoRemove(true) // Automatically remove the container when it exits.
-                .withBinds(
-                        new Bind(assignmentRepositoryPath.toString(), new Volume("/" + LocalCIBuildJobExecutionService.LocalCIBuildJobRepositoryType.ASSIGNMENT + "-repository")),
-                        new Bind(testRepositoryPath.toString(), new Volume("/" + LocalCIBuildJobExecutionService.LocalCIBuildJobRepositoryType.TEST + "-repository")),
-                        new Bind(buildScriptFilePath.toString(), new Volume("/script.sh")));
+    public HostConfig createVolumeConfig(Path assignmentRepositoryPath, Path testRepositoryPath, Path[] auxiliaryRepositoriesPaths, String[] auxiliaryRepositoryNames,
+            Path buildScriptPath) {
+
+        Bind[] binds = new Bind[3 + auxiliaryRepositoriesPaths.length];
+        binds[0] = new Bind(assignmentRepositoryPath.toString(), new Volume("/" + LocalCIBuildJobExecutionService.LocalCIBuildJobRepositoryType.ASSIGNMENT + "-repository"));
+        binds[1] = new Bind(testRepositoryPath.toString(), new Volume("/" + LocalCIBuildJobExecutionService.LocalCIBuildJobRepositoryType.TEST + "-repository"));
+        for (int i = 0; i < auxiliaryRepositoriesPaths.length; i++) {
+            binds[2 + i] = new Bind(auxiliaryRepositoriesPaths[i].toString(), new Volume("/" + auxiliaryRepositoryNames[i] + "-repository"));
+        }
+        binds[2 + auxiliaryRepositoriesPaths.length] = new Bind(buildScriptPath.toString(), new Volume("/script.sh"));
+
+        return HostConfig.newHostConfig().withAutoRemove(true).withBinds(binds); // Automatically remove the container when it exits.
     }
 
     /**
@@ -196,4 +202,103 @@ public void stopContainer(String containerName) {
         ExecCreateCmdResponse createStopContainerFileCmdResponse = dockerClient.execCreateCmd(containerId).withCmd("touch", "stop_container.txt").exec();
         dockerClient.execStartCmd(createStopContainerFileCmdResponse.getId()).exec(new ResultCallback.Adapter<>());
     }
+
+    /**
+     * Creates a build script for a given programming exercise.
+     * The build script is stored in a file in the local-ci-scripts directory.
+     * The build script is used to build the programming exercise in a Docker container.
+     *
+     * @param programmingExercise   the programming exercise for which to create the build script
+     * @param auxiliaryRepositories the auxiliary repositories of the programming exercise
+     * @return the path to the build script file
+     */
+    public Path createBuildScript(ProgrammingExercise programmingExercise, List<AuxiliaryRepository> auxiliaryRepositories) {
+        Long programmingExerciseId = programmingExercise.getId();
+        boolean hasAuxiliaryRepositories = auxiliaryRepositories != null && !auxiliaryRepositories.isEmpty();
+
+        Path scriptsPath = Path.of("local-ci-scripts");
+
+        if (!Files.exists(scriptsPath)) {
+            try {
+                Files.createDirectory(scriptsPath);
+            }
+            catch (IOException e) {
+                throw new LocalCIException("Failed to create directory for local CI scripts", e);
+            }
+        }
+
+        String buildScriptPath = scriptsPath.toAbsolutePath() + "/" + programmingExerciseId.toString() + "-build.sh";
+
+        StringBuilder buildScript = new StringBuilder("""
+                #!/bin/bash
+                mkdir /repositories
+                cd /repositories
+                """);
+
+        // Checkout tasks
+        buildScript.append("""
+                git clone --depth 1 --branch $ARTEMIS_DEFAULT_BRANCH file:///test-repository
+                git clone --depth 1 --branch $ARTEMIS_DEFAULT_BRANCH file:///assignment-repository
+                """);
+
+        if (hasAuxiliaryRepositories) {
+            for (AuxiliaryRepository auxiliaryRepository : auxiliaryRepositories) {
+                buildScript.append("git clone --depth 1 --branch $ARTEMIS_DEFAULT_BRANCH file:///").append(auxiliaryRepository.getName()).append("-repository\n");
+            }
+        }
+
+        buildScript.append("""
+                cd assignment-repository
+                if [ -n "$ARTEMIS_ASSIGNMENT_REPOSITORY_COMMIT_HASH" ]; then
+                    git fetch --depth 1 origin "$ARTEMIS_ASSIGNMENT_REPOSITORY_COMMIT_HASH"
+                    git checkout "$ARTEMIS_ASSIGNMENT_REPOSITORY_COMMIT_HASH"
+                fi
+                mkdir /repositories/test-repository/assignment
+                cp -a /repositories/assignment-repository/. /repositories/test-repository/assignment/
+                """);
+
+        // Copy auxiliary repositories to checkout directories
+        if (hasAuxiliaryRepositories) {
+            for (AuxiliaryRepository auxiliaryRepository : auxiliaryRepositories) {
+                buildScript.append("cp -a /repositories/").append(auxiliaryRepository.getName()).append("-repository/. /repositories/test-repository/")
+                        .append(auxiliaryRepository.getCheckoutDirectory()).append("/\n");
+            }
+        }
+
+        buildScript.append("cd /repositories/test-repository\n");
+
+        // programming language specific tasks
+        buildScript.append("""
+                chmod +x gradlew
+                sed -i -e 's/\\r$//' gradlew
+                ./gradlew clean test""");
+
+        try {
+            BufferedWriter writer = new BufferedWriter(new FileWriter(buildScriptPath));
+            writer.write(buildScript.toString());
+            writer.close();
+        }
+        catch (IOException e) {
+            throw new LocalCIException("Failed to create build script file", e);
+        }
+
+        return Path.of(buildScriptPath);
+    }
+
+    /**
+     * Deletes the build script for a given programming exercise.
+     * The build script is stored in a file in the local-ci-scripts directory.
+     *
+     * @param exerciseID the ID of the programming exercise for which to delete the build script
+     */
+    public void deleteScriptFile(String exerciseID) {
+        Path scriptsPath = Path.of("local-ci-scripts");
+        String buildScriptPath = scriptsPath.toAbsolutePath() + "/" + exerciseID + "-build.sh";
+        try {
+            Files.deleteIfExists(Path.of(buildScriptPath));
+        }
+        catch (IOException e) {
+            throw new LocalCIException("Failed to delete build script file", e);
+        }
+    }
 }