From f2ad596ee759b2939dc6b9b2e3e6a814f606b03b Mon Sep 17 00:00:00 2001 From: Laurenz Blumentritt <38919977+laurenzfb@users.noreply.github.com> Date: Sun, 15 Oct 2023 18:36:41 +0200 Subject: [PATCH] Programming exercises: Support python exercises for local continuous integration (#7369) --- .../LocalCIBuildJobExecutionService.java | 83 ++++++++++------- .../LocalCIBuildJobManagementService.java | 23 +++-- .../localci/LocalCIContainerService.java | 91 ++++++++++++++----- ...alCIProgrammingLanguageFeatureService.java | 2 +- .../connectors/localci/LocalCIService.java | 12 ++- .../localvc/LocalVCRepositoryUrl.java | 10 ++ 6 files changed, 154 insertions(+), 67 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIBuildJobExecutionService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIBuildJobExecutionService.java index a2a160e4e2aa..dec7453e8f85 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIBuildJobExecutionService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIBuildJobExecutionService.java @@ -43,8 +43,8 @@ /** * This service contains the logic to execute a build job for a programming exercise participation in the local CI system. - * The {@link #runBuildJob(ProgrammingExerciseParticipation, String, String)} method is wrapped into a Callable by the {@link LocalCIBuildJobManagementService} and submitted to the - * executor service. + * The {@link #runBuildJob(ProgrammingExerciseParticipation, String, String, String)} method is wrapped into a Callable by the {@link LocalCIBuildJobManagementService} and + * submitted to the executor service. */ @Service @Profile("localci") @@ -68,8 +68,8 @@ public class LocalCIBuildJobExecutionService { @Value("${artemis.version-control.url}") private URL localVCBaseUrl; - @Value("${artemis.version-control.local-vcs-repo-path}") - private String localVCBasePath; + @Value("${artemis.repo-clone-path}") + private String repoClonePath; public LocalCIBuildJobExecutionService(LocalCIBuildPlanService localCIBuildPlanService, Optional versionControlService, LocalCIContainerService localCIContainerService, AuxiliaryRepositoryRepository auxiliaryRepositoryRepository, XMLInputFactory localCIXMLInputFactory) { @@ -104,10 +104,11 @@ public String toString() { * @param commitHash The commit hash of the commit that should be built. If it is null, the latest commit of the default branch will be built. * @param containerName The name of the Docker container that will be used to run the build job. * It needs to be prepared beforehand to stop and remove the container if something goes wrong here. + * @param dockerImage The Docker image that will be used to run the build job. * @return The build result. * @throws LocalCIException If some error occurs while preparing or running the build job. */ - public LocalCIBuildResult runBuildJob(ProgrammingExerciseParticipation participation, String commitHash, String containerName) { + public LocalCIBuildResult runBuildJob(ProgrammingExerciseParticipation participation, String commitHash, String containerName, String dockerImage) { // Update the build plan status to "BUILDING". localCIBuildPlanService.updateBuildPlanStatus(participation, ContinuousIntegrationService.BuildStatus.BUILDING); @@ -144,7 +145,7 @@ public LocalCIBuildResult runBuildJob(ProgrammingExerciseParticipation participa for (int i = 0; i < auxiliaryRepositories.size(); i++) { auxiliaryRepositoriesUrls[i] = new LocalVCRepositoryUrl(auxiliaryRepositories.get(i).getRepositoryUrl(), localVCBaseUrl); - auxiliaryRepositoriesPaths[i] = auxiliaryRepositoriesUrls[i].getLocalRepositoryPath(localVCBasePath).toAbsolutePath(); + auxiliaryRepositoriesPaths[i] = auxiliaryRepositoriesUrls[i].getRepoClonePath(repoClonePath).toAbsolutePath(); auxiliaryRepositoryNames[i] = auxiliaryRepositories.get(i).getName(); } } @@ -157,8 +158,8 @@ public LocalCIBuildResult runBuildJob(ProgrammingExerciseParticipation participa throw new LocalCIException("Error while creating LocalVCRepositoryUrl", e); } - Path assignmentRepositoryPath = assignmentRepositoryUrl.getLocalRepositoryPath(localVCBasePath).toAbsolutePath(); - Path testsRepositoryPath = testsRepositoryUrl.getLocalRepositoryPath(localVCBasePath).toAbsolutePath(); + Path assignmentRepositoryPath = assignmentRepositoryUrl.getRepoClonePath(repoClonePath).toAbsolutePath(); + Path testsRepositoryPath = testsRepositoryUrl.getRepoClonePath(repoClonePath).toAbsolutePath(); String branch; try { @@ -171,7 +172,7 @@ public LocalCIBuildResult runBuildJob(ProgrammingExerciseParticipation participa // Create the container from the "ls1tum/artemis-maven-template" image with the local paths to the Git repositories and the shell script bound to it. Also give the // container information about the branch and commit hash to be used. // This does not start the container yet. - CreateContainerResponse container = localCIContainerService.configureContainer(containerName, branch, commitHash); + CreateContainerResponse container = localCIContainerService.configureContainer(containerName, branch, commitHash, dockerImage); return runScriptAndParseResults(participation, containerName, container.getId(), branch, commitHash, assignmentRepositoryPath, testsRepositoryPath, auxiliaryRepositoriesPaths, auxiliaryRepositoryNames, buildScriptPath); @@ -227,7 +228,7 @@ private LocalCIBuildResult runScriptAndParseResults(ProgrammingExerciseParticipa return constructFailedBuildResult(branch, assignmentRepoCommitHash, testRepoCommitHash, buildCompletedDate); } - List testResultsPaths = getTestResultPath(participation.getProgrammingExercise()); + List testResultsPaths = getTestResultPaths(participation.getProgrammingExercise()); // Get an input stream of the test result files. List testResultsTarInputStreams = new ArrayList<>(); @@ -269,34 +270,47 @@ private LocalCIBuildResult runScriptAndParseResults(ProgrammingExerciseParticipa // --- Helper methods ---- - private List getTestResultPath(ProgrammingExercise programmingExercise) { - List testResultPaths = new ArrayList<>(); + private List getTestResultPaths(ProgrammingExercise programmingExercise) { switch (programmingExercise.getProgrammingLanguage()) { case JAVA, KOTLIN -> { - if (ProjectType.isMavenProject(programmingExercise.getProjectType())) { - if (programmingExercise.hasSequentialTestRuns()) { - testResultPaths.add("/repositories/test-repository/structural/target/surefire-reports"); - testResultPaths.add("/repositories/test-repository/behavior/target/surefire-reports"); - } - else { - testResultPaths.add("/repositories/test-repository/target/surefire-reports"); - } - } - else { - if (programmingExercise.hasSequentialTestRuns()) { - testResultPaths.add("/repositories/test-repository/build/test-results/behaviorTests"); - testResultPaths.add("/repositories/test-repository/build/test-results/structuralTests"); - } - else { - testResultPaths.add("/repositories/test-repository/build/test-results/test"); - } - } - return testResultPaths; + return getJavaKotlinTestResultPaths(programmingExercise); + } + case PYTHON -> { + return getPythonTestResultPaths(); } default -> throw new IllegalArgumentException("Programming language " + programmingExercise.getProgrammingLanguage() + " is not supported"); } } + private List getJavaKotlinTestResultPaths(ProgrammingExercise programmingExercise) { + List testResultPaths = new ArrayList<>(); + if (ProjectType.isMavenProject(programmingExercise.getProjectType())) { + if (programmingExercise.hasSequentialTestRuns()) { + testResultPaths.add("/repositories/test-repository/structural/target/surefire-reports"); + testResultPaths.add("/repositories/test-repository/behavior/target/surefire-reports"); + } + else { + testResultPaths.add("/repositories/test-repository/target/surefire-reports"); + } + } + else { + if (programmingExercise.hasSequentialTestRuns()) { + testResultPaths.add("/repositories/test-repository/build/test-results/behaviorTests"); + testResultPaths.add("/repositories/test-repository/build/test-results/structuralTests"); + } + else { + testResultPaths.add("/repositories/test-repository/build/test-results/test"); + } + } + return testResultPaths; + } + + private List getPythonTestResultPaths() { + List testResultPaths = new ArrayList<>(); + testResultPaths.add("/repositories/test-repository/test-reports"); + return testResultPaths; + } + private LocalCIBuildResult parseTestResults(List testResultsTarInputStreams, String assignmentRepoBranchName, String assignmentRepoCommitHash, String testsRepoCommitHash, ZonedDateTime buildCompletedDate) throws IOException, XMLStreamException { @@ -330,7 +344,8 @@ private boolean isValidTestResultFile(TarArchiveEntry tarArchiveEntry) { int lastIndexOfSlash = name.lastIndexOf('/'); String result = (lastIndexOfSlash != -1 && lastIndexOfSlash + 1 < name.length()) ? name.substring(lastIndexOfSlash + 1) : name; - return !tarArchiveEntry.isDirectory() && result.endsWith(".xml") && result.startsWith("TEST-"); + // Java test result files are named "TEST-*.xml", Python test result files are named "*results.xml". + return !tarArchiveEntry.isDirectory() && ((result.endsWith(".xml") && result.startsWith("TEST-")) || result.endsWith("results.xml")); } private String readTarEntryContent(TarArchiveInputStream tarArchiveInputStream) throws IOException { @@ -356,6 +371,10 @@ private void processTestResultFile(String testResultFileString, List addBuildJobToQueue(ProgrammingExerc ProgrammingExercise programmingExercise = participation.getProgrammingExercise(); - List supportedProjectTypes = localCIProgrammingLanguageFeatureService.getProgrammingLanguageFeatures(programmingExercise.getProgrammingLanguage()) - .projectTypes(); + ProgrammingLanguage programmingLanguage = programmingExercise.getProgrammingLanguage(); + + ProjectType projectType = programmingExercise.getProjectType(); + + String dockerImage = programmingLanguageConfiguration.getImage(programmingLanguage, Optional.ofNullable(projectType)); - var projectType = programmingExercise.getProjectType(); + List supportedProjectTypes = localCIProgrammingLanguageFeatureService.getProgrammingLanguageFeatures(programmingLanguage).projectTypes(); if (projectType != null && !supportedProjectTypes.contains(programmingExercise.getProjectType())) { throw new LocalCIException("The project type " + programmingExercise.getProjectType() + " is not supported by the local CI."); @@ -95,7 +102,7 @@ public CompletableFuture addBuildJobToQueue(ProgrammingExerc String containerName = "artemis-local-ci-" + participation.getId() + "-" + ZonedDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmssSSS")); // Prepare a Callable that will later be called. It contains the actual steps needed to execute the build job. - Callable buildJob = () -> localCIBuildJobExecutionService.runBuildJob(participation, commitHash, containerName); + Callable buildJob = () -> localCIBuildJobExecutionService.runBuildJob(participation, commitHash, containerName, dockerImage); // Wrap the buildJob Callable in a BuildJobTimeoutCallable, so that the build job is cancelled if it takes too long. BuildJobTimeoutCallable timedBuildJob = new BuildJobTimeoutCallable<>(buildJob, timeoutSeconds); diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIContainerService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIContainerService.java index 0732e58357ec..e83d5387126d 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIContainerService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIContainerService.java @@ -66,10 +66,11 @@ public LocalCIContainerService(DockerClient dockerClient) { * @param containerName the name of the container to be created * @param branch the branch to checkout * @param commitHash the commit hash to checkout. If it is null, the latest commit of the branch will be checked out. + * @param image the Docker image to use for the container * @return {@link CreateContainerResponse} that can be used to start the container */ - public CreateContainerResponse configureContainer(String containerName, String branch, String commitHash) { - return dockerClient.createContainerCmd(dockerImage).withName(containerName).withHostConfig(HostConfig.newHostConfig().withAutoRemove(true)) + public CreateContainerResponse configureContainer(String containerName, String branch, String commitHash, String image) { + return dockerClient.createContainerCmd(image).withName(containerName).withHostConfig(HostConfig.newHostConfig().withAutoRemove(true)) .withEnv("ARTEMIS_BUILD_TOOL=gradle", "ARTEMIS_DEFAULT_BRANCH=" + branch, "ARTEMIS_ASSIGNMENT_REPOSITORY_COMMIT_HASH=" + (commitHash != null ? commitHash : "")) // Command to run when the container starts. This is the command that will be executed in the container's main process, which runs in the foreground and blocks the // container from exiting until it finishes. @@ -307,45 +308,64 @@ public Path createBuildScript(ProgrammingExercise programmingExercise, List scriptForJavaKotlin(programmingExercise, buildScript, hasSequentialTestRuns); + case PYTHON -> scriptForPython(buildScript); default -> throw new IllegalArgumentException("No build stage setup for programming language " + programmingExercise.getProgrammingLanguage()); } @@ -359,6 +379,23 @@ public Path createBuildScript(ProgrammingExercise programmingExercise, List auxiliaryRepositories) { + StringBuilder buildScript = new StringBuilder(); + for (AuxiliaryRepository auxiliaryRepository : auxiliaryRepositories) { + buildScript.append(" git clone --depth 1 --branch $ARTEMIS_DEFAULT_BRANCH file:///").append(auxiliaryRepository.getName()).append("-repository\n"); + } + return buildScript; + } + + private StringBuilder copyAuxiliaryRepositories(List auxiliaryRepositories, String source) { + StringBuilder buildScript = new StringBuilder(); + for (AuxiliaryRepository auxiliaryRepository : auxiliaryRepositories) { + buildScript.append(" cp -a ").append(source).append(auxiliaryRepository.getName()).append("-repository/. /repositories/test-repository/") + .append(auxiliaryRepository.getCheckoutDirectory()).append("/\n"); + } + return buildScript; + } + private void scriptForJavaKotlin(ProgrammingExercise programmingExercise, StringBuilder buildScript, boolean hasSequentialTestRuns) { boolean isMaven = ProjectType.isMavenProject(programmingExercise.getProjectType()); @@ -398,6 +435,18 @@ private void scriptForJavaKotlin(ProgrammingExercise programmingExercise, String } } + private void scriptForPython(StringBuilder buildScript) { + buildScript.append(""" + python3 -m compileall . -q || error=true + if [ ! $error ] + then + pytest --junitxml=test-reports/results.xml + else + exit 1 + fi + """); + } + /** * Deletes the build script for a given programming exercise. * The build script is stored in a file in the local-ci-scripts directory. diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIProgrammingLanguageFeatureService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIProgrammingLanguageFeatureService.java index 3be1c6fca9c8..30603c6ff307 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIProgrammingLanguageFeatureService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIProgrammingLanguageFeatureService.java @@ -23,7 +23,7 @@ public LocalCIProgrammingLanguageFeatureService() { // TODO LOCALVC_CI: Local CI is not supporting EMPTY at the moment. programmingLanguageFeatures.put(JAVA, new ProgrammingLanguageFeature(JAVA, true, false, true, true, false, List.of(PLAIN_GRADLE, GRADLE_GRADLE, PLAIN_MAVEN, MAVEN_MAVEN), false, false, true)); - // TODO LOCALVC_CI: Local CI is not supporting Python at the moment. + programmingLanguageFeatures.put(PYTHON, new ProgrammingLanguageFeature(PYTHON, false, false, true, false, false, List.of(), false, false, true)); // TODO LOCALVC_CI: Local CI is not supporting C at the moment. // TODO LOCALVC_CI: Local CI is not supporting Haskell at the moment. // TODO LOCALVC_CI: Local CI is not supporting Kotlin at the moment. diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIService.java index e8306d789aa7..ee4f0e261052 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIService.java @@ -5,12 +5,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; +import de.tum.in.www1.artemis.config.ProgrammingLanguageConfiguration; import de.tum.in.www1.artemis.domain.ProgrammingExercise; import de.tum.in.www1.artemis.domain.VcsRepositoryUrl; import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseParticipation; @@ -38,20 +38,22 @@ public class LocalCIService extends AbstractContinuousIntegrationService { private final LocalCIDockerService localCIDockerService; - @Value("${artemis.continuous-integration.build.images.java.default}") - String dockerImage; + private final ProgrammingLanguageConfiguration programmingLanguageConfiguration; public LocalCIService(ProgrammingSubmissionRepository programmingSubmissionRepository, FeedbackRepository feedbackRepository, BuildLogEntryService buildLogService, - BuildLogStatisticsEntryRepository buildLogStatisticsEntryRepository, TestwiseCoverageService testwiseCoverageService, LocalCIDockerService localCIDockerService) { + BuildLogStatisticsEntryRepository buildLogStatisticsEntryRepository, TestwiseCoverageService testwiseCoverageService, LocalCIDockerService localCIDockerService, + ProgrammingLanguageConfiguration programmingLanguageConfiguration) { super(programmingSubmissionRepository, feedbackRepository, buildLogService, buildLogStatisticsEntryRepository, testwiseCoverageService); this.localCIDockerService = localCIDockerService; + this.programmingLanguageConfiguration = programmingLanguageConfiguration; } @Override public void createBuildPlanForExercise(ProgrammingExercise programmingExercise, String planKey, VcsRepositoryUrl sourceCodeRepositoryURL, VcsRepositoryUrl testRepositoryURL, VcsRepositoryUrl solutionRepositoryURL) { // Only check whether the docker image needed for the build plan exists. - localCIDockerService.pullDockerImage(dockerImage); + localCIDockerService.pullDockerImage( + programmingLanguageConfiguration.getImage(programmingExercise.getProgrammingLanguage(), Optional.ofNullable(programmingExercise.getProjectType()))); } @Override diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/localvc/LocalVCRepositoryUrl.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/localvc/LocalVCRepositoryUrl.java index d9a3067f46c6..68cacd5648a0 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/localvc/LocalVCRepositoryUrl.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/localvc/LocalVCRepositoryUrl.java @@ -151,4 +151,14 @@ public boolean isPracticeRepository() { public Path getLocalRepositoryPath(String localVCBasePath) { return Paths.get(localVCBasePath, projectKey, repositorySlug + ".git"); } + + /** + * Get the path to the cloned repository + * + * @param baseRepoClonePath the base path of the cloned repositories + * @return the path to the cloned repository + */ + public Path getRepoClonePath(String baseRepoClonePath) { + return Paths.get(baseRepoClonePath, "git", projectKey, repositorySlug); + } }