From f3967d973a65d6c95cbdfeca8c5da0031ada32d4 Mon Sep 17 00:00:00 2001 From: Stephan Krusche Date: Tue, 12 Sep 2023 18:38:41 +0200 Subject: [PATCH 01/15] Programming exercises: Fix issues with binary files when importing and starting programming exercises (#7178) --- .../artemis/aop/logging/LoggingAspect.java | 8 +- .../tum/in/www1/artemis/domain/UserGroup.java | 27 ++++- .../in/www1/artemis/service/FileService.java | 106 +++++++++++------- .../vcs/AbstractVersionControlService.java | 31 ++++- .../ProgrammingExerciseExportService.java | 4 +- ...grammingExerciseImportFromFileService.java | 5 +- .../ProgrammingExerciseImportService.java | 21 ++-- .../ProgrammingExerciseRepositoryService.java | 2 +- .../www1/artemis/service/FileServiceTest.java | 16 +-- 9 files changed, 148 insertions(+), 72 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/aop/logging/LoggingAspect.java b/src/main/java/de/tum/in/www1/artemis/aop/logging/LoggingAspect.java index 1fe5d3b127fb..b78cc86e65cb 100644 --- a/src/main/java/de/tum/in/www1/artemis/aop/logging/LoggingAspect.java +++ b/src/main/java/de/tum/in/www1/artemis/aop/logging/LoggingAspect.java @@ -10,6 +10,7 @@ import org.springframework.core.env.Environment; import org.springframework.core.env.Profiles; +import de.tum.in.www1.artemis.service.connectors.vcs.AbstractVersionControlService; import tech.jhipster.config.JHipsterConstants; /** @@ -53,10 +54,14 @@ public void applicationPackagePointcut() { */ @AfterThrowing(pointcut = "applicationPackagePointcut() && springBeanPointcut()", throwing = "e") public void logAfterThrowing(JoinPoint joinPoint, Throwable e) { + if (AbstractVersionControlService.isReadFullyShortReadOfBlockException(e)) { + // ignore + return; + } + if (env.acceptsProfiles(Profiles.of(JHipsterConstants.SPRING_PROFILE_DEVELOPMENT))) { log.error("Exception in {}.{}() with cause = \'{}\' and exception = \'{}\'", joinPoint.getSignature().getDeclaringTypeName(), joinPoint.getSignature().getName(), e.getCause() != null ? e.getCause() : "NULL", e.getMessage(), e); - } else { log.error("Exception in {}.{}() with cause = {}", joinPoint.getSignature().getDeclaringTypeName(), joinPoint.getSignature().getName(), @@ -86,7 +91,6 @@ public Object logAround(ProceedingJoinPoint joinPoint) throws Throwable { } catch (IllegalArgumentException e) { log.error("Illegal argument: {} in {}.{}()", Arrays.toString(joinPoint.getArgs()), joinPoint.getSignature().getDeclaringTypeName(), joinPoint.getSignature().getName()); - throw e; } } diff --git a/src/main/java/de/tum/in/www1/artemis/domain/UserGroup.java b/src/main/java/de/tum/in/www1/artemis/domain/UserGroup.java index 091be72ed18e..ebb09576e010 100644 --- a/src/main/java/de/tum/in/www1/artemis/domain/UserGroup.java +++ b/src/main/java/de/tum/in/www1/artemis/domain/UserGroup.java @@ -1,6 +1,7 @@ package de.tum.in.www1.artemis.domain; import java.io.Serializable; +import java.util.Objects; import javax.persistence.*; @@ -18,12 +19,36 @@ public class UserGroup { private String group; @Embeddable - public class UserGroupKey implements Serializable { + public static class UserGroupKey implements Serializable { @Column(name = "user_id") private Long userId; @Column(name = "`groups`") private String group; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + UserGroupKey that = (UserGroupKey) o; + + if (!Objects.equals(userId, that.userId)) { + return false; + } + return Objects.equals(group, that.group); + } + + @Override + public int hashCode() { + int result = userId != null ? userId.hashCode() : 0; + result = 31 * result + (group != null ? group.hashCode() : 0); + return result; + } } } diff --git a/src/main/java/de/tum/in/www1/artemis/service/FileService.java b/src/main/java/de/tum/in/www1/artemis/service/FileService.java index 9e7ff41a1af9..84877534deb6 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/FileService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/FileService.java @@ -1,11 +1,11 @@ package de.tum.in.www1.artemis.service; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import java.io.*; import java.net.URLDecoder; import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -59,6 +59,13 @@ public class FileService implements DisposableBean { private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors()); + /** + * A list of common binary file extensions. + * Extensions must be lower-case without leading dots. + */ + private static final Set binaryFileExtensions = Set.of("png", "jpg", "jpeg", "heic", "gif", "tiff", "psd", "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "pages", + "numbers", "key", "odt", "zip", "rar", "7z", "tar", "iso", "mdb", "sqlite", "exe", "jar"); + /** * The list of file extensions that are allowed to be uploaded in a Markdown editor. * Extensions must be lower-case without leading dots. @@ -631,7 +638,7 @@ private Path getTargetPath(final Resource resource, final Path prefix, final Pat filePath = resource.getFile().toPath(); } else { - final String url = URLDecoder.decode(resource.getURL().toString(), StandardCharsets.UTF_8); + final String url = URLDecoder.decode(resource.getURL().toString(), UTF_8); filePath = Path.of(url); } @@ -738,7 +745,7 @@ public void replacePlaceholderSections(String filePath, Map sec throw new FilePathParsingException("File " + filePath + " should be updated but does not exist."); } - try (var reader = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8)); var writer = new BufferedWriter(new FileWriter(tempFile, StandardCharsets.UTF_8))) { + try (var reader = new BufferedReader(new FileReader(file, UTF_8)); var writer = new BufferedWriter(new FileWriter(tempFile, UTF_8))) { Map.Entry matchingStartPattern = null; String line = reader.readLine(); while (line != null) { @@ -856,29 +863,27 @@ public void replaceVariablesInFileName(String startPath, String targetString, St /** * This replaces all occurrences of the target Strings with the replacement Strings in the given file and saves the file *

- * {@link #replaceVariablesInFile(String, Map) replaceVariablesInFile} + * {@link #replaceVariablesInFile(Path, Map) replaceVariablesInFile} * * @param startPath the path where the start directory is located * @param replacements the replacements that should be applied - * @throws IOException if an issue occurs on file access for the replacement of the variables. */ - public void replaceVariablesInFileRecursive(String startPath, Map replacements) throws IOException { + public void replaceVariablesInFileRecursive(Path startPath, Map replacements) { replaceVariablesInFileRecursive(startPath, replacements, Collections.emptyList()); } /** * This replaces all occurrences of the target Strings with the replacement Strings in the given file and saves the file *

- * {@link #replaceVariablesInFile(String, Map) replaceVariablesInFile} + * {@link #replaceVariablesInFile(Path, Map) replaceVariablesInFile} * * @param startPath the path where the start directory is located * @param replacements the replacements that should be applied * @param filesToIgnore the name of files for which no replacement should be done - * @throws IOException if an issue occurs on file access for the replacement of the variables. */ - public void replaceVariablesInFileRecursive(String startPath, Map replacements, List filesToIgnore) throws IOException { + public void replaceVariablesInFileRecursive(Path startPath, Map replacements, List filesToIgnore) { log.debug("Replacing {} in files in directory {}", replacements, startPath); - File directory = new File(startPath); + File directory = startPath.toFile(); if (!directory.exists() || !directory.isDirectory()) { throw new RuntimeException("Files in directory " + startPath + " should be replaced but the directory does not exist."); } @@ -889,7 +894,7 @@ public void replaceVariablesInFileRecursive(String startPath, Map replacements) throws IOException { + public void replaceVariablesInFile(Path filePath, Map replacements) { log.debug("Replacing {} in file {}", replacements, filePath); - // https://stackoverflow.com/questions/3935791/find-and-replace-words-lines-in-a-file - Path replaceFilePath = Path.of(filePath); - Charset charset = StandardCharsets.UTF_8; - String fileContent = Files.readString(replaceFilePath, charset); - for (Map.Entry replacement : replacements.entrySet()) { - fileContent = fileContent.replace(replacement.getKey(), replacement.getValue()); + if (isBinaryFile(filePath)) { + // do not try to read binary files with 'readString' + return; + } + try { + // Note: Java does not offer a good way to check if a file is binary or not. If the basic check above fails (e.g. due to a custom binary file from an instructor), + // but the file is still binary, we try to read it. In case the method readString fails, we only log this below, but continue, because the exception should NOT + // interrupt the ongoing process + String fileContent = Files.readString(filePath, UTF_8); + for (Map.Entry replacement : replacements.entrySet()) { + fileContent = fileContent.replace(replacement.getKey(), replacement.getValue()); + } + Files.writeString(filePath, fileContent, UTF_8); } - Files.writeString(replaceFilePath, fileContent, charset); + catch (IOException ex) { + log.warn("Exception {} occurred when trying to replace {} in (binary) file {}", ex.getMessage(), replacements, filePath); + // continue + } + } + + /** + * very simple and non-exhaustive check for the most common binary files such as images + * Unfortunately, Java cannot determine this correctly, so we need to provide typical file endings here + * + * @param filePath the path of the file + * @return whether the simple check for file endings determines the underlying file to be binary (true) or not (false) + */ + private static boolean isBinaryFile(Path filePath) { + final String fileExtension = FilenameUtils.getExtension(filePath.getFileName().toString()); + return binaryFileExtensions.stream().anyMatch(fileExtension::equalsIgnoreCase); } /** * This normalizes all line endings to UNIX-line-endings recursively from the startPath. *

- * {@link #normalizeLineEndings(String) normalizeLineEndings} + * {@link #normalizeLineEndings(Path) normalizeLineEndings} * * @param startPath the path where the start directory is located * @throws IOException if an issue occurs on file access for the normalizing of the line endings. */ - public void normalizeLineEndingsDirectory(String startPath) throws IOException { + public void normalizeLineEndingsDirectory(Path startPath) throws IOException { log.debug("Normalizing file endings in directory {}", startPath); - File directory = new File(startPath); + File directory = startPath.toFile(); if (!directory.exists() || !directory.isDirectory()) { throw new RuntimeException("File endings in directory " + startPath + " should be normalized but the directory does not exist."); } @@ -948,7 +974,7 @@ public void normalizeLineEndingsDirectory(String startPath) throws IOException { Collection files = FileUtils.listFiles(directory, FileFilterUtils.trueFileFilter(), directoryFileFilter); for (File file : files) { - normalizeLineEndings(file.getAbsolutePath()); + normalizeLineEndings(file.toPath()); } } @@ -960,28 +986,29 @@ public void normalizeLineEndingsDirectory(String startPath) throws IOException { * @param filePath the path where the file is located * @throws IOException if an issue occurs on file access for the normalizing of the line endings. */ - public void normalizeLineEndings(String filePath) throws IOException { + public void normalizeLineEndings(Path filePath) throws IOException { log.debug("Normalizing line endings in file {}", filePath); + if (isBinaryFile(filePath)) { + // do not try to read binary files with 'readString' + return; + } // https://stackoverflow.com/questions/3776923/how-can-i-normalize-the-eol-character-in-java - Path replaceFilePath = Path.of(filePath); - Charset charset = StandardCharsets.UTF_8; - - String fileContent = Files.readString(replaceFilePath, charset); + String fileContent = Files.readString(filePath, UTF_8); fileContent = fileContent.replaceAll("\\r\\n?", "\n"); - Files.writeString(replaceFilePath, fileContent, charset); + Files.writeString(filePath, fileContent, UTF_8); } /** * This converts all files to the UTF-8 encoding recursively from the startPath. *

- * {@link #convertToUTF8(String) convertToUTF8} + * {@link #convertToUTF8(Path) convertToUTF8} * * @param startPath the path where the start directory is located * @throws IOException if an issue occurs on file access when converting to UTF-8. */ - public void convertToUTF8Directory(String startPath) throws IOException { + public void convertToUTF8Directory(Path startPath) throws IOException { log.debug("Converting files in directory {} to UTF-8", startPath); - File directory = new File(startPath); + File directory = startPath.toFile(); if (!directory.exists() || !directory.isDirectory()) { throw new RuntimeException("Files in directory " + startPath + " should be converted to UTF-8 but the directory does not exist."); } @@ -992,7 +1019,7 @@ public void convertToUTF8Directory(String startPath) throws IOException { Collection files = FileUtils.listFiles(directory, FileFilterUtils.trueFileFilter(), directoryFileFilter); for (File file : files) { - convertToUTF8(file.getAbsolutePath()); + convertToUTF8(file.toPath()); } } @@ -1003,17 +1030,16 @@ public void convertToUTF8Directory(String startPath) throws IOException { * @param filePath the path where the file is located * @throws IOException if an issue occurs on file access when converting to UTF-8. */ - public void convertToUTF8(String filePath) throws IOException { + public void convertToUTF8(Path filePath) throws IOException { log.debug("Converting file {} to UTF-8", filePath); - Path replaceFilePath = Path.of(filePath); - byte[] contentArray = Files.readAllBytes(replaceFilePath); + byte[] contentArray = Files.readAllBytes(filePath); Charset charset = detectCharset(contentArray); log.debug("Detected charset for file {} is {}", filePath, charset.name()); String fileContent = new String(contentArray, charset); - Files.writeString(replaceFilePath, fileContent, StandardCharsets.UTF_8); + Files.writeString(filePath, fileContent, UTF_8); } /** @@ -1132,7 +1158,7 @@ public void createDirectory(Path path) { * @return Path to the written file */ public Path writeStringToFile(String stringToWrite, Path path) { - try (var outStream = new OutputStreamWriter(new FileOutputStream(path.toString()), StandardCharsets.UTF_8)) { + try (var outStream = new OutputStreamWriter(new FileOutputStream(path.toString()), UTF_8)) { outStream.write(stringToWrite); } catch (IOException e) { diff --git a/src/main/java/de/tum/in/www1/artemis/service/connectors/vcs/AbstractVersionControlService.java b/src/main/java/de/tum/in/www1/artemis/service/connectors/vcs/AbstractVersionControlService.java index e4810c43451d..8523a53018e6 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/connectors/vcs/AbstractVersionControlService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/connectors/vcs/AbstractVersionControlService.java @@ -1,12 +1,12 @@ package de.tum.in.www1.artemis.service.connectors.vcs; -import static de.tum.in.www1.artemis.config.Constants.*; - import java.io.IOException; import java.nio.file.Path; +import java.util.Objects; import org.apache.commons.io.FileUtils; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.internal.JGitText; import org.hibernate.Hibernate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,8 +123,15 @@ public VcsRepositoryUrl copyRepository(String sourceProjectKey, String sourceRep // copy by pushing the source's content to the target's repo gitService.pushSourceToTargetRepo(targetRepo, targetRepoUrl, sourceBranch); } - catch (GitAPIException | VersionControlException e) { + catch (GitAPIException | VersionControlException ex) { + if (isReadFullyShortReadOfBlockException(ex)) { + // NOTE: we ignore this particular error: it sometimes happens when pushing code that includes binary files, however the push operation typically worked correctly + // TODO: verify that the push operation actually worked correctly, e.g. by comparing the number of commits in the source and target repo + log.warn("TransportException/EOFException with 'Short read of block' when copying repository {} to {}. Will ignore it", sourceRepoUrl, targetRepoUrl); + return targetRepoUrl; + } Path localPath = gitService.getDefaultLocalPathOfRepo(targetRepoUrl); + // clean up in case of an error try { if (targetRepo != null) { // delete the target repo if an error occurs @@ -135,16 +142,30 @@ public VcsRepositoryUrl copyRepository(String sourceProjectKey, String sourceRep FileUtils.deleteDirectory(localPath.toFile()); } } - catch (IOException ex) { + catch (IOException ioException) { // ignore log.error("Could not delete directory of the failed cloned repository in: {}", localPath); } - throw new VersionControlException("Could not copy repository " + sourceRepositoryName + " to the target repository " + targetRepositoryName, e); + throw new VersionControlException("Could not copy repository " + sourceRepositoryName + " to the target repository " + targetRepositoryName, ex); } return targetRepoUrl; } + /** + * checks for a specific exception that we would like to ignore + * + * @param ex the exception + * @return whether we found the specific one or not + */ + public static boolean isReadFullyShortReadOfBlockException(Throwable ex) { + return ex instanceof org.eclipse.jgit.api.errors.TransportException transportException + && transportException.getCause() instanceof org.eclipse.jgit.errors.TransportException innerTransportException + && innerTransportException.getCause() instanceof java.io.EOFException eofException && eofException.getMessage().equals(JGitText.get().shortReadOfBlock) + && Objects.equals(eofException.getStackTrace()[0].getClassName(), "org.eclipse.jgit.util.IO") + && Objects.equals(eofException.getStackTrace()[0].getMethodName(), "readFully"); + } + @Override public String getOrRetrieveBranchOfParticipation(ProgrammingExerciseParticipation participation) { if (participation instanceof ProgrammingExerciseStudentParticipation studentParticipation) { diff --git a/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseExportService.java b/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseExportService.java index 74326f490bc0..ac7bed156470 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseExportService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseExportService.java @@ -670,8 +670,8 @@ private Path createZipForRepositoryWithParticipation(final ProgrammingExercise p if (repositoryExportOptions.isNormalizeCodeStyle()) { try { log.debug("Normalizing code style for participation {}", participation); - fileService.normalizeLineEndingsDirectory(repository.getLocalPath().toString()); - fileService.convertToUTF8Directory(repository.getLocalPath().toString()); + fileService.normalizeLineEndingsDirectory(repository.getLocalPath()); + fileService.convertToUTF8Directory(repository.getLocalPath()); } catch (IOException ex) { log.warn("Cannot normalize code style in the repository {} due to the following exception: {}", repository.getLocalPath(), ex.getMessage()); diff --git a/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseImportFromFileService.java b/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseImportFromFileService.java index 2151f85aff20..61dce26dc239 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseImportFromFileService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseImportFromFileService.java @@ -78,6 +78,7 @@ public ProgrammingExercise importProgrammingExerciseFromFile(ProgrammingExercise checkRepositoriesExist(importExerciseDir); var oldShortName = getProgrammingExerciseFromDetailsFile(importExerciseDir).getShortName(); programmingExerciseService.validateNewProgrammingExerciseSettings(programmingExerciseForImport, course); + // TODO: creating the whole exercise (from template) is a bad solution in this case, we do not want the template content, instead we want the file content of the zip importedProgrammingExercise = programmingExerciseService.createProgrammingExercise(programmingExerciseForImport); if (Boolean.TRUE.equals(programmingExerciseForImport.isStaticCodeAnalysisEnabled())) { staticCodeAnalysisService.createDefaultCategories(importedProgrammingExercise); @@ -131,9 +132,9 @@ private void importRepositoriesFromFile(ProgrammingExercise newExercise, Path ba gitService.commitAndPush(testRepo, "Import tests from file", true, null); } - private void replaceImportedExerciseShortName(Map replacements, Repository... repositories) throws IOException { + private void replaceImportedExerciseShortName(Map replacements, Repository... repositories) { for (Repository repository : repositories) { - fileService.replaceVariablesInFileRecursive(repository.getLocalPath().toString(), replacements, SHORT_NAME_REPLACEMENT_EXCLUSIONS); + fileService.replaceVariablesInFileRecursive(repository.getLocalPath(), replacements, SHORT_NAME_REPLACEMENT_EXCLUSIONS); } } diff --git a/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseImportService.java b/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseImportService.java index 18484590eded..f3a347493af8 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseImportService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseImportService.java @@ -95,18 +95,18 @@ public void importRepositories(final ProgrammingExercise templateExercise, final String sourceBranch = versionControl.getOrRetrieveBranchOfExercise(templateExercise); + // TODO: in case one of those operations fail, we should do error handling and revert all previous operations versionControl.copyRepository(sourceProjectKey, templateRepoName, sourceBranch, targetProjectKey, RepositoryType.TEMPLATE.getName()); versionControl.copyRepository(sourceProjectKey, solutionRepoName, sourceBranch, targetProjectKey, RepositoryType.SOLUTION.getName()); versionControl.copyRepository(sourceProjectKey, testRepoName, sourceBranch, targetProjectKey, RepositoryType.TESTS.getName()); - List auxiliaryRepositories = templateExercise.getAuxiliaryRepositories(); - for (int i = 0; i < auxiliaryRepositories.size(); i++) { - AuxiliaryRepository auxiliaryRepository = auxiliaryRepositories.get(i); - String repositoryUrl = versionControl - .copyRepository(sourceProjectKey, auxiliaryRepository.getRepositoryName(), sourceBranch, targetProjectKey, auxiliaryRepository.getName()).toString(); - AuxiliaryRepository newAuxiliaryRepository = newExercise.getAuxiliaryRepositories().get(i); - newAuxiliaryRepository.setRepositoryUrl(repositoryUrl); - auxiliaryRepositoryRepository.save(newAuxiliaryRepository); + List auxRepos = templateExercise.getAuxiliaryRepositories(); + for (int i = 0; i < auxRepos.size(); i++) { + AuxiliaryRepository auxRepo = auxRepos.get(i); + var repoUrl = versionControl.copyRepository(sourceProjectKey, auxRepo.getRepositoryName(), sourceBranch, targetProjectKey, auxRepo.getName()).toString(); + AuxiliaryRepository newAuxRepo = newExercise.getAuxiliaryRepositories().get(i); + newAuxRepo.setRepositoryUrl(repoUrl); + auxiliaryRepositoryRepository.save(newAuxRepo); } // Unprotect the default branch of the template exercise repo. @@ -255,12 +255,11 @@ private void adjustProjectNames(ProgrammingExercise templateExercise, Programmin * @param repositoryName the name of the repository that should be adjusted * @param user the user which performed the action (used as Git author) * @throws GitAPIException If the checkout/push of one repository fails - * @throws IOException If the values in the files could not be replaced */ - private void adjustProjectName(Map replacements, String projectKey, String repositoryName, User user) throws GitAPIException, IOException { + private void adjustProjectName(Map replacements, String projectKey, String repositoryName, User user) throws GitAPIException { final var repositoryUrl = versionControlService.orElseThrow().getCloneRepositoryUrl(projectKey, repositoryName); Repository repository = gitService.getOrCheckoutRepository(repositoryUrl, true); - fileService.replaceVariablesInFileRecursive(repository.getLocalPath().toAbsolutePath().toString(), replacements, List.of("gradle-wrapper.jar")); + fileService.replaceVariablesInFileRecursive(repository.getLocalPath().toAbsolutePath(), replacements, List.of("gradle-wrapper.jar")); gitService.stageAllChanges(repository); gitService.commitAndPush(repository, "Template adjusted by Artemis", true, user); repository.setFiles(null); // Clear cache to avoid multiple commits when Artemis server is not restarted between attempts diff --git a/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseRepositoryService.java b/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseRepositoryService.java index 4bc551968cbc..688cacd177a0 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseRepositoryService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseRepositoryService.java @@ -580,7 +580,7 @@ void replacePlaceholders(final ProgrammingExercise programmingExercise, final Re replacements.put("${exerciseName}", programmingExercise.getTitle()); replacements.put("${studentWorkingDirectory}", Constants.STUDENT_WORKING_DIRECTORY); replacements.put("${packaging}", programmingExercise.hasSequentialTestRuns() ? "pom" : "jar"); - fileService.replaceVariablesInFileRecursive(repository.getLocalPath().toAbsolutePath().toString(), replacements, List.of("gradle-wrapper.jar")); + fileService.replaceVariablesInFileRecursive(repository.getLocalPath().toAbsolutePath(), replacements, List.of("gradle-wrapper.jar")); } /** diff --git a/src/test/java/de/tum/in/www1/artemis/service/FileServiceTest.java b/src/test/java/de/tum/in/www1/artemis/service/FileServiceTest.java index c49477ca5012..30a3dc719c1d 100644 --- a/src/test/java/de/tum/in/www1/artemis/service/FileServiceTest.java +++ b/src/test/java/de/tum/in/www1/artemis/service/FileServiceTest.java @@ -104,7 +104,7 @@ void normalizeFileEndingsUnix_normalized() throws IOException { int size = Files.readAllBytes(Path.of(".", "exportTest", "LineEndingsUnix.java")).length; assertThat(size).isEqualTo(129); - fileService.normalizeLineEndings(Path.of(".", "exportTest", "LineEndingsUnix.java").toString()); + fileService.normalizeLineEndings(Path.of(".", "exportTest", "LineEndingsUnix.java")); size = Files.readAllBytes(Path.of(".", "exportTest", "LineEndingsUnix.java")).length; assertThat(size).isEqualTo(129); } @@ -122,7 +122,7 @@ void normalizeFileEndingsWindows_normalized() throws IOException { int size = Files.readAllBytes(Path.of(".", "exportTest", "LineEndingsWindows.java")).length; assertThat(size).isEqualTo(136); - fileService.normalizeLineEndings(Path.of(".", "exportTest", "LineEndingsWindows.java").toString()); + fileService.normalizeLineEndings(Path.of(".", "exportTest", "LineEndingsWindows.java")); size = Files.readAllBytes(Path.of(".", "exportTest", "LineEndingsWindows.java")).length; assertThat(size).isEqualTo(129); } @@ -140,7 +140,7 @@ void normalizeEncodingISO_8559_1() throws IOException { Charset charset = fileService.detectCharset(Files.readAllBytes(Path.of(".", "exportTest", "EncodingISO_8559_1.java"))); assertThat(charset).isEqualTo(StandardCharsets.ISO_8859_1); - fileService.convertToUTF8(Path.of(".", "exportTest", "EncodingISO_8559_1.java").toString()); + fileService.convertToUTF8(Path.of(".", "exportTest", "EncodingISO_8559_1.java")); charset = fileService.detectCharset(Files.readAllBytes(Path.of(".", "exportTest", "EncodingISO_8559_1.java"))); assertThat(charset).isEqualTo(StandardCharsets.UTF_8); } @@ -156,7 +156,7 @@ void replacePlaceHolder() throws IOException { Map replacements = new HashMap<>(); replacements.put("${exerciseName}", "SomeCoolExerciseName"); - fileService.replaceVariablesInFileRecursive(pomXml.getParent(), replacements); + fileService.replaceVariablesInFileRecursive(pomXml.getParentFile().toPath(), replacements); fileContent = FileUtils.readFileToString(pomXml, Charset.defaultCharset()); assertThat(fileContent).doesNotContain("${exerciseName}").contains("SomeCoolExerciseName"); @@ -173,7 +173,7 @@ void replacePlaceHolderIgnoreNames() throws IOException { Map replacements = new HashMap<>(); replacements.put("${exerciseName}", "SomeCoolExerciseName"); - fileService.replaceVariablesInFileRecursive(pomXml.getParent(), replacements, List.of("pom.xml")); + fileService.replaceVariablesInFileRecursive(pomXml.getParentFile().toPath(), replacements, List.of("pom.xml")); fileContent = FileUtils.readFileToString(pomXml, Charset.defaultCharset()); assertThat(fileContent).contains("${exerciseName}").doesNotContain("SomeCoolExerciseName"); @@ -288,19 +288,19 @@ void testPublicPathForActualPathOrThrow_shouldThrowException() { @Test void testReplaceVariablesInFileRecursive_shouldThrowException() { - assertThatRuntimeException().isThrownBy(() -> fileService.replaceVariablesInFileRecursive("some-path", new HashMap<>())) + assertThatRuntimeException().isThrownBy(() -> fileService.replaceVariablesInFileRecursive(Path.of("some-path"), new HashMap<>())) .withMessageEndingWith("should be replaced but the directory does not exist."); } @Test void testNormalizeLineEndingsDirectory_shouldThrowException() { - assertThatRuntimeException().isThrownBy(() -> fileService.normalizeLineEndingsDirectory("some-path")) + assertThatRuntimeException().isThrownBy(() -> fileService.normalizeLineEndingsDirectory(Path.of("some-path"))) .withMessageEndingWith("should be normalized but the directory does not exist."); } @Test void testConvertToUTF8Directory_shouldThrowException() { - assertThatRuntimeException().isThrownBy(() -> fileService.convertToUTF8Directory("some-path")) + assertThatRuntimeException().isThrownBy(() -> fileService.convertToUTF8Directory(Path.of("some-path"))) .withMessageEndingWith("should be converted to UTF-8 but the directory does not exist."); } From 64187ce9be0ca6492bc172afe9477ca6dd191357 Mon Sep 17 00:00:00 2001 From: Stephan Krusche Date: Tue, 12 Sep 2023 19:07:04 +0200 Subject: [PATCH 02/15] Development: Bump version to 6.5.0 --- build.gradle | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 27e7282568f4..19452c131ffa 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ plugins { } group = "de.tum.in.www1.artemis" -version = "6.4.3" +version = "6.5.0" description = "Interactive Learning with Individual Feedback" sourceCompatibility=17 diff --git a/package-lock.json b/package-lock.json index 426607e0d086..6ce1f04ecd2a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "artemis", - "version": "6.4.3", + "version": "6.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "artemis", - "version": "6.4.3", + "version": "6.5.0", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index c741d72fd202..33c99708233e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "artemis", - "version": "6.4.3", + "version": "6.5.0", "description": "Interactive Learning with Individual Feedback", "private": true, "license": "MIT", From dc35b8cc23c2bf6fdea2ea80784305c365c9bd61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 08:27:36 +0200 Subject: [PATCH 03/15] Development: Update version for docker/setup-qemu-action from 2 to 3 (#7183) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0837a57891cf..cb2cb27529f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -115,7 +115,7 @@ jobs: if: ${{ github.event_name == 'release' }} uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 # Build and Push to GitHub Container Registry From 4321727b537cf572df5b5935621c522602cb1c00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 08:27:54 +0200 Subject: [PATCH 04/15] Development: Update version for docker/setup-buildx-action from 2 to 3 (#7184) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cb2cb27529f4..8d07af38982d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -117,7 +117,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 # Build and Push to GitHub Container Registry - name: Login to GitHub Container Registry uses: docker/login-action@v2 From 4332d9e11655ed417350356ef0a3acf2c509df4a Mon Sep 17 00:00:00 2001 From: Florian Gareis Date: Wed, 13 Sep 2023 18:20:51 +0200 Subject: [PATCH 05/15] Development: Fix sorry cypress for cypress version 13 (#7191) --- src/test/cypress/cypress.config.ts | 10 ++++- src/test/cypress/package-lock.json | 71 ++++++++++++++++++++---------- src/test/cypress/package.json | 10 ++--- 3 files changed, 62 insertions(+), 29 deletions(-) diff --git a/src/test/cypress/cypress.config.ts b/src/test/cypress/cypress.config.ts index 50dce7d9019e..fa5c941d1de5 100644 --- a/src/test/cypress/cypress.config.ts +++ b/src/test/cypress/cypress.config.ts @@ -1,5 +1,6 @@ import { defineConfig } from 'cypress'; import { cloudPlugin } from 'cypress-cloud/plugin'; +import fs from 'fs'; export default defineConfig({ clientCertificates: [ @@ -38,7 +39,6 @@ export default defineConfig({ screenshotsFolder: 'screenshots', videosFolder: 'videos', video: true, - videoUploadOnPasses: false, screenshotOnRunFailure: true, viewportWidth: 1920, viewportHeight: 1080, @@ -65,6 +65,14 @@ export default defineConfig({ return null; }, }); + on('after:spec', (spec: Cypress.Spec, results: CypressCommandLine.RunResult) => { + if (results && results.video) { + const failures = results.tests.some((test) => test.attempts.some((attempt) => attempt.state === 'failed')); + if (!failures) { + fs.unlinkSync(results.video); + } + } + }); on('before:browser:launch', (browser, launchOptions) => { launchOptions.args.push('--lang=en'); return launchOptions; diff --git a/src/test/cypress/package-lock.json b/src/test/cypress/package-lock.json index f96cb4c9180c..632fff350ea3 100644 --- a/src/test/cypress/package-lock.json +++ b/src/test/cypress/package-lock.json @@ -8,13 +8,13 @@ "license": "MIT", "devDependencies": { "@4tw/cypress-drag-drop": "2.2.4", - "@types/node": "20.5.9", - "cypress": "13.1.0", - "cypress-cloud": "1.9.4", + "@types/node": "20.6.0", + "cypress": "13.2.0", + "cypress-cloud": "1.10.0-beta.4", "cypress-file-upload": "5.0.8", "cypress-wait-until": "2.0.1", "typescript": "5.2.2", - "uuid": "9.0.0", + "uuid": "9.0.1", "wait-on": "7.0.1" } }, @@ -301,9 +301,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.5.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.9.tgz", - "integrity": "sha512-PcGNd//40kHAS3sTlzKB9C9XL4K0sTup8nbG5lC14kzEteTNuAFh9u5nA0o5TWnSG2r/JNPRXFVcHJIIeRlmqQ==", + "version": "20.6.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.6.0.tgz", + "integrity": "sha512-najjVq5KN2vsH2U/xyh2opaSEz6cZMR2SetLIlxlj08nOcmPOemJmUK2o4kUzfLqfrWE0PIrNeE16XhYDd3nqg==", "dev": true }, "node_modules/@types/sinonjs__fake-timers": { @@ -904,15 +904,15 @@ } }, "node_modules/cypress": { - "version": "13.1.0", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.1.0.tgz", - "integrity": "sha512-LUKxCYlB973QBFls1Up4FAE9QIYobT+2I8NvvAwMfQS2YwsWbr6yx7y9hmsk97iqbHkKwZW3MRjoK1RToBFVdQ==", + "version": "13.2.0", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.2.0.tgz", + "integrity": "sha512-AvDQxBydE771GTq0TR4ZUBvv9m9ffXuB/ueEtpDF/6gOcvFR96amgwSJP16Yhqw6VhmwqspT5nAGzoxxB+D89g==", "dev": true, "hasInstallScript": true, "dependencies": { "@cypress/request": "^3.0.0", "@cypress/xvfb": "^1.2.4", - "@types/node": "^16.18.39", + "@types/node": "^18.17.5", "@types/sinonjs__fake-timers": "8.1.1", "@types/sizzle": "^2.3.2", "arch": "^2.2.0", @@ -962,9 +962,9 @@ } }, "node_modules/cypress-cloud": { - "version": "1.9.4", - "resolved": "https://registry.npmjs.org/cypress-cloud/-/cypress-cloud-1.9.4.tgz", - "integrity": "sha512-zItu3zTtSOFMfKExlqOrWXA8A3aI2fhKLGuVDDVYFtW+uthrSlImVfIl+Yj+KB8lnJmLR/+z94Q3GvCPHKQzxg==", + "version": "1.10.0-beta.4", + "resolved": "https://registry.npmjs.org/cypress-cloud/-/cypress-cloud-1.10.0-beta.4.tgz", + "integrity": "sha512-8pe+ifmf8Uotx9lVL4Crq9LQokAa8U6/09p+wj9XEZoEiY/FJhbbvOygduqZmF6BaQaDAf5q1DagKKHmLXGErA==", "dev": true, "dependencies": { "@cypress/commit-info": "^2.2.0", @@ -975,8 +975,10 @@ "commander": "^10.0.0", "common-path-prefix": "^3.0.0", "cy2": "^3.4.2", + "date-fns": "^2.30.0", "debug": "^4.3.4", "execa": "^5.1.1", + "fast-safe-stringify": "^2.1.1", "getos": "^3.2.1", "globby": "^11.1.0", "is-absolute": "^1.0.0", @@ -995,9 +997,6 @@ }, "engines": { "node": ">=14.7.0" - }, - "peerDependencies": { - "cypress": ">=10.0.0" } }, "node_modules/cypress-cloud/node_modules/commander": { @@ -1072,9 +1071,9 @@ "dev": true }, "node_modules/cypress/node_modules/@types/node": { - "version": "16.18.48", - "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.48.tgz", - "integrity": "sha512-mlaecDKQ7rIZrYD7iiKNdzFb6e/qD5I9U1rAhq+Fd+DWvYVs+G2kv74UFHmSOlg5+i/vF3XxuR522V4u8BqO+Q==", + "version": "18.17.15", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.15.tgz", + "integrity": "sha512-2yrWpBk32tvV/JAd3HNHWuZn/VDN1P+72hWirHnvsvTGSqbANi+kSeuQR9yAHnbvaBvHDsoTdXV0Fe+iRtHLKA==", "dev": true }, "node_modules/dashdash": { @@ -1089,6 +1088,22 @@ "node": ">=0.10" } }, + "node_modules/date-fns": { + "version": "2.30.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", + "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.21.0" + }, + "engines": { + "node": ">=0.11" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/date-fns" + } + }, "node_modules/dayjs": { "version": "1.11.9", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.9.tgz", @@ -1330,6 +1345,12 @@ "node": ">=8.6.0" } }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "dev": true + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -3036,10 +3057,14 @@ } }, "node_modules/uuid": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", - "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], "bin": { "uuid": "dist/bin/uuid" } diff --git a/src/test/cypress/package.json b/src/test/cypress/package.json index 53fb0f5c540d..12a8aded34da 100644 --- a/src/test/cypress/package.json +++ b/src/test/cypress/package.json @@ -8,13 +8,13 @@ ], "devDependencies": { "@4tw/cypress-drag-drop": "2.2.4", - "@types/node": "20.5.9", - "cypress": "13.1.0", - "cypress-cloud": "1.9.4", + "@types/node": "20.6.0", + "cypress": "13.2.0", + "cypress-cloud": "1.10.0-beta.4", "cypress-file-upload": "5.0.8", "cypress-wait-until": "2.0.1", "typescript": "5.2.2", - "uuid": "9.0.0", + "uuid": "9.0.1", "wait-on": "7.0.1" }, "overrides": { @@ -30,7 +30,7 @@ "cypress:open": "cypress open", "cypress:run": "cypress run --browser=chrome", "cypress:setup": "cypress install && cypress run --quiet --spec init/ImportUsers.cy.ts", - "cypress:record:mysql": "npx cypress-cloud run --parallel --record --ci-build-id \"${SORRY_CYPRESS_BRANCH_NAME} #${SORRY_CYPRESS_BUILD_ID} ${SORRY_CYPRESS_RERUN_COUNT} (MySQL)\"", + "cypress:record:mysql": "npx cypress-cloud run --cloud-debug --parallel --record --ci-build-id \"${SORRY_CYPRESS_BRANCH_NAME} #${SORRY_CYPRESS_BUILD_ID} ${SORRY_CYPRESS_RERUN_COUNT} (MySQL)\"", "cypress:record:postgres": "npx cypress-cloud run --parallel --record --ci-build-id \"${SORRY_CYPRESS_BRANCH_NAME} #${SORRY_CYPRESS_BUILD_ID} ${SORRY_CYPRESS_RERUN_COUNT} (Postgres)\"", "update": "npm-upgrade" } From a51749b743c9c5d06315a4feda9e056d10c43818 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Thu, 14 Sep 2023 10:11:36 +0200 Subject: [PATCH 06/15] General: Fix an issue on the user settings page (#7195) --- .../user-settings-container.component.html | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/webapp/app/shared/user-settings/user-settings-container/user-settings-container.component.html b/src/main/webapp/app/shared/user-settings/user-settings-container/user-settings-container.component.html index 732d8bee607c..e8363986a020 100644 --- a/src/main/webapp/app/shared/user-settings/user-settings-container/user-settings-container.component.html +++ b/src/main/webapp/app/shared/user-settings/user-settings-container/user-settings-container.component.html @@ -3,7 +3,6 @@

-
{{ currentUser.name }} From 1fa8b4ee7a699acb91be00e80785d22dde051ed1 Mon Sep 17 00:00:00 2001 From: Florian Gareis Date: Thu, 14 Sep 2023 10:12:46 +0200 Subject: [PATCH 07/15] Development: Fix cypress dependency issue (#7196) --- {.bamboo => .ci}/E2E-tests/cleanup.sh | 0 {.bamboo => .ci}/E2E-tests/execute.sh | 4 ++-- {.bamboo => .ci}/migration-check/cleanup.sh | 0 {.bamboo => .ci}/migration-check/execute.sh | 4 ++-- docker/cypress-E2E-tests-mysql.yml | 2 +- docker/cypress-E2E-tests-postgres.yml | 2 +- src/test/cypress/package.json | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) rename {.bamboo => .ci}/E2E-tests/cleanup.sh (100%) rename {.bamboo => .ci}/E2E-tests/execute.sh (95%) rename {.bamboo => .ci}/migration-check/cleanup.sh (100%) rename {.bamboo => .ci}/migration-check/execute.sh (94%) diff --git a/.bamboo/E2E-tests/cleanup.sh b/.ci/E2E-tests/cleanup.sh similarity index 100% rename from .bamboo/E2E-tests/cleanup.sh rename to .ci/E2E-tests/cleanup.sh diff --git a/.bamboo/E2E-tests/execute.sh b/.ci/E2E-tests/execute.sh similarity index 95% rename from .bamboo/E2E-tests/execute.sh rename to .ci/E2E-tests/execute.sh index 92de1a959efd..971d05dea8a6 100755 --- a/.bamboo/E2E-tests/execute.sh +++ b/.ci/E2E-tests/execute.sh @@ -4,10 +4,10 @@ DB=$1 if [ "$DB" = "mysql" ]; then COMPOSE_FILE="cypress-E2E-tests-mysql.yml" -elif [ "$DB" = "postgresql" ]; then +elif [ "$DB" = "postgres" ]; then COMPOSE_FILE="cypress-E2E-tests-postgres.yml" else - echo "Invalid database type. Please choose either mysql or postgresql." + echo "Invalid database type. Please choose either mysql or postgres." exit 1 fi diff --git a/.bamboo/migration-check/cleanup.sh b/.ci/migration-check/cleanup.sh similarity index 100% rename from .bamboo/migration-check/cleanup.sh rename to .ci/migration-check/cleanup.sh diff --git a/.bamboo/migration-check/execute.sh b/.ci/migration-check/execute.sh similarity index 94% rename from .bamboo/migration-check/execute.sh rename to .ci/migration-check/execute.sh index 9b9ecdceb41f..50c42043f1ec 100755 --- a/.bamboo/migration-check/execute.sh +++ b/.ci/migration-check/execute.sh @@ -4,10 +4,10 @@ DB=$1 if [ "$DB" = "mysql" ]; then COMPOSE_FILE="artemis-migration-check-mysql.yml" -elif [ "$DB" = "postgresql" ]; then +elif [ "$DB" = "postgres" ]; then COMPOSE_FILE="artemis-migration-check-postgres.yml" else - echo "Invalid database type. Please choose either mysql or postgresql." + echo "Invalid database type. Please choose either mysql or postgres." exit 1 fi diff --git a/docker/cypress-E2E-tests-mysql.yml b/docker/cypress-E2E-tests-mysql.yml index 02a040a68dcd..835b6880e033 100644 --- a/docker/cypress-E2E-tests-mysql.yml +++ b/docker/cypress-E2E-tests-mysql.yml @@ -44,7 +44,7 @@ services: environment: CYPRESS_DB_TYPE: "MySQL" SORRY_CYPRESS_PROJECT_ID: "artemis-mysql" - command: sh -c "cd /app/artemis/src/test/cypress && chmod 777 /root && npm ci && npm run cypress:setup && (npm run cypress:record:mysql & sleep 60 && npm run cypress:record:mysql & wait)" + command: sh -c "cd /app/artemis/src/test/cypress && chmod 777 /root && npm ci --legacy-peer-deps && npm run cypress:setup && (npm run cypress:record:mysql & sleep 60 && npm run cypress:record:mysql & wait)" networks: artemis: diff --git a/docker/cypress-E2E-tests-postgres.yml b/docker/cypress-E2E-tests-postgres.yml index e2396270d899..3d24c792a177 100644 --- a/docker/cypress-E2E-tests-postgres.yml +++ b/docker/cypress-E2E-tests-postgres.yml @@ -44,7 +44,7 @@ services: environment: CYPRESS_DB_TYPE: "Postgres" SORRY_CYPRESS_PROJECT_ID: "artemis-postgres" - command: sh -c "cd /app/artemis/src/test/cypress && chmod 777 /root && npm ci && npm run cypress:setup && (npm run cypress:record:postgres & sleep 60 && npm run cypress:record:postgres & wait)" + command: sh -c "cd /app/artemis/src/test/cypress && chmod 777 /root && npm ci --legacy-peer-deps && npm run cypress:setup && (npm run cypress:record:postgres & sleep 60 && npm run cypress:record:postgres & wait)" networks: artemis: diff --git a/src/test/cypress/package.json b/src/test/cypress/package.json index 12a8aded34da..e8a82103f70a 100644 --- a/src/test/cypress/package.json +++ b/src/test/cypress/package.json @@ -30,7 +30,7 @@ "cypress:open": "cypress open", "cypress:run": "cypress run --browser=chrome", "cypress:setup": "cypress install && cypress run --quiet --spec init/ImportUsers.cy.ts", - "cypress:record:mysql": "npx cypress-cloud run --cloud-debug --parallel --record --ci-build-id \"${SORRY_CYPRESS_BRANCH_NAME} #${SORRY_CYPRESS_BUILD_ID} ${SORRY_CYPRESS_RERUN_COUNT} (MySQL)\"", + "cypress:record:mysql": "npx cypress-cloud run --parallel --record --ci-build-id \"${SORRY_CYPRESS_BRANCH_NAME} #${SORRY_CYPRESS_BUILD_ID} ${SORRY_CYPRESS_RERUN_COUNT} (MySQL)\"", "cypress:record:postgres": "npx cypress-cloud run --parallel --record --ci-build-id \"${SORRY_CYPRESS_BRANCH_NAME} #${SORRY_CYPRESS_BUILD_ID} ${SORRY_CYPRESS_RERUN_COUNT} (Postgres)\"", "update": "npm-upgrade" } From 0d369e9c780665fdc11f3b2dd62675ed0bd40c7e Mon Sep 17 00:00:00 2001 From: Stephan Krusche Date: Thu, 14 Sep 2023 10:16:02 +0200 Subject: [PATCH 08/15] Development: Fix issues with cypress dependencies --- src/test/cypress/package-lock.json | 10 +++++----- src/test/cypress/package.json | 7 ++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/test/cypress/package-lock.json b/src/test/cypress/package-lock.json index 632fff350ea3..677ef5fe04c8 100644 --- a/src/test/cypress/package-lock.json +++ b/src/test/cypress/package-lock.json @@ -7,7 +7,7 @@ "name": "artemis_cypress", "license": "MIT", "devDependencies": { - "@4tw/cypress-drag-drop": "2.2.4", + "@4tw/cypress-drag-drop": "2.2.5", "@types/node": "20.6.0", "cypress": "13.2.0", "cypress-cloud": "1.10.0-beta.4", @@ -19,12 +19,12 @@ } }, "node_modules/@4tw/cypress-drag-drop": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/@4tw/cypress-drag-drop/-/cypress-drag-drop-2.2.4.tgz", - "integrity": "sha512-6mmjJ0SHjciI0JNofdVcN1LYLbvUcUT3oY2O+1HaTlQlpKPlX9kBc040Sik6RYFK7cgWvUNwoDoDoGSKA4IS/g==", + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/@4tw/cypress-drag-drop/-/cypress-drag-drop-2.2.5.tgz", + "integrity": "sha512-3ghTmzhOmUqeN6U3QmUnKRUxI7OMLbJA4hHUY/eS/FhWJgxbiGgcaELbolWnBAOpajPXcsNQGYEj9brd59WH6A==", "dev": true, "peerDependencies": { - "cypress": "^2.1.0 || ^3.1.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0 || ^12.0.0" + "cypress": "2 - 13" } }, "node_modules/@babel/runtime": { diff --git a/src/test/cypress/package.json b/src/test/cypress/package.json index e8a82103f70a..26ce69d2666e 100644 --- a/src/test/cypress/package.json +++ b/src/test/cypress/package.json @@ -7,7 +7,7 @@ "node_modules" ], "devDependencies": { - "@4tw/cypress-drag-drop": "2.2.4", + "@4tw/cypress-drag-drop": "2.2.5", "@types/node": "20.6.0", "cypress": "13.2.0", "cypress-cloud": "1.10.0-beta.4", @@ -21,10 +21,7 @@ "semver": "7.5.3", "word-wrap": "1.2.3", "debug": "4.3.4", - "tough-cookie": "4.1.3", - "@4tw/cypress-drag-drop": { - "cypress": "13.1.0" - } + "tough-cookie": "4.1.3" }, "scripts": { "cypress:open": "cypress open", From a2f1397627201ca289b73227a3703153edad5b48 Mon Sep 17 00:00:00 2001 From: Lennart Keller <44754405+lennart-keller@users.noreply.github.com> Date: Thu, 14 Sep 2023 10:42:23 +0200 Subject: [PATCH 09/15] Development: Fix Postgres Incompatibility for conversation user information query (#7190) --- .../repository/metis/conversation/ConversationRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/de/tum/in/www1/artemis/repository/metis/conversation/ConversationRepository.java b/src/main/java/de/tum/in/www1/artemis/repository/metis/conversation/ConversationRepository.java index 93ee6dbb5d11..d522170180f5 100644 --- a/src/main/java/de/tum/in/www1/artemis/repository/metis/conversation/ConversationRepository.java +++ b/src/main/java/de/tum/in/www1/artemis/repository/metis/conversation/ConversationRepository.java @@ -64,7 +64,7 @@ default Conversation findByIdElseThrow(long conversationId) { LEFT JOIN Post p ON conv.id = p.conversation.id AND (p.creationDate > cp.lastRead OR (channel.isCourseWide IS true AND cp.lastRead IS null)) WHERE conv.id IN :conversationIds AND (channel.isCourseWide IS true OR (conv.id = cp.conversation.id AND cp.user.id = :userId)) - GROUP BY conv.id + GROUP BY conv.id, cp.id, cp.isModerator, cp.isFavorite, cp.isHidden, cp.lastRead """) List getUserInformationForConversations(@Param("conversationIds") Iterable conversationIds, @Param("userId") Long userId); From 61a94902c84ddf608bba8dceee388c46b81c98a4 Mon Sep 17 00:00:00 2001 From: Stephan Krusche Date: Thu, 14 Sep 2023 10:45:33 +0200 Subject: [PATCH 10/15] Programming exercises: Fix missing branch configuration (#7189) --- .../in/www1/artemis/web/rest/ProgrammingExerciseResource.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/ProgrammingExerciseResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/ProgrammingExerciseResource.java index fd265be5796f..99d4d4ed88f7 100644 --- a/src/main/java/de/tum/in/www1/artemis/web/rest/ProgrammingExerciseResource.java +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/ProgrammingExerciseResource.java @@ -266,6 +266,9 @@ public ResponseEntity updateProgrammingExercise(@RequestBod // Forbid conversion between normal course exercise and exam exercise exerciseService.checkForConversionBetweenExamAndCourseExercise(updatedProgrammingExercise, programmingExerciseBeforeUpdate, ENTITY_NAME); + // Ignore changes to the default branch + updatedProgrammingExercise.setBranch(programmingExerciseBeforeUpdate.getBranch()); + if (updatedProgrammingExercise.getAuxiliaryRepositories() == null) { // make sure the default value is set properly updatedProgrammingExercise.setAuxiliaryRepositories(new ArrayList<>()); From 9e8bd093c101740d1b5630732d3906711837534f Mon Sep 17 00:00:00 2001 From: Ludwig <33753999+4ludwig4@users.noreply.github.com> Date: Thu, 14 Sep 2023 10:51:37 +0200 Subject: [PATCH 11/15] Development: Add postgres docker compose setup (#6759) --- .gitignore | 2 + .../artemis-postgres-data/.gitkeep | 0 docker/artemis-dev-mysql.yml | 7 +-- docker/artemis-dev-postgres.yml | 39 ++++++++++++++++ docker/artemis-migration-check-postgres.yml | 6 ++- docker/artemis-prod-postgres.yml | 38 ++++++++++++++++ docker/artemis/config/cypress-mysql.env | 40 ++--------------- docker/artemis/config/cypress-postgres.env | 45 +------------------ docker/artemis/config/cypress.env | 44 ++++++++++++++++++ docker/artemis/config/dev.env | 11 +++++ docker/artemis/config/postgres.env | 9 ++++ docker/artemis/config/prod.env | 15 ++++++- docker/cypress-E2E-tests-mysql.yml | 1 + docker/cypress-E2E-tests-postgres.yml | 16 ++++--- docker/mailhog.yml | 2 +- docker/{postgresql.yml => postgres.yml} | 19 ++++---- docker/{postgresql => postgres}/default.env | 1 + docker/test-server-postgresql.yml | 14 +++--- docs/admin/productionSetupTips.rst | 21 +++++++++ docs/dev/setup.rst | 16 +++++-- 20 files changed, 231 insertions(+), 115 deletions(-) create mode 100644 docker/.docker-data/artemis-postgres-data/.gitkeep create mode 100644 docker/artemis-dev-postgres.yml create mode 100644 docker/artemis-prod-postgres.yml create mode 100644 docker/artemis/config/cypress.env create mode 100644 docker/artemis/config/dev.env create mode 100644 docker/artemis/config/postgres.env rename docker/{postgresql.yml => postgres.yml} (64%) rename docker/{postgresql => postgres}/default.env (65%) diff --git a/.gitignore b/.gitignore index f897ea313607..fe02c2ff3875 100644 --- a/.gitignore +++ b/.gitignore @@ -185,6 +185,8 @@ data-exports/ !/docker/.docker-data/artemis-data/.gitkeep /docker/.docker-data/artemis-mysql-data/* !/docker/.docker-data/artemis-mysql-data/.gitkeep +/docker/.docker-data/artemis-postgres-data/* +!/docker/.docker-data/artemis-postgres-data/.gitkeep ###################### # Cypress diff --git a/docker/.docker-data/artemis-postgres-data/.gitkeep b/docker/.docker-data/artemis-postgres-data/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/docker/artemis-dev-mysql.yml b/docker/artemis-dev-mysql.yml index 2e218cf60b7e..65655ef1b970 100644 --- a/docker/artemis-dev-mysql.yml +++ b/docker/artemis-dev-mysql.yml @@ -17,11 +17,8 @@ services: # expose the port to make it reachable docker internally even if the external port mapping changes expose: - "5005" - environment: - # The following enables the Java Remote Debugging port. More infos in the documentation: - # https://ls1intum.github.io/Artemis/dev/setup.html#debugging-with-docker - _JAVA_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 - SPRING_PROFILES_ACTIVE: artemis,scheduling,athena,dev,docker + env_file: + - ./artemis/config/dev.env depends_on: mysql: condition: service_healthy diff --git a/docker/artemis-dev-postgres.yml b/docker/artemis-dev-postgres.yml new file mode 100644 index 000000000000..5e1344511b6f --- /dev/null +++ b/docker/artemis-dev-postgres.yml @@ -0,0 +1,39 @@ +# ---------------------------------------------------------------------------------------------------------------------- +# Artemis-Dev-Postgres Setup +# ---------------------------------------------------------------------------------------------------------------------- + +services: + artemis-app: + extends: + file: ./artemis.yml + service: artemis-app + # just add this linux workaround for docker compose in a development version of artemis as developers + # might want to access external services on the docker host + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "8080:8080" + - "5005:5005" # Java Remote Debugging port declared in the java cmd options + # expose the port to make it reachable docker internally even if the external port mapping changes + expose: + - "5005" + env_file: + - ./artemis/config/dev.env + - ./artemis/config/postgres.env + depends_on: + postgres: + condition: service_healthy + postgres: + extends: + file: ./postgres.yml + service: postgres + +networks: + artemis: + driver: "bridge" + name: artemis +volumes: + artemis-postgres-data: + name: artemis-postgres-data + artemis-data: + name: artemis-data diff --git a/docker/artemis-migration-check-postgres.yml b/docker/artemis-migration-check-postgres.yml index 2ddd82d07132..0d801be844d2 100644 --- a/docker/artemis-migration-check-postgres.yml +++ b/docker/artemis-migration-check-postgres.yml @@ -8,6 +8,8 @@ services: file: ./artemis.yml service: artemis-app env_file: + - ./artemis/config/postgres.env + - ./artemis/config/cypress.env - ./artemis/config/cypress-postgres.env - ./artemis/config/migration-check.env depends_on: @@ -32,7 +34,7 @@ networks: driver: "bridge" name: artemis volumes: - artemis-postgresql-data: - name: artemis-postgresql-data + artemis-postgres-data: + name: artemis-postgres-data artemis-data: name: artemis-data diff --git a/docker/artemis-prod-postgres.yml b/docker/artemis-prod-postgres.yml new file mode 100644 index 000000000000..fdb0c691f0ef --- /dev/null +++ b/docker/artemis-prod-postgres.yml @@ -0,0 +1,38 @@ +# ---------------------------------------------------------------------------------------------------------------------- +# Artemis-Prod-Postgres Setup +# ---------------------------------------------------------------------------------------------------------------------- + +services: + artemis-app: + extends: + file: ./artemis.yml + service: artemis-app + depends_on: + postgres: + condition: service_healthy + restart: on-failure:3 + volumes: + - ./.docker-data/artemis-data:/opt/artemis/data + env_file: + - ./artemis/config/postgres.env + postgres: + extends: + file: ./postgres.yml + service: postgres + restart: on-failure:3 + volumes: + - ./.docker-data/artemis-postgres-data:/var/lib/postgresql/data + nginx: + extends: + file: ./nginx.yml + service: nginx + # the artemis-app service needs to be started, otherwise there are problems with name resolution in docker + depends_on: + artemis-app: + condition: service_started + restart: on-failure:3 + +networks: + artemis: + driver: "bridge" + name: artemis diff --git a/docker/artemis/config/cypress-mysql.env b/docker/artemis/config/cypress-mysql.env index 17177a7afbf3..b9f006a4b395 100644 --- a/docker/artemis/config/cypress-mysql.env +++ b/docker/artemis/config/cypress-mysql.env @@ -1,34 +1,21 @@ +# ---------------------------------------------------------------------------------------------------------------------- +# Artemis configuration overrides for the Cypress E2E MySQL setups +# ---------------------------------------------------------------------------------------------------------------------- + SPRING_PROFILES_ACTIVE="artemis,scheduling,bamboo,bitbucket,jira,prod,docker" SPRING_DATASOURCE_URL="jdbc:mysql://artemis-mysql:3306/Artemis?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useUnicode=true&characterEncoding=utf8&useSSL=false&useLegacyDatetimeCode=false&serverTimezone=UTC" SPRING_DATASOURCE_USERNAME="root" -SPRING_DATASOURCE_PASSWORD="" -SPRING_DATASOURCE_HIKARI_MAXIMUMPOOLSIZE="100" SPRING_JPA_DATABASE_PLATFORM="org.hibernate.dialect.MySQL8Dialect" SPRING_JPA_DATABASE="MYSQL" -SPRING_JPA_HIBERNATE_CONNECTION_CHARSET="utf8mb4" -SPRING_JPA_HIBERNATE_CONNECTION_CHARACTERENCODING="utf8mb4" -SPRING_JPA_HIBERNATE_CONNECTION_USEUNICODE="true" - -SPRING_PROMETHEUS_MONITORINGIP="131.159.89.160" - -# for bamboo and bitbucket notifications on /api/programming-exercises/new-result it seems like port -# 54321 is internally open for the bamboo agents -# also can't use SSL for this as the hostnames are not integrated in the self-signed certificate -SERVER_URL="http://${HOST_HOSTNAME}:54321" -# We don't need secure passwords for testing. Lower rounds will speed up tests. 4 is the lowest -ARTEMIS_BCRYPTSALTROUNDS="4" ARTEMIS_USERMANAGEMENT_USEEXTERNAL="true" ARTEMIS_USERMANAGEMENT_EXTERNAL_URL="https://jira-prelive.ase.in.tum.de" ARTEMIS_USERMANAGEMENT_EXTERNAL_USER="${bamboo_jira_prelive_admin_user}" ARTEMIS_USERMANAGEMENT_EXTERNAL_PASSWORD="${bamboo_jira_prelive_admin_password}" ARTEMIS_USERMANAGEMENT_EXTERNAL_ADMINGROUPNAME="artemis-dev" -ARTEMIS_USERMANAGEMENT_INTERNALADMIN_USERNAME="${bamboo_artemis_admin_username}" -ARTEMIS_USERMANAGEMENT_INTERNALADMIN_PASSWORD="${bamboo_artemis_admin_password}" -ARTEMIS_USERMANAGEMENT_LOGIN_ACCOUNTNAME="TUM" ARTEMIS_VERSIONCONTROL_URL="https://bitbucket-prelive.ase.in.tum.de" ARTEMIS_VERSIONCONTROL_USER="${bamboo_jira_prelive_admin_user}" @@ -40,22 +27,3 @@ ARTEMIS_CONTINUOUSINTEGRATION_PASSWORD="${bamboo_jira_prelive_admin_password}" ARTEMIS_CONTINUOUSINTEGRATION_TOKEN="${bamboo_ARTEMIS_CONTINUOUS_INTEGRATION_TOKEN_SECRET}" ARTEMIS_CONTINUOUSINTEGRATION_ARTEMISAUTHENTICATIONTOKENVALUE="${bamboo_ARTEMIS_CONTINUOUS_INTEGRATION_ARTEMIS_AUTHENTICATION_TOKEN_VALUE_SECRET}" ARTEMIS_CONTINUOUSINTEGRATION_VCSAPPLICATIONLINKNAME="Bitbucket Prelive" -ARTEMIS_CONTINUOUSINTEGRATION_EMPTYCOMMITNECESSARY="true" - -ARTEMIS_APOLLON_CONVERSIONSERVICEURL="https://apollon.ase.in.tum.de/api/converter" - -# Token is valid 3 days -JHIPSTER_SECURITY_AUTHENTICATION_JWT_TOKENVALIDITYINSECONDS="259200" -# Token is valid 30 days -JHIPSTER_SECURITY_AUTHENTICATION_JWT_TOKENVALIDITYINSECONDSFORREMEMBERME="2592000" - -# Properties to be exposed on the /info management endpoint - -INFO_IMPRINT="https://ase.in.tum.de/lehrstuhl_1/component/content/article/179-imprint" -INFO_TESTSERVER="true" -INFO_TEXTASSESSMENTANALYTICSENABLED="true" -INFO_STUDENTEXAMSTORESESSIONDATA="true" - -LOGGING_FILE_NAME="/opt/artemis/data/artemis.log" - -MANAGEMENT_METRICS_EXPORT_PROMETHEUS_ENABLED="true" diff --git a/docker/artemis/config/cypress-postgres.env b/docker/artemis/config/cypress-postgres.env index d0ab68263c90..73254a9afebc 100644 --- a/docker/artemis/config/cypress-postgres.env +++ b/docker/artemis/config/cypress-postgres.env @@ -1,34 +1,10 @@ # ---------------------------------------------------------------------------------------------------------------------- -# Artemis configurations for Postgres setups +# Artemis configuration overrides for the Cypress E2E Postgres setups # ---------------------------------------------------------------------------------------------------------------------- SPRING_PROFILES_ACTIVE="artemis,scheduling,jenkins,gitlab,prod,docker" -SPRING_DATASOURCE_URL="jdbc:postgresql://artemis-postgresql:5432/Artemis?sslmode=disable" -SPRING_DATASOURCE_USERNAME="postgres" -SPRING_DATASOURCE_PASSWORD="" -SPRING_DATASOURCE_HIKARI_MAXIMUMPOOLSIZE="100" - -SPRING_JPA_DATABASE_PLATFORM="org.hibernate.dialect.PostgreSQL10Dialect" -SPRING_JPA_DATABASE="POSTGRESQL" -SPRING_JPA_HIBERNATE_CONNECTION_CHARSET="utf8mb4" -SPRING_JPA_HIBERNATE_CONNECTION_CHARACTERENCODING="utf8mb4" -SPRING_JPA_HIBERNATE_CONNECTION_USEUNICODE="true" - -SPRING_PROMETHEUS_MONITORINGIP="131.159.89.160" - -# for bamboo and bitbucket notifications on /api/programming-exercises/new-result it seems like port -# 54321 is internally open for the bamboo agents -# also can't use SSL for this as the hostnames are not integrated in the self-signed certificate -SERVER_URL="http://${HOST_HOSTNAME}:54321" - -# We don't need secure passwords for testing. Lower rounds will speed up tests. 4 is the lowest -ARTEMIS_BCRYPTSALTROUNDS="4" - ARTEMIS_USERMANAGEMENT_USEEXTERNAL="false" -ARTEMIS_USERMANAGEMENT_INTERNALADMIN_USERNAME="${bamboo_artemis_admin_username}" -ARTEMIS_USERMANAGEMENT_INTERNALADMIN_PASSWORD="${bamboo_artemis_admin_password}" -ARTEMIS_USERMANAGEMENT_LOGIN_ACCOUNTNAME="TUM" ARTEMIS_VERSIONCONTROL_URL="https://gitlab-test.artemis.in.tum.de" ARTEMIS_VERSIONCONTROL_USER="${bamboo_gitlab_admin_user}" @@ -44,26 +20,7 @@ ARTEMIS_CONTINUOUSINTEGRATION_SECRETPUSHTOKEN="${bamboo_jenkins_secret_push_toke ARTEMIS_CONTINUOUSINTEGRATION_VCSCREDENTIALS="${bamboo_jenkins_vcs_credentials_secret}" ARTEMIS_CONTINUOUSINTEGRATION_ARTEMISAUTHENTICATIONTOKENKEY="${bamboo_jenkins_artemis_ci_authentication_token_key_secret}" ARTEMIS_CONTINUOUSINTEGRATION_ARTEMISAUTHENTICATIONTOKENVALUE="${bamboo_jenkins_artemis_ci_authentication_token_value_secret}" -ARTEMIS_CONTINUOUSINTEGRATION_EMPTYCOMMITNECESSARY="true" ARTEMIS_CONTINUOUSINTEGRATION_BUILDTIMEOUT="30" -ARTEMIS_APOLLON_CONVERSIONSERVICEURL="https://apollon.ase.in.tum.de/api/converter" - -# Token is valid 3 days -JHIPSTER_SECURITY_AUTHENTICATION_JWT_TOKENVALIDITYINSECONDS="259200" -# Token is valid 30 days -JHIPSTER_SECURITY_AUTHENTICATION_JWT_TOKENVALIDITYINSECONDSFORREMEMBERME="2592000" - -# Properties to be exposed on the /info management endpoint - -INFO_IMPRINT="https://ase.in.tum.de/lehrstuhl_1/component/content/article/179-imprint" -INFO_TESTSERVER="true" -INFO_TEXTASSESSMENTANALYTICSENABLED="true" -INFO_STUDENTEXAMSTORESESSIONDATA="true" - -LOGGING_FILE_NAME="/opt/artemis/data/artemis.log" - -MANAGEMENT_METRICS_EXPORT_PROMETHEUS_ENABLED="true" - JENKINS_INTERNALURLS_CIURL="https://jenkins-test.artemis.in.tum.de" JENKINS_INTERNALURLS_VCNURL="https://gitlab-test.artemis.in.tum.de" diff --git a/docker/artemis/config/cypress.env b/docker/artemis/config/cypress.env new file mode 100644 index 000000000000..c056a8d9004c --- /dev/null +++ b/docker/artemis/config/cypress.env @@ -0,0 +1,44 @@ +# ---------------------------------------------------------------------------------------------------------------------- +# Common Artemis configurations for the Cypress E2E MySQL and Postgres setups +# ---------------------------------------------------------------------------------------------------------------------- + +SPRING_DATASOURCE_PASSWORD="" +SPRING_DATASOURCE_HIKARI_MAXIMUMPOOLSIZE="100" + +SPRING_JPA_HIBERNATE_CONNECTION_CHARSET="utf8mb4" +SPRING_JPA_HIBERNATE_CONNECTION_CHARACTERENCODING="utf8mb4" +SPRING_JPA_HIBERNATE_CONNECTION_USEUNICODE="true" + +SPRING_PROMETHEUS_MONITORINGIP="131.159.89.160" + +# for bamboo and bitbucket notifications on /api/programming-exercises/new-result it seems like port +# 54321 is internally open for the bamboo agents +# also can't use SSL for this as the hostnames are not integrated in the self-signed certificate +SERVER_URL="http://${HOST_HOSTNAME}:54321" + +# We don't need secure passwords for testing. Lower rounds will speed up tests. 4 is the lowest +ARTEMIS_BCRYPTSALTROUNDS="4" + +ARTEMIS_USERMANAGEMENT_INTERNALADMIN_USERNAME="${bamboo_artemis_admin_username}" +ARTEMIS_USERMANAGEMENT_INTERNALADMIN_PASSWORD="${bamboo_artemis_admin_password}" +ARTEMIS_USERMANAGEMENT_LOGIN_ACCOUNTNAME="TUM" + +ARTEMIS_CONTINUOUSINTEGRATION_EMPTYCOMMITNECESSARY="true" + +ARTEMIS_APOLLON_CONVERSIONSERVICEURL="https://apollon.ase.in.tum.de/api/converter" + +# Token is valid 3 days +JHIPSTER_SECURITY_AUTHENTICATION_JWT_TOKENVALIDITYINSECONDS="259200" +# Token is valid 30 days +JHIPSTER_SECURITY_AUTHENTICATION_JWT_TOKENVALIDITYINSECONDSFORREMEMBERME="2592000" + +# Properties to be exposed on the /info management endpoint + +INFO_IMPRINT="https://ase.in.tum.de/lehrstuhl_1/component/content/article/179-imprint" +INFO_TESTSERVER="true" +INFO_TEXTASSESSMENTANALYTICSENABLED="true" +INFO_STUDENTEXAMSTORESESSIONDATA="true" + +LOGGING_FILE_NAME="/opt/artemis/data/artemis.log" + +MANAGEMENT_METRICS_EXPORT_PROMETHEUS_ENABLED="true" diff --git a/docker/artemis/config/dev.env b/docker/artemis/config/dev.env new file mode 100644 index 000000000000..4dc4617261bd --- /dev/null +++ b/docker/artemis/config/dev.env @@ -0,0 +1,11 @@ +# ---------------------------------------------------------------------------------------------------------------------- +# Artemis configurations for Dev Artemis setups +# ---------------------------------------------------------------------------------------------------------------------- +# The default Artemis Spring profiles for Docker are defined here. +# ---------------------------------------------------------------------------------------------------------------------- + +SPRING_PROFILES_ACTIVE: artemis,scheduling,athena,dev,docker + +# The following enables the Java Remote Debugging port. More infos in the documentation: +# https://ls1intum.github.io/Artemis/dev/setup.html#debugging-with-docker +_JAVA_OPTIONS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 diff --git a/docker/artemis/config/postgres.env b/docker/artemis/config/postgres.env new file mode 100644 index 000000000000..2c8a07e95f37 --- /dev/null +++ b/docker/artemis/config/postgres.env @@ -0,0 +1,9 @@ +# ---------------------------------------------------------------------------------------------------------------------- +# Artemis configurations for Postgres setups +# ---------------------------------------------------------------------------------------------------------------------- + +SPRING_DATASOURCE_URL="jdbc:postgresql://artemis-postgres:5432/Artemis?sslmode=disable" +SPRING_DATASOURCE_USERNAME="postgres" + +SPRING_JPA_DATABASE_PLATFORM="org.hibernate.dialect.PostgreSQL10Dialect" +SPRING_JPA_DATABASE="POSTGRESQL" diff --git a/docker/artemis/config/prod.env b/docker/artemis/config/prod.env index 3f708bdc709e..a86a87bcf51a 100644 --- a/docker/artemis/config/prod.env +++ b/docker/artemis/config/prod.env @@ -1,6 +1,15 @@ +# ---------------------------------------------------------------------------------------------------------------------- +# Artemis configurations for Prod Artemis setups +# ---------------------------------------------------------------------------------------------------------------------- +# The default Artemis Spring profiles for Docker are defined here. +# ---------------------------------------------------------------------------------------------------------------------- SPRING_PROFILES_ACTIVE="artemis,scheduling,athena,prod,docker" +# ---------------------------------------------------------------------------------------------------------------------- # Secret Overrides +# ---------------------------------------------------------------------------------------------------------------------- +# Change these default secrets in another not-commited environment override file for prod systems! +# ---------------------------------------------------------------------------------------------------------------------- ARTEMIS_ATHENA_SECRET="abcdefg12345" ARTEMIS_USERMANAGEMENT_INTERNALADMIN_USERNAME="artemis_admin" ARTEMIS_USERMANAGEMENT_INTERNALADMIN_PASSWORD="artemis_admin" @@ -17,6 +26,10 @@ SPRING_WEBSOCKET_BROKER_PASSWORD="guest" JHIPSTER_SECURITY_AUTHENTICATION_JWT_BASE64SECRET="bXktc2VjcmV0LWtleS13aGljaC1zaG91bGQtYmUtY2hhbmdlZC1pbi1wcm9kdWN0aW9uLWFuZC1iZS1iYXNlNjQtZW5jb2RlZAo=" JHIPSTER_REGISTRY_PASSWORD="AN-ADMIN-PASSWORD-THAT-MUST-BE-CHANGED (FROM REGISTRY CONFIG)" - +# ---------------------------------------------------------------------------------------------------------------------- # Plain Prod Artemis Overrides +# ---------------------------------------------------------------------------------------------------------------------- +# Keep these at a minimum! Try to change the default values either in the application-docker.yml or even better +# in one of the other application.yml or application-*.yml files. +# ---------------------------------------------------------------------------------------------------------------------- ARTEMIS_USERMANAGEMENT_USEEXTERNAL="false" diff --git a/docker/cypress-E2E-tests-mysql.yml b/docker/cypress-E2E-tests-mysql.yml index 835b6880e033..96f9fe67989f 100644 --- a/docker/cypress-E2E-tests-mysql.yml +++ b/docker/cypress-E2E-tests-mysql.yml @@ -16,6 +16,7 @@ services: mysql: condition: service_healthy env_file: + - ./artemis/config/cypress.env - ./artemis/config/cypress-mysql.env nginx: diff --git a/docker/cypress-E2E-tests-postgres.yml b/docker/cypress-E2E-tests-postgres.yml index 3d24c792a177..a42d2232adc7 100644 --- a/docker/cypress-E2E-tests-postgres.yml +++ b/docker/cypress-E2E-tests-postgres.yml @@ -1,21 +1,23 @@ # ---------------------------------------------------------------------------------------------------------------------- -# Cypress Setup PostgreSQL +# Cypress Setup Postgres # ---------------------------------------------------------------------------------------------------------------------- services: - postgresql: + postgres: extends: - file: ./postgresql.yml - service: postgresql + file: ./postgres.yml + service: postgres artemis-app: extends: file: ./artemis.yml service: artemis-app depends_on: - postgresql: + postgres: condition: service_healthy env_file: + - ./artemis/config/postgres.env + - ./artemis/config/cypress.env - ./artemis/config/cypress-postgres.env nginx: @@ -51,7 +53,7 @@ networks: driver: "bridge" name: artemis volumes: - artemis-postgresql-data: - name: artemis-postgresql-data + artemis-postgres-data: + name: artemis-postgres-data artemis-data: name: artemis-data diff --git a/docker/mailhog.yml b/docker/mailhog.yml index 5145f67cdff4..70af79096071 100644 --- a/docker/mailhog.yml +++ b/docker/mailhog.yml @@ -7,7 +7,7 @@ services: mailhog: container_name: artemis-mailhog - image: mailhog/mailhog + image: docker.io/mailhog/mailhog pull_policy: always ports: - "1025:1025" diff --git a/docker/postgresql.yml b/docker/postgres.yml similarity index 64% rename from docker/postgresql.yml rename to docker/postgres.yml index aec7d23cfee7..097c9530baee 100644 --- a/docker/postgresql.yml +++ b/docker/postgres.yml @@ -3,19 +3,22 @@ # ---------------------------------------------------------------------------------------------------------------------- services: - postgresql: - container_name: artemis-postgresql - image: postgres:15.3-alpine + postgres: + container_name: artemis-postgres + image: docker.io/library/postgres:15.3-alpine pull_policy: always user: postgres command: ["postgres", "-c", "max_connections=200"] volumes: - - artemis-postgresql-data:/var/lib/postgresql/data + - artemis-postgres-data:/var/lib/postgresql/data # DO NOT use this default file for production systems! env_file: - - ./postgresql/default.env + - ./postgres/default.env ports: - - "5432:5432" + - "127.0.0.1:5432:5432" + # expose the port to make it reachable docker internally even if the external port mapping changes + expose: + - "5432" healthcheck: test: pg_isready -U postgres -d Artemis interval: 5s @@ -32,5 +35,5 @@ networks: name: artemis volumes: - artemis-postgresql-data: - name: artemis-postgresql-data + artemis-postgres-data: + name: artemis-postgres-data diff --git a/docker/postgresql/default.env b/docker/postgres/default.env similarity index 65% rename from docker/postgresql/default.env rename to docker/postgres/default.env index 9d69c3af68ff..d92a5b5f2722 100644 --- a/docker/postgresql/default.env +++ b/docker/postgres/default.env @@ -1,3 +1,4 @@ POSTGRES_HOST_AUTH_METHOD=trust POSTGRES_USER=postgres POSTGRES_DB=Artemis +PGDATA=/var/lib/postgresql/data/pgdata diff --git a/docker/test-server-postgresql.yml b/docker/test-server-postgresql.yml index 140d77a0cb3b..e9740206ed52 100644 --- a/docker/test-server-postgresql.yml +++ b/docker/test-server-postgresql.yml @@ -1,5 +1,5 @@ # ---------------------------------------------------------------------------------------------------------------------- -# Setup for a test server with PostgreSQL +# Setup for a test server with Postgres # ---------------------------------------------------------------------------------------------------------------------- # It is designed to take in a lot of environment variables to take in all the configuration of the test server. # ---------------------------------------------------------------------------------------------------------------------- @@ -11,7 +11,7 @@ services: service: artemis-app image: ghcr.io/ls1intum/artemis:${ARTEMIS_DOCKER_TAG:-latest} depends_on: - postgresql: + postgres: condition: service_healthy pull_policy: always restart: always @@ -22,15 +22,15 @@ services: - ${ARTEMIS_LEGAL_MOUNT:-./.docker-data/artemis-legal}:/opt/artemis/legal - ${ARTEMIS_DATA_EXPORT_MOUNT:-./.docker-data/artemis-data-exports}:/opt/artemis/data-exports - postgresql: + postgres: extends: - file: ./postgresql.yml - service: postgresql + file: ./postgres.yml + service: postgres restart: always env_file: - - ${DATABASE_ENV_FILE:-./postgresql/default.env} + - ${DATABASE_ENV_FILE:-./postgres/default.env} volumes: - - ${DATABASE_VOLUME_MOUNT:-./.docker-data/artemis-postgresql-data}:/var/lib/postgresql/data + - ${DATABASE_VOLUME_MOUNT:-./.docker-data/artemis-postgres-data}:/var/lib/postgresql/data nginx: extends: diff --git a/docs/admin/productionSetupTips.rst b/docs/admin/productionSetupTips.rst index fadaebb46766..cc95bee1b4ee 100644 --- a/docs/admin/productionSetupTips.rst +++ b/docs/admin/productionSetupTips.rst @@ -35,3 +35,24 @@ Place the webpage that should be shown in case of Artemis being unreachable (in error_page 501 502 503 /service_down.html; } + +## Gather all Docker Compose-related tips here which are not relevant for developers! +.. _docker_compose_setup_prod: + +Docker Compose Setup +-------------------- + +The :ref:`development section of the documentation ` provides a introduction to +Docker Compose setups for Artemis. +This section provides additional information for administrators. + +File Permissions +^^^^^^^^^^^^^^^^ +If you use the production Docker Compose Setups (``artemis-prod-*.yml``) with bind mounts change +the file permissions accordingly: + +.. code:: bash + + sudo chown -R $(id -u):70 docker/.docker-data/artemis-postgres-data + sudo chown -R $(id -u):999 docker/.docker-data/artemis-mysql-data + sudo chown -R $(id -u):1337 docker/.docker-data/artemis-data diff --git a/docs/dev/setup.rst b/docs/dev/setup.rst index 082a792a6be0..47d87eb8fa82 100644 --- a/docs/dev/setup.rst +++ b/docs/dev/setup.rst @@ -119,7 +119,8 @@ PostgreSQL Setup No special PostgreSQL settings are required. You can either use your package manager’s version, or set it up using a container. -An example Docker Compose setup based on the `official container image `_ is provided in ``src/main/docker/postgresql.yml``. +An example Docker Compose setup based on the `official container image `_ +is provided in ``src/main/docker/postgres.yml``. When setting up the Artemis server, the following values need to be added/updated in the server configuration (see setup steps below) to connect to PostgreSQL instead of MySQL: @@ -719,6 +720,8 @@ HTTP. We need to extend the configuration in the file ------------------------------------------------------------------------------------------------------------------------ +.. _docker_compose_setup_dev: + Alternative: Docker Compose Setup --------------------------------- @@ -765,7 +768,11 @@ The easiest way to configure a local deployment via Docker is a deployment with In the directory ``docker/`` you can find the following *docker compose* files for different **setups**: * ``artemis-dev-mysql.yml``: **Artemis-Dev-MySQL** Setup containing the development build of Artemis and a MySQL DB +* ``artemis-dev-postgres.yml``: **Artemis-Dev-Postgres** Setup containing the development build of Artemis and + a PostgreSQL DB * ``artemis-prod-mysql.yml``: **Artemis-Prod-MySQL** Setup containing the production build of Artemis and a MySQL DB +* ``artemis-prod-postgres.yml``: **Artemis-Prod-Postgres** Setup containing the production build of Artemis and + a PostgreSQL DB * ``atlassian.yml``: **Atlassian** Setup containing a Jira, Bitbucket and Bamboo instance (see `Bamboo, Bitbucket and Jira Setup Guide <#bamboo-bitbucket-and-jira-setup>`__ for the configuration of this setup) @@ -775,14 +782,15 @@ In the directory ``docker/`` you can find the following *docker compose* files f * ``monitoring.yml``: **Prometheus-Grafana** Setup containing a Prometheus and Grafana instance * ``mysql.yml``: **MySQL** Setup containing a MySQL DB instance * ``nginx.yml``: **Nginx** Setup containing a preconfigured Nginx instance -* ``postgresql.yml``: **PostgreSQL** Setup containing a PostgreSQL DB instance +* ``postgres.yml``: **Postgres** Setup containing a PostgreSQL DB instance -Two example commands to run such setups: +Three example commands to run such setups: .. code:: bash docker compose -f docker/atlassian.yml up docker compose -f docker/mysql.yml -f docker/gitlab-jenkins.yml up + docker compose -f docker/artemis-dev-postgres.yml up .. tip:: There is also a single ``docker-compose.yml`` in the directory ``docker/`` which mirrors the setup of ``artemis-prod-mysql.yml``. @@ -796,7 +804,7 @@ is defined in the following files: * ``artemis.yml``: **Artemis Service** * ``mysql.yml``: **MySQL DB Service** * ``nginx.yml``: **Nginx Service** -* ``postgresql.yml``: **PostgreSQL DB Service** +* ``postgres.yml``: **PostgreSQL DB Service** * ``gitlab.yml``: **GitLab Service** * ``jenkins.yml``: **Jenkins Service** From d3075d01c145185312f6d9ff777e6fc7ab9f42ce Mon Sep 17 00:00:00 2001 From: Lennart Keller <44754405+lennart-keller@users.noreply.github.com> Date: Thu, 14 Sep 2023 10:57:04 +0200 Subject: [PATCH 12/15] Communication: Create channels for test-exams (#7171) --- .../service/metis/conversation/ChannelService.java | 3 --- .../app/exam/manage/exams/exam-update.component.ts | 2 +- .../tum/in/www1/artemis/exam/ExamIntegrationTest.java | 5 ++++- .../spec/component/exam/exam-update.component.spec.ts | 11 ++++++++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/service/metis/conversation/ChannelService.java b/src/main/java/de/tum/in/www1/artemis/service/metis/conversation/ChannelService.java index 5bf92bc64838..1bea6402da04 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/metis/conversation/ChannelService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/metis/conversation/ChannelService.java @@ -283,9 +283,6 @@ public Channel createExerciseChannel(Exercise exercise, Optional channel * @return the created channel */ public Channel createExamChannel(Exam exam, Optional channelName) { - if (exam.isTestExam()) { - return null; - } Channel channelToCreate = createDefaultChannel(channelName, "exam-", exam.getTitle()); channelToCreate.setIsPublic(false); channelToCreate.setExam(exam); diff --git a/src/main/webapp/app/exam/manage/exams/exam-update.component.ts b/src/main/webapp/app/exam/manage/exams/exam-update.component.ts index 3541b4d1a46a..0997d8694461 100644 --- a/src/main/webapp/app/exam/manage/exams/exam-update.component.ts +++ b/src/main/webapp/app/exam/manage/exams/exam-update.component.ts @@ -74,7 +74,7 @@ export class ExamUpdateComponent implements OnInit { next: (response: HttpResponse) => { this.exam.course = response.body!; this.course = response.body!; - this.hideChannelNameInput = exam.testExam || (exam.id !== undefined && exam.channelName === undefined) || !isMessagingEnabled(this.course); + this.hideChannelNameInput = (exam.id !== undefined && exam.channelName === undefined) || !isMessagingEnabled(this.course); }, error: (err: HttpErrorResponse) => onError(this.alertService, err), }); diff --git a/src/test/java/de/tum/in/www1/artemis/exam/ExamIntegrationTest.java b/src/test/java/de/tum/in/www1/artemis/exam/ExamIntegrationTest.java index 3d3a23e2b37e..b325756679bd 100644 --- a/src/test/java/de/tum/in/www1/artemis/exam/ExamIntegrationTest.java +++ b/src/test/java/de/tum/in/www1/artemis/exam/ExamIntegrationTest.java @@ -1028,9 +1028,12 @@ private List createExamsWithInvalidDates(Course course) { void testCreateTestExam_asInstructor() throws Exception { // Test the creation of a test exam Exam examA = ExamFactory.generateTestExam(course1); - request.post("/api/courses/" + course1.getId() + "/exams", examA, HttpStatus.CREATED); + URI examUri = request.post("/api/courses/" + course1.getId() + "/exams", examA, HttpStatus.CREATED); + Exam savedExam = request.get(String.valueOf(examUri), HttpStatus.OK, Exam.class); verify(examAccessService).checkCourseAccessForInstructorElseThrow(course1.getId()); + Channel channelFromDB = channelRepository.findChannelByExamId(savedExam.getId()); + assertThat(channelFromDB).isNotNull(); } @Test diff --git a/src/test/javascript/spec/component/exam/exam-update.component.spec.ts b/src/test/javascript/spec/component/exam/exam-update.component.spec.ts index bc8f77f8d679..a5e668ad7e97 100644 --- a/src/test/javascript/spec/component/exam/exam-update.component.spec.ts +++ b/src/test/javascript/spec/component/exam/exam-update.component.spec.ts @@ -12,7 +12,7 @@ import { of, throwError } from 'rxjs'; import { RouterTestingModule } from '@angular/router/testing'; import { FormsModule } from '@angular/forms'; import { FontAwesomeTestingModule } from '@fortawesome/angular-fontawesome/testing'; -import { Course } from 'app/entities/course.model'; +import { Course, CourseInformationSharingConfiguration } from 'app/entities/course.model'; import { NgbModule } from '@ng-bootstrap/ng-bootstrap'; import { FormDateTimePickerComponent } from 'app/shared/date-time-picker/date-time-picker.component'; import { MarkdownEditorComponent } from 'app/shared/markdown-editor/markdown-editor.component'; @@ -57,6 +57,7 @@ describe('Exam Update Component', () => { const course = new Course(); course.id = 1; + course.courseInformationSharingConfiguration = CourseInformationSharingConfiguration.COMMUNICATION_AND_MESSAGING; const routes = [ { path: 'course-management/:courseId/exams/:examId', component: DummyComponent }, { path: 'course-management/:courseId/exams', component: DummyComponent }, @@ -211,6 +212,14 @@ describe('Exam Update Component', () => { expect(component.isValidConfiguration).toBeFalse(); }); + it('should show channel name input for test exams', fakeAsync(() => { + examWithoutExercises.testExam = true; + examWithoutExercises.channelName = 'test-exam'; + component.ngOnInit(); + tick(); + expect(component.hideChannelNameInput).toBeFalse(); + })); + it('should validate the example solution publication date correctly', () => { const newExamWithoutExercises = new Exam(); newExamWithoutExercises.id = 2; From 9acb62fd1a12adf0efa96dbd505349522ddc2f11 Mon Sep 17 00:00:00 2001 From: Dominik Remo <47261058+DominikRemo@users.noreply.github.com> Date: Thu, 14 Sep 2023 10:58:13 +0200 Subject: [PATCH 13/15] Development: Disable secure cookies for local testing with dev profile (#7162) --- .../tum/in/www1/artemis/security/jwt/JWTCookieService.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/security/jwt/JWTCookieService.java b/src/main/java/de/tum/in/www1/artemis/security/jwt/JWTCookieService.java index f16565a28e9f..bc58f81579c9 100644 --- a/src/main/java/de/tum/in/www1/artemis/security/jwt/JWTCookieService.java +++ b/src/main/java/de/tum/in/www1/artemis/security/jwt/JWTCookieService.java @@ -17,6 +17,8 @@ public class JWTCookieService { private static final String CYPRESS_PROFILE = "cypress"; + private static final String DEVELOPMENT_PROFILE = "dev"; + private final TokenProvider tokenProvider; private final Environment environment; @@ -56,12 +58,13 @@ public ResponseCookie buildLogoutCookie() { */ private ResponseCookie buildJWTCookie(String jwt, Duration duration) { + // TODO - Remove cypress workaround once cypress uses https and find a better solution for testing locally in Safari Collection activeProfiles = Arrays.asList(environment.getActiveProfiles()); - boolean isCypress = activeProfiles.contains(CYPRESS_PROFILE); + boolean isSecure = !activeProfiles.contains(CYPRESS_PROFILE) && !activeProfiles.contains(DEVELOPMENT_PROFILE); return ResponseCookie.from(JWT_COOKIE_NAME, jwt).httpOnly(true) // Must be httpOnly .sameSite("Lax") // Must be Lax to allow navigation links to Artemis to work - .secure(!isCypress) // Must be secure - TODO - Remove cypress workaround once cypress uses https + .secure(isSecure) // Must be secure .path("/") // Must be "/" to be sent in ALL request .maxAge(duration) // Duration should match the duration of the jwt .build(); // Build cookie From 21e611bde4b188bfc199f6d0c5b4fa397a6bd0f5 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Thu, 14 Sep 2023 10:59:12 +0200 Subject: [PATCH 14/15] Development: Execute architecture tests during java style action (#7160) --- .github/workflows/test.yml | 10 ++++++ build.gradle | 13 +++++-- .../artemis/AbstractArchitectureTest.java | 13 ++++--- ...ava => AuthorizationArchitectureTest.java} | 31 +--------------- .../AuthorizationEndpointTest.java | 36 +++++++++++++++++++ 5 files changed, 65 insertions(+), 38 deletions(-) rename src/test/java/de/tum/in/www1/artemis/authorization/{AuthorizationTest.java => AuthorizationArchitectureTest.java} (73%) create mode 100644 src/test/java/de/tum/in/www1/artemis/authorization/AuthorizationEndpointTest.java diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1cd3fd6437fd..5daa40581328 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -209,6 +209,16 @@ jobs: - name: Java Documentation run: ./gradlew checkstyleMain -x webapp if: success() || failure() + - name: Java Architecture Tests + run: ./gradlew test -DincludeTags='ArchitectureTest' -x webapp + if: success() || failure() + - name: Test Report + uses: dorny/test-reporter@v1 + if: success() || failure() # run this step even if previous step failed + with: + name: Java Architecture Tests + path: build/test-results/test/*.xml + reporter: java-junit client-tests: runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 19452c131ffa..b59c9016479f 100644 --- a/build.gradle +++ b/build.gradle @@ -96,10 +96,17 @@ modernizer { } // Execute the test cases: ./gradlew test - +// Execute only architecture tests: ./gradlew test -DincludeTags='ArchitectureTest' test { - useJUnitPlatform() - exclude "**/*IT*", "**/*IntTest*" + if (System.getProperty("includeTags")) { + useJUnitPlatform { + includeTags System.getProperty("includeTags") + } + } else { + useJUnitPlatform() + exclude "**/*IT*", "**/*IntTest*" + } + testLogging { events "FAILED", "SKIPPED" } diff --git a/src/test/java/de/tum/in/www1/artemis/AbstractArchitectureTest.java b/src/test/java/de/tum/in/www1/artemis/AbstractArchitectureTest.java index 1f0a80ac3352..ded1b8c42ab4 100644 --- a/src/test/java/de/tum/in/www1/artemis/AbstractArchitectureTest.java +++ b/src/test/java/de/tum/in/www1/artemis/AbstractArchitectureTest.java @@ -3,12 +3,14 @@ import static org.assertj.core.api.Assertions.assertThat; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import com.tngtech.archunit.core.domain.JavaClasses; import com.tngtech.archunit.core.importer.ClassFileImporter; import com.tngtech.archunit.core.importer.ImportOption; -public abstract class AbstractArchitectureTest extends AbstractSpringIntegrationBambooBitbucketJiraTest { +@Tag("ArchitectureTest") +public abstract class AbstractArchitectureTest { protected static final String ARTEMIS_PACKAGE = "de.tum.in.www1.artemis"; @@ -20,10 +22,11 @@ public abstract class AbstractArchitectureTest extends AbstractSpringIntegration @BeforeAll static void loadClasses() { - testClasses = new ClassFileImporter().withImportOption(new ImportOption.OnlyIncludeTests()).importPackages(ARTEMIS_PACKAGE); - productionClasses = new ClassFileImporter().withImportOption(new ImportOption.DoNotIncludeTests()).importPackages(ARTEMIS_PACKAGE); - allClasses = new ClassFileImporter().importPackages(ARTEMIS_PACKAGE); - + if (allClasses == null) { + testClasses = new ClassFileImporter().withImportOption(new ImportOption.OnlyIncludeTests()).importPackages(ARTEMIS_PACKAGE); + productionClasses = new ClassFileImporter().withImportOption(new ImportOption.DoNotIncludeTests()).importPackages(ARTEMIS_PACKAGE); + allClasses = new ClassFileImporter().importPackages(ARTEMIS_PACKAGE); + } ensureClassSetsNonEmpty(); ensureAllClassesFound(); } diff --git a/src/test/java/de/tum/in/www1/artemis/authorization/AuthorizationTest.java b/src/test/java/de/tum/in/www1/artemis/authorization/AuthorizationArchitectureTest.java similarity index 73% rename from src/test/java/de/tum/in/www1/artemis/authorization/AuthorizationTest.java rename to src/test/java/de/tum/in/www1/artemis/authorization/AuthorizationArchitectureTest.java index 6974f06d5f6a..49d7c38acc90 100644 --- a/src/test/java/de/tum/in/www1/artemis/authorization/AuthorizationTest.java +++ b/src/test/java/de/tum/in/www1/artemis/authorization/AuthorizationArchitectureTest.java @@ -2,34 +2,16 @@ import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.*; -import java.lang.reflect.InvocationTargetException; -import java.util.Map; -import java.util.stream.Collectors; - import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.method.HandlerMethod; -import org.springframework.web.servlet.mvc.method.RequestMappingInfo; -import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping; import com.tngtech.archunit.lang.ArchRule; import de.tum.in.www1.artemis.AbstractArchitectureTest; import de.tum.in.www1.artemis.security.annotations.*; -/** - * Contains the one automatic test covering all rest endpoints for authorization tests. - */ -class AuthorizationTest extends AbstractArchitectureTest { - - @Autowired - private ApplicationContext applicationContext; - - @Autowired - private AuthorizationTestService authorizationTestService; +class AuthorizationArchitectureTest extends AbstractArchitectureTest { private static final String ARTEMIS_PACKAGE = "de.tum.in.www1.artemis"; @@ -39,17 +21,6 @@ class AuthorizationTest extends AbstractArchitectureTest { private static final String REST_OPEN_PACKAGE = REST_BASE_PACKAGE + ".open"; - @Test - void testEndpoints() throws InvocationTargetException, IllegalAccessException { - var requestMappingHandlerMapping = applicationContext.getBean("requestMappingHandlerMapping", RequestMappingHandlerMapping.class); - Map endpointMap = requestMappingHandlerMapping.getHandlerMethods(); - // Filter out endpoints that should not be tested. - endpointMap = endpointMap.entrySet().stream().filter(entry -> authorizationTestService.validEndpointToTest(entry.getValue(), false)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - authorizationTestService.testEndpoints(endpointMap); - } - @Test void testNoPreAuthorizeOnRestControllers() { ArchRule rule = noClasses().that().areAnnotatedWith(RestController.class).should().beAnnotatedWith(PreAuthorize.class).because( diff --git a/src/test/java/de/tum/in/www1/artemis/authorization/AuthorizationEndpointTest.java b/src/test/java/de/tum/in/www1/artemis/authorization/AuthorizationEndpointTest.java new file mode 100644 index 000000000000..365a07916d17 --- /dev/null +++ b/src/test/java/de/tum/in/www1/artemis/authorization/AuthorizationEndpointTest.java @@ -0,0 +1,36 @@ +package de.tum.in.www1.artemis.authorization; + +import java.util.Map; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.web.method.HandlerMethod; +import org.springframework.web.servlet.mvc.method.RequestMappingInfo; +import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping; + +import de.tum.in.www1.artemis.AbstractSpringIntegrationBambooBitbucketJiraTest; + +/** + * Contains the one automatic test covering all rest endpoints for authorization tests. + */ +class AuthorizationEndpointTest extends AbstractSpringIntegrationBambooBitbucketJiraTest { + + @Autowired + private ApplicationContext applicationContext; + + @Autowired + private AuthorizationTestService authorizationTestService; + + @Test + void testEndpoints() { + var requestMappingHandlerMapping = applicationContext.getBean("requestMappingHandlerMapping", RequestMappingHandlerMapping.class); + Map endpointMap = requestMappingHandlerMapping.getHandlerMethods(); + // Filter out endpoints that should not be tested. + endpointMap = endpointMap.entrySet().stream().filter(entry -> authorizationTestService.validEndpointToTest(entry.getValue(), false)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + authorizationTestService.testEndpoints(endpointMap); + } +} From dfe461c601842f8dfe190b12446f2d6f1d2739d2 Mon Sep 17 00:00:00 2001 From: Florian Gareis Date: Thu, 14 Sep 2023 11:55:12 +0200 Subject: [PATCH 15/15] Development: Remove --legacy-peer-deps and fix postgres file link (#7198) --- docker/artemis-migration-check-postgres.yml | 4 ++-- docker/cypress-E2E-tests-mysql.yml | 2 +- docker/cypress-E2E-tests-postgres.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/artemis-migration-check-postgres.yml b/docker/artemis-migration-check-postgres.yml index 0d801be844d2..5f7cf661c39f 100644 --- a/docker/artemis-migration-check-postgres.yml +++ b/docker/artemis-migration-check-postgres.yml @@ -17,8 +17,8 @@ services: condition: service_healthy postgresql: extends: - file: ./postgresql.yml - service: postgresql + file: ./postgres.yml + service: postgres migration-check: image: alpine container_name: migration-check diff --git a/docker/cypress-E2E-tests-mysql.yml b/docker/cypress-E2E-tests-mysql.yml index 96f9fe67989f..99e96f8b7c87 100644 --- a/docker/cypress-E2E-tests-mysql.yml +++ b/docker/cypress-E2E-tests-mysql.yml @@ -45,7 +45,7 @@ services: environment: CYPRESS_DB_TYPE: "MySQL" SORRY_CYPRESS_PROJECT_ID: "artemis-mysql" - command: sh -c "cd /app/artemis/src/test/cypress && chmod 777 /root && npm ci --legacy-peer-deps && npm run cypress:setup && (npm run cypress:record:mysql & sleep 60 && npm run cypress:record:mysql & wait)" + command: sh -c "cd /app/artemis/src/test/cypress && chmod 777 /root && npm ci && npm run cypress:setup && (npm run cypress:record:mysql & sleep 60 && npm run cypress:record:mysql & wait)" networks: artemis: diff --git a/docker/cypress-E2E-tests-postgres.yml b/docker/cypress-E2E-tests-postgres.yml index a42d2232adc7..0c2ed641ee70 100644 --- a/docker/cypress-E2E-tests-postgres.yml +++ b/docker/cypress-E2E-tests-postgres.yml @@ -46,7 +46,7 @@ services: environment: CYPRESS_DB_TYPE: "Postgres" SORRY_CYPRESS_PROJECT_ID: "artemis-postgres" - command: sh -c "cd /app/artemis/src/test/cypress && chmod 777 /root && npm ci --legacy-peer-deps && npm run cypress:setup && (npm run cypress:record:postgres & sleep 60 && npm run cypress:record:postgres & wait)" + command: sh -c "cd /app/artemis/src/test/cypress && chmod 777 /root && npm ci && npm run cypress:setup && (npm run cypress:record:postgres & sleep 60 && npm run cypress:record:postgres & wait)" networks: artemis: