+ * The data export can be downloaded if its state is either EMAIL_SENT or DOWNLOADED.
+ * The state is EMAIL_SENT if the data export has been created and the user has been notified via email.
+ * The state is DOWNLOADED if the user has downloaded the data export at least once.
+ *
+ * @return true if the data export can be downloaded, false otherwise
+ */
public boolean isDownloadable() {
return this == DOWNLOADED || this == EMAIL_SENT;
}
+ /**
+ * Checks if the data export has been downloaded.
+ *
+ * The data export has been downloaded if its state is either DOWNLOADED or DOWNLOADED_DELETED.
+ * The state is DOWNLOADED if the user has downloaded the data export at least once, but it has not been deleted yet.
+ * The state is DOWNLOADED_DELETED if the user has downloaded the data export at least once, and it has been deleted.
+ *
+ * @return true if the data export has been downloaded, false otherwise
+ */
public boolean hasBeenDownloaded() {
return this == DOWNLOADED || this == DOWNLOADED_DELETED;
}
diff --git a/src/main/java/de/tum/in/www1/artemis/repository/DataExportRepository.java b/src/main/java/de/tum/in/www1/artemis/repository/DataExportRepository.java
index 9315ccd9d78a..4344193419f0 100644
--- a/src/main/java/de/tum/in/www1/artemis/repository/DataExportRepository.java
+++ b/src/main/java/de/tum/in/www1/artemis/repository/DataExportRepository.java
@@ -1,5 +1,6 @@
package de.tum.in.www1.artemis.repository;
+import java.util.List;
import java.util.Set;
import org.springframework.data.jpa.repository.JpaRepository;
@@ -55,12 +56,23 @@ default DataExport findByIdElseThrow(long dataExportId) {
""")
Set findAllToBeDeleted();
+ /**
+ * Find all data exports for the given user ordered by their request date descending.
+ * We use this sorting because this allows us to always get the latest data export without a doing any other calculations.
+ *
+ * This is relevant if more than one data export exists that can be downloaded.
+ * This can happen if the user had requested a data export that was created and the admin requested another data export for the same user that has been created.
+ *
+ * @param userId the id of the user to find the data exports for
+ * @return a list of data exports for the given user ordered by their request date descending
+ */
@Query("""
SELECT dataExport
FROM DataExport dataExport
WHERE dataExport.user.id = :userId
+ ORDER BY dataExport.createdDate DESC
""")
- Set findAllDataExportsByUserId(long userId);
+ List findAllDataExportsByUserIdOrderByRequestDateDesc(long userId);
@Query("""
SELECT dataExport
diff --git a/src/main/java/de/tum/in/www1/artemis/service/FileService.java b/src/main/java/de/tum/in/www1/artemis/service/FileService.java
index 84877534deb6..662e15d40ebf 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/FileService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/FileService.java
@@ -26,7 +26,6 @@
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.lang3.math.NumberUtils;
-import org.apache.pdfbox.io.MemoryUsageSetting;
import org.apache.pdfbox.multipdf.PDFMergerUtility;
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
import org.apache.tomcat.util.http.fileupload.IOUtils;
@@ -1203,7 +1202,7 @@ public Optional mergePdfFiles(List paths, String mergedPdfFileNa
for (String path : paths) {
File file = new File(path);
if (file.exists()) {
- pdfMerger.addSource(new File(path));
+ pdfMerger.addSource(file);
}
}
@@ -1212,7 +1211,7 @@ public Optional mergePdfFiles(List paths, String mergedPdfFileNa
pdfMerger.setDestinationDocumentInformation(pdDocumentInformation);
pdfMerger.setDestinationStream(outputStream);
- pdfMerger.mergeDocuments(MemoryUsageSetting.setupTempFileOnly());
+ pdfMerger.mergeDocuments(null);
}
catch (IOException e) {
diff --git a/src/main/java/de/tum/in/www1/artemis/service/LectureUnitProcessingService.java b/src/main/java/de/tum/in/www1/artemis/service/LectureUnitProcessingService.java
index e73e33e39546..fc04fa5bf323 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/LectureUnitProcessingService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/LectureUnitProcessingService.java
@@ -6,6 +6,7 @@
import javax.validation.constraints.NotNull;
+import org.apache.pdfbox.Loader;
import org.apache.pdfbox.multipdf.Splitter;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentInformation;
@@ -55,18 +56,22 @@ public LectureUnitProcessingService(SlideSplitterService slideSplitterService, F
*/
public List splitAndSaveUnits(LectureUnitInformationDTO lectureUnitInformationDTO, MultipartFile file, Lecture lecture) throws IOException {
- try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); PDDocument document = PDDocument.load(file.getBytes())) {
+ try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); PDDocument document = Loader.loadPDF(file.getBytes())) {
List units = new ArrayList<>();
Splitter pdfSplitter = new Splitter();
for (LectureUnitSplitDTO lectureUnit : lectureUnitInformationDTO.units()) {
+ // make sure output stream doesn't contain old data
+ outputStream.reset();
+
AttachmentUnit attachmentUnit = new AttachmentUnit();
Attachment attachment = new Attachment();
PDDocumentInformation pdDocumentInformation = new PDDocumentInformation();
pdfSplitter.setStartPage(lectureUnit.startPage());
pdfSplitter.setEndPage(lectureUnit.endPage());
- pdfSplitter.setSplitAtPage(lectureUnit.endPage());
+ // split only based on start and end page
+ pdfSplitter.setSplitAtPage(document.getNumberOfPages());
List documentUnits = pdfSplitter.split(document);
pdDocumentInformation.setTitle(lectureUnit.unitName());
@@ -165,7 +170,7 @@ public LectureUnitInformationDTO getSplitUnitData(MultipartFile file) {
* @return The prepared map
*/
private Outline separateIntoUnits(MultipartFile file) throws IOException {
- try (PDDocument document = PDDocument.load(file.getBytes())) {
+ try (PDDocument document = Loader.loadPDF(file.getBytes())) {
Map outlineMap = new HashMap<>();
Splitter pdfSplitter = new Splitter();
PDFTextStripper pdfStripper = new PDFTextStripper();
diff --git a/src/main/java/de/tum/in/www1/artemis/service/ResultService.java b/src/main/java/de/tum/in/www1/artemis/service/ResultService.java
index 46aeaa058d6f..cd624ad5be21 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/ResultService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/ResultService.java
@@ -217,7 +217,7 @@ public List filterFeedbackForClient(Result result) {
* @param result a result of this participation
*/
public void filterSensitiveInformationIfNecessary(final Participation participation, final Result result) {
- this.filterSensitiveInformationIfNecessary(participation, List.of(result));
+ this.filterSensitiveInformationIfNecessary(participation, List.of(result), Optional.empty());
}
/**
@@ -225,29 +225,41 @@ public void filterSensitiveInformationIfNecessary(final Participation participat
*
* @param participation the results belong to.
* @param results collection of results of this participation
+ * @param user the user for which the information should be filtered if it is an empty optional, the currently logged-in user is used
*/
- public void filterSensitiveInformationIfNecessary(final Participation participation, final Collection results) {
+ public void filterSensitiveInformationIfNecessary(final Participation participation, final Collection results, Optional user) {
results.forEach(Result::filterSensitiveInformation);
- if (!authCheckService.isAtLeastTeachingAssistantForExercise(participation.getExercise())) {
- // The test cases marked as after_due_date should only be shown after all
- // students can no longer submit so that no unfair advantage is possible.
- //
- // For course exercises, this applies only to automatic results. For manual ones the instructors
- // are responsible to set an appropriate assessment due date.
- //
- // For exams, we filter sensitive results until the results are published.
- // For test exam exercises, this is the case when the student submitted the test exam.
-
- Exercise exercise = participation.getExercise();
- if (exercise.isExamExercise()) {
- filterSensitiveFeedbacksInExamExercise(participation, results, exercise);
+ if (user.isPresent()) {
+ if (!authCheckService.isAtLeastTeachingAssistantForExercise(participation.getExercise(), user.get())) {
+ filterInformation(participation, results);
}
- else {
- filterSensitiveFeedbackInCourseExercise(participation, results, exercise);
+ }
+ else {
+ if (!authCheckService.isAtLeastTeachingAssistantForExercise(participation.getExercise())) {
+ filterInformation(participation, results);
}
}
}
+ private void filterInformation(Participation participation, Collection results) {
+ // The test cases marked as after_due_date should only be shown after all
+ // students can no longer submit so that no unfair advantage is possible.
+ //
+ // For course exercises, this applies only to automatic results. For manual ones the instructors
+ // are responsible to set an appropriate assessment due date.
+ //
+ // For exams, we filter sensitive results until the results are published.
+ // For test exam exercises, this is the case when the student submitted the test exam.
+
+ Exercise exercise = participation.getExercise();
+ if (exercise.isExamExercise()) {
+ filterSensitiveFeedbacksInExamExercise(participation, results, exercise);
+ }
+ else {
+ filterSensitiveFeedbackInCourseExercise(participation, results, exercise);
+ }
+ }
+
private void filterSensitiveFeedbackInCourseExercise(Participation participation, Collection results, Exercise exercise) {
boolean beforeLatestDueDate = exerciseDateService.isBeforeLatestDueDate(exercise);
boolean participationBeforeDueDate = exerciseDateService.isBeforeDueDate(participation);
diff --git a/src/main/java/de/tum/in/www1/artemis/service/SlideSplitterService.java b/src/main/java/de/tum/in/www1/artemis/service/SlideSplitterService.java
index 87558232de2f..623f49fe8933 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/SlideSplitterService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/SlideSplitterService.java
@@ -8,6 +8,7 @@
import javax.imageio.ImageIO;
import org.apache.commons.io.FilenameUtils;
+import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
@@ -48,7 +49,7 @@ public SlideSplitterService(FileService fileService, SlideRepository slideReposi
public void splitAttachmentUnitIntoSingleSlides(AttachmentUnit attachmentUnit) {
String attachmentPath = fileService.actualPathForPublicPath(attachmentUnit.getAttachment().getLink());
File file = new File(attachmentPath);
- try (PDDocument document = PDDocument.load(file)) {
+ try (PDDocument document = Loader.loadPDF(file)) {
String pdfFilename = file.getName();
splitAttachmentUnitIntoSingleSlides(document, attachmentUnit, pdfFilename);
}
diff --git a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCommunicationDataService.java b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCommunicationDataService.java
index 3b05483bd5e2..639cd17f67ac 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCommunicationDataService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCommunicationDataService.java
@@ -25,6 +25,8 @@
/**
* A service to create the communication data export for users
+ * This includes messages (posts), thread replies (answer posts) and reactions to posts and answer posts
+ * All communication data is exported per course and stored in a CSV file.
*/
@Service
public class DataExportCommunicationDataService {
@@ -67,6 +69,12 @@ public void createCommunicationDataExport(long userId, Path workingDirectory) th
createCommunicationDataExportIfReactionsToAnswerPostsExist(workingDirectory, reactionsToAnswerPostsPerCourse);
}
+ /**
+ * Creates the communication data export for a course if only reactions to answer posts exist
+ *
+ * @param workingDirectory the directory where the export is stored
+ * @param reactionsToAnswerPostsPerCourse the reactions to answer posts grouped by course
+ */
private void createCommunicationDataExportIfReactionsToAnswerPostsExist(Path workingDirectory, Map> reactionsToAnswerPostsPerCourse) throws IOException {
// it can happen that only answer post reactions exist in a course but neither posts, nor answer posts nor reactions to posts
for (var entry : reactionsToAnswerPostsPerCourse.entrySet()) {
@@ -78,6 +86,13 @@ private void createCommunicationDataExportIfReactionsToAnswerPostsExist(Path wor
}
}
+ /**
+ * Creates the communication data export for a course if only reactions to posts (and potentially to answer posts) exist
+ *
+ * @param workingDirectory the directory where the export is stored
+ * @param reactionsToPostsPerCourse the reactions to posts grouped by course
+ * @param reactionsToAnswerPostsPerCourse the reactions to answer posts grouped by course
+ */
private void createCommunicationDataExportIfReactionsToPostsExist(Path workingDirectory, Map> reactionsToPostsPerCourse,
Map> reactionsToAnswerPostsPerCourse) throws IOException {
// it can happen that only reactions exist in a course but no post or answer post
@@ -91,6 +106,14 @@ private void createCommunicationDataExportIfReactionsToPostsExist(Path workingDi
}
}
+ /**
+ * Creates the communication data export for a course if only answer posts (and potentially reactions to post and answer posts) exist
+ *
+ * @param workingDirectory the directory where the export is stored
+ * @param answerPostsPerCourse the answer posts grouped by course
+ * @param reactionsToPostsPerCourse the reactions to posts grouped by course
+ * @param reactionsToAnswerPostsPerCourse the reactions to answer posts grouped by course
+ */
private void createCommunicationDataExportIfAnswerPostsExist(Path workingDirectory, Map> answerPostsPerCourse,
Map> reactionsToPostsPerCourse, Map> reactionsToAnswerPostsPerCourse) throws IOException {
// it can happen that an answer post and reactions exist in a course but no post
@@ -105,6 +128,15 @@ private void createCommunicationDataExportIfAnswerPostsExist(Path workingDirecto
}
}
+ /**
+ * Creates the communication data export for a course if posts exist
+ *
+ * @param workingDirectory the directory where the export is stored
+ * @param postsPerCourse the posts grouped by course
+ * @param answerPostsPerCourse the answer posts grouped by course
+ * @param reactionsToPostsPerCourse the reactions to posts grouped by course
+ * @param reactionsToAnswerPostsPerCourse the reactions to answer posts grouped by course
+ */
private void createCommunicationDataExportIfPostsExist(Path workingDirectory, Map> postsPerCourse, Map> answerPostsPerCourse,
Map> reactionsToPostsPerCourse, Map> reactionsToAnswerPostsPerCourse) throws IOException {
// this covers all cases where at least one post in a course exists
@@ -121,6 +153,15 @@ private void createCommunicationDataExportIfPostsExist(Path workingDirectory, Ma
}
}
+ /**
+ * Creates the actual CSV file containing the communication data for a course
+ *
+ * @param courseDir the directory where the CSV file is stored
+ * @param postsInCourse the posts in the course
+ * @param answerPostsInCourse the answer posts in the course
+ * @param postReactionsInCourse the reactions to posts in the course
+ * @param answerPostReactionsInCourse the reactions to answer posts in the course
+ */
private void createCommunicationDataCsvFile(Path courseDir, List postsInCourse, List answerPostsInCourse, List postReactionsInCourse,
List answerPostReactionsInCourse) throws IOException {
diff --git a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCreationService.java b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCreationService.java
index 6385ad16e8c5..bbc02bf87f73 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCreationService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCreationService.java
@@ -30,6 +30,8 @@
/**
* A service to create data exports for users
+ * This service is responsible for creating the data export, delegating most tasks to the {@link DataExportExerciseCreationService} and {@link DataExportExamCreationService}
+ * and notifying the user about the creation.
*/
@Service
public class DataExportCreationService {
@@ -88,7 +90,7 @@ private DataExport createDataExportWithContent(DataExport dataExport) throws IOE
var userId = dataExport.getUser().getId();
var user = dataExport.getUser();
var workingDirectory = prepareDataExport(dataExport);
- dataExportExerciseCreationService.createExercisesExport(workingDirectory, userId);
+ dataExportExerciseCreationService.createExercisesExport(workingDirectory, user);
dataExportExamCreationService.createExportForExams(userId, workingDirectory);
dataExportCommunicationDataService.createCommunicationDataExport(userId, workingDirectory);
addGeneralUserInformation(user, workingDirectory);
@@ -97,6 +99,17 @@ private DataExport createDataExportWithContent(DataExport dataExport) throws IOE
return finishDataExportCreation(dataExport, dataExportPath);
}
+ /**
+ * Adds a markdown file with the title README.md to the data export.
+ *
+ * This file contains information Art. 15 GDPR requires us to provide to the user.
+ * The file is retrieved from the resources folder.
+ * The file is added to the root of the data export.
+ *
+ * @param workingDirectory the directory in which the data export is created
+ * @throws IOException if the file could not be copied
+ * @throws URISyntaxException if the resource file path is invalid
+ */
private void addReadmeFile(Path workingDirectory) throws IOException, URISyntaxException {
var readmeInDataExportPath = workingDirectory.resolve("README.md");
var readmeTemplatePath = Path.of("templates", "dataexport", "README.md");
@@ -105,6 +118,8 @@ private void addReadmeFile(Path workingDirectory) throws IOException, URISyntaxE
/**
* Creates the data export for the given user.
+ *
+ * This includes creation of the export and notifying the user about the creation.
*
* @param dataExport the data export to be created
* @return true if the export was successful, false otherwise
@@ -129,6 +144,15 @@ public boolean createDataExport(DataExport dataExport) {
return true;
}
+ /**
+ * Handles the case of a failed data export creation.
+ *
+ * This includes setting the state of the data export to failed, notifying the user about the failure and sending an email to the admin with the exception why the export
+ * failed.
+ *
+ * @param dataExport the data export that failed to be created
+ * @param exception the exception that occurred during the creation
+ */
private void handleCreationFailure(DataExport dataExport, Exception exception) {
dataExport.setDataExportState(DataExportState.FAILED);
dataExport = dataExportRepository.save(dataExport);
@@ -141,6 +165,13 @@ private void handleCreationFailure(DataExport dataExport, Exception exception) {
mailService.sendDataExportFailedEmailToAdmin(admin.get(), dataExport, exception);
}
+ /**
+ * Finishes the creation of the data export by setting the file path to the zip file, the state to EMAIL_SENT and the creation finished date.
+ *
+ * @param dataExport the data export whose creation is finished
+ * @param dataExportPath the path to the zip file containing the data export
+ * @return the updated data export from the database
+ */
private DataExport finishDataExportCreation(DataExport dataExport, Path dataExportPath) {
dataExport.setFilePath(dataExportPath.toString());
dataExport.setCreationFinishedDate(ZonedDateTime.now());
@@ -149,6 +180,15 @@ private DataExport finishDataExportCreation(DataExport dataExport, Path dataExpo
return dataExportRepository.save(dataExport);
}
+ /**
+ * Prepares the data export by creating the working directory, scheduling it for deletion and setting the state to IN_CREATION.
+ *
+ * If the path where the data exports are stored does not exist yet, it will be created.
+ *
+ * @param dataExport the data export to be prepared
+ * @return the path to the working directory
+ * @throws IOException if the working directory could not be created
+ */
private Path prepareDataExport(DataExport dataExport) throws IOException {
if (!Files.exists(dataExportsPath)) {
Files.createDirectories(dataExportsPath);
@@ -161,6 +201,14 @@ private Path prepareDataExport(DataExport dataExport) throws IOException {
return workingDirectory;
}
+ /**
+ * Adds the general user information to the data export.
+ *
+ * This includes the login, name, email, and registration number (matriculation number).
+ *
+ * @param user the user for which the information should be added
+ * @param workingDirectory the directory in which the information should be stored
+ */
private void addGeneralUserInformation(User user, Path workingDirectory) throws IOException {
String[] headers = { "login", "name", "email", "registration number" };
CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(headers).build();
@@ -171,11 +219,18 @@ private void addGeneralUserInformation(User user, Path workingDirectory) throws
}
}
+ /**
+ * Creates the zip file containing the data export.
+ *
+ * @param userLogin the login of the user for which the data export was created
+ * @param workingDirectory the directory containing the data export
+ * @return the path to the zip file
+ * @throws IOException if the zip file could not be created
+ */
private Path createDataExportZipFile(String userLogin, Path workingDirectory) throws IOException {
// There should actually never exist more than one data export for a user at a time (once the feature is fully implemented), but to be sure the name is unique, we add the
// current timestamp
return zipFileService.createZipFileWithFolderContent(dataExportsPath.resolve("data-export_" + userLogin + ZonedDateTime.now().toEpochSecond() + ZIP_FILE_EXTENSION),
workingDirectory, null);
-
}
}
diff --git a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExamCreationService.java b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExamCreationService.java
index 126c8c204a61..af105f6ba49d 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExamCreationService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExamCreationService.java
@@ -7,9 +7,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -18,14 +16,18 @@
import org.springframework.stereotype.Service;
import de.tum.in.www1.artemis.domain.Course;
+import de.tum.in.www1.artemis.domain.GradingScale;
import de.tum.in.www1.artemis.domain.ProgrammingExercise;
import de.tum.in.www1.artemis.domain.exam.StudentExam;
+import de.tum.in.www1.artemis.repository.GradingScaleRepository;
import de.tum.in.www1.artemis.repository.StudentExamRepository;
import de.tum.in.www1.artemis.service.exam.ExamService;
import de.tum.in.www1.artemis.web.rest.dto.ExamScoresDTO;
/**
* A service to create the data export for exams the user has participated in.
+ * This includes exercise participations and general information such as working time.
+ * Results are only included if the results are already published.
*/
@Service
public class DataExportExamCreationService {
@@ -38,11 +40,14 @@ public class DataExportExamCreationService {
private final ExamService examService;
- public DataExportExamCreationService(StudentExamRepository studentExamRepository, DataExportExerciseCreationService dataExportExerciseCreationService,
- ExamService examService) {
+ private final GradingScaleRepository gradingScaleRepository;
+
+ public DataExportExamCreationService(StudentExamRepository studentExamRepository, DataExportExerciseCreationService dataExportExerciseCreationService, ExamService examService,
+ GradingScaleRepository gradingScaleRepository) {
this.studentExamRepository = studentExamRepository;
this.dataExportExerciseCreationService = dataExportExerciseCreationService;
this.examService = examService;
+ this.gradingScaleRepository = gradingScaleRepository;
}
/**
@@ -61,14 +66,24 @@ public void createExportForExams(long userId, Path workingDirectory) throws IOEx
var exam = studentExam.getExam();
var examTitle = exam.getSanitizedExamTitle();
var courseDirPath = retrieveCourseDirPath(workingDirectory, exam.getCourse());
- createDirectoryIfNotExistent(courseDirPath);
+ var examsDirPath = courseDirPath.resolve("exams");
+ createDirectoryIfNotExistent(examsDirPath);
var examDirectoryName = EXAM_DIRECTORY_PREFIX + examTitle + "_" + studentExam.getId();
- var examWorkingDir = Files.createDirectories(courseDirPath.resolve(examDirectoryName));
- createStudentExamExport(studentExam, examWorkingDir);
+ var examWorkingDirPath = examsDirPath.resolve(examDirectoryName);
+ createDirectoryIfNotExistent(examWorkingDirPath);
+ createStudentExamExport(studentExam, examWorkingDirPath);
}
}
}
+ /**
+ * Creates the data export for the given student exam.
+ *
+ * This includes extracting all exercise participations, general exam information such as working time, and the results if the results are published.
+ *
+ * @param studentExam the student exam belonging to the user for which the data export should be created
+ * @param examWorkingDir the directory in which the information about the exam should be stored
+ */
private void createStudentExamExport(StudentExam studentExam, Path examWorkingDir) throws IOException {
for (var exercise : studentExam.getExercises()) {
// since the behavior is undefined if multiple student exams for the same exam and student combination exist, the exercise can be null
@@ -76,10 +91,10 @@ private void createStudentExamExport(StudentExam studentExam, Path examWorkingDi
continue;
}
if (exercise instanceof ProgrammingExercise programmingExercise) {
- dataExportExerciseCreationService.createProgrammingExerciseExport(programmingExercise, examWorkingDir, studentExam.getUser().getId());
+ dataExportExerciseCreationService.createProgrammingExerciseExport(programmingExercise, examWorkingDir, studentExam.getUser());
}
else {
- dataExportExerciseCreationService.createNonProgrammingExerciseExport(exercise, examWorkingDir, studentExam.getUser().getId());
+ dataExportExerciseCreationService.createNonProgrammingExerciseExport(exercise, examWorkingDir, studentExam.getUser());
}
}
// leave out the results if the results are not published yet to avoid leaking information through the data export
@@ -89,11 +104,18 @@ private void createStudentExamExport(StudentExam studentExam, Path examWorkingDi
addGeneralExamInformation(studentExam, examWorkingDir);
}
+ /**
+ * Adds the results of the student to the data export.
+ *
+ * @param studentExam the student exam for which the results should be added
+ * @param examWorkingDir the directory in which the results should be stored
+ */
private void addExamScores(StudentExam studentExam, Path examWorkingDir) throws IOException {
var studentExamGrade = examService.getStudentExamGradeForDataExport(studentExam);
var studentResult = studentExamGrade.studentResult();
+ var gradingScale = gradingScaleRepository.findByExamId(studentExam.getExam().getId());
List headers = new ArrayList<>();
- var examResults = getExamResultsStreamToPrint(studentResult, headers);
+ var examResults = getExamResultsStreamToPrint(studentResult, headers, gradingScale);
CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(headers.toArray(new String[0])).build();
try (final CSVPrinter printer = new CSVPrinter(
Files.newBufferedWriter(examWorkingDir.resolve(EXAM_DIRECTORY_PREFIX + studentExam.getId() + "_result" + CSV_FILE_EXTENSION)), csvFormat)) {
@@ -102,21 +124,29 @@ private void addExamScores(StudentExam studentExam, Path examWorkingDir) throws
}
}
- private Stream getExamResultsStreamToPrint(ExamScoresDTO.StudentResult studentResult, List headers) {
+ /**
+ * Returns a stream of the exam results that should be included in the exam results CSV file.
+ *
+ * @param studentResult the result belonging to the student exam
+ * @param headers a list containing the column headers that should be included in the CSV file
+ * @param gradingScaleOptional the optional grading scale of the exam
+ * @return a stream of information that should be included in the exam results CSV file
+ */
+ private Stream getExamResultsStreamToPrint(ExamScoresDTO.StudentResult studentResult, List headers, Optional gradingScaleOptional) {
var builder = Stream.builder();
if (studentResult.overallPointsAchieved() != null) {
builder.add(studentResult.overallPointsAchieved());
headers.add("overall points");
}
- if (studentResult.hasPassed() != null) {
+ if (studentResult.hasPassed() != null && gradingScaleOptional.isPresent()) {
builder.add(studentResult.hasPassed());
headers.add("passed");
}
- if (studentResult.overallGrade() != null) {
+ if (studentResult.overallGrade() != null && gradingScaleOptional.isPresent()) {
builder.add(studentResult.overallGrade());
headers.add("overall grade");
}
- if (studentResult.gradeWithBonus() != null) {
+ if (studentResult.gradeWithBonus() != null && gradingScaleOptional.isPresent()) {
builder.add(studentResult.gradeWithBonus());
headers.add("grade with bonus");
}
@@ -127,15 +157,63 @@ private Stream getExamResultsStreamToPrint(ExamScoresDTO.StudentResult studen
return builder.build();
}
+ /**
+ * Adds general information about the student exam to the data export.
+ *
+ * This includes information such as if the exam was started, if it is a test exam, when it was started, if it was submitted, when it was submitted, the working time, and the
+ * individual end of the working time.
+ *
+ * @param studentExam the student exam for which the information should be added
+ * @param examWorkingDir the directory in which the information should be stored
+ */
private void addGeneralExamInformation(StudentExam studentExam, Path examWorkingDir) throws IOException {
- String[] headers = { "started", "testExam", "started at", "submitted", "submitted at", "working time (in minutes)", "individual end date" };
- CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(headers).build();
+ List headers = new ArrayList<>();
+ var generalExamInformation = getGeneralExamInformationStreamToPrint(studentExam, headers);
+ CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(headers.toArray(new String[0])).build();
try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(examWorkingDir.resolve(EXAM_DIRECTORY_PREFIX + studentExam.getId() + CSV_FILE_EXTENSION)), csvFormat)) {
- printer.printRecord(studentExam.isStarted(), studentExam.isTestExam(), studentExam.getStartedDate(), studentExam.isSubmitted(), studentExam.getSubmissionDate(),
- studentExam.getWorkingTime() / 60, studentExam.getIndividualEndDate());
+ printer.printRecord(generalExamInformation);
printer.flush();
}
}
+ /**
+ * Returns a stream of the general exam information that should be included in the general exam information CSV file.
+ * Do not include information if it is not available, this means null.
+ *
+ * @param studentExam the student exam for which the information should be added
+ * @param headers a list containing the column headers that should be included in the CSV file
+ * @return a stream of information that should be included in the general exam information CSV file
+ */
+ private Stream getGeneralExamInformationStreamToPrint(StudentExam studentExam, List headers) {
+ var builder = Stream.builder();
+ if (studentExam.isStarted() != null) {
+ builder.add(studentExam.isStarted());
+ headers.add("started");
+ }
+ headers.add("test exam");
+ builder.add(studentExam.isTestExam());
+ if (studentExam.getStartedDate() != null) {
+ builder.add(studentExam.getStartedDate());
+ headers.add("started at");
+ }
+ if (studentExam.isSubmitted() != null) {
+ builder.add(studentExam.isSubmitted());
+ headers.add("submitted");
+ }
+ if (studentExam.getSubmissionDate() != null) {
+ builder.add(studentExam.getSubmissionDate());
+ headers.add("submitted at");
+ }
+ if (studentExam.getWorkingTime() != null) {
+ builder.add(studentExam.getWorkingTime() / 60);
+ headers.add("working time (in minutes)");
+ }
+ if (studentExam.getIndividualEndDate() != null) {
+ builder.add(studentExam.getIndividualEndDate());
+ headers.add("individual end date");
+ }
+ return builder.build();
+ }
+
}
diff --git a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExerciseCreationService.java b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExerciseCreationService.java
index f29034fbf690..73a08df82c95 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExerciseCreationService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExerciseCreationService.java
@@ -1,7 +1,9 @@
package de.tum.in.www1.artemis.service.dataexport;
import static de.tum.in.www1.artemis.service.dataexport.DataExportQuizExerciseCreationService.TXT_FILE_EXTENSION;
+import static de.tum.in.www1.artemis.service.dataexport.DataExportUtil.createDirectoryIfNotExistent;
import static de.tum.in.www1.artemis.service.dataexport.DataExportUtil.retrieveCourseDirPath;
+import static de.tum.in.www1.artemis.service.util.RoundingUtil.roundToNDecimalPlaces;
import java.io.File;
import java.io.IOException;
@@ -20,7 +22,7 @@
import org.springframework.stereotype.Service;
import de.tum.in.www1.artemis.domain.*;
-import de.tum.in.www1.artemis.domain.enumeration.ComplaintType;
+import de.tum.in.www1.artemis.domain.enumeration.*;
import de.tum.in.www1.artemis.domain.metis.AnswerPost;
import de.tum.in.www1.artemis.domain.modeling.ModelingSubmission;
import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseStudentParticipation;
@@ -29,20 +31,23 @@
import de.tum.in.www1.artemis.repository.ComplaintRepository;
import de.tum.in.www1.artemis.repository.ExerciseRepository;
import de.tum.in.www1.artemis.repository.plagiarism.PlagiarismCaseRepository;
-import de.tum.in.www1.artemis.service.ExerciseDateService;
-import de.tum.in.www1.artemis.service.FileService;
+import de.tum.in.www1.artemis.service.*;
import de.tum.in.www1.artemis.service.connectors.apollon.ApollonConversionService;
import de.tum.in.www1.artemis.service.programming.ProgrammingExerciseExportService;
import de.tum.in.www1.artemis.web.rest.dto.RepositoryExportOptionsDTO;
/**
- * A service to create the data export for exercise participations of the user
+ * A service to create the data export for exercise participations of the user.
+ * It is responsible for creating the export for programming exercises and modeling, text, and file upload exercises.
+ * For quiz exercises it delegates the creation of the export to {@link DataExportQuizExerciseCreationService}.
*/
@Service
public class DataExportExerciseCreationService {
private static final String PDF_FILE_EXTENSION = ".pdf";
+ private static final String EXERCISE_PREFIX = "exercise_";
+
static final String CSV_FILE_EXTENSION = ".csv";
private final Path repoClonePath;
@@ -64,10 +69,14 @@ public class DataExportExerciseCreationService {
private final ExerciseRepository exerciseRepository;
+ private final ResultService resultService;
+
+ private final AuthorizationCheckService authCheckService;
+
public DataExportExerciseCreationService(@Value("${artemis.repo-download-clone-path}") Path repoClonePath, FileService fileService,
ProgrammingExerciseExportService programmingExerciseExportService, DataExportQuizExerciseCreationService dataExportQuizExerciseCreationService,
PlagiarismCaseRepository plagiarismCaseRepository, Optional apollonConversionService, ComplaintRepository complaintRepository,
- ExerciseRepository exerciseRepository) {
+ ExerciseRepository exerciseRepository, ResultService resultService, AuthorizationCheckService authCheckService) {
this.fileService = fileService;
this.programmingExerciseExportService = programmingExerciseExportService;
this.dataExportQuizExerciseCreationService = dataExportQuizExerciseCreationService;
@@ -76,53 +85,57 @@ public DataExportExerciseCreationService(@Value("${artemis.repo-download-clone-p
this.complaintRepository = complaintRepository;
this.exerciseRepository = exerciseRepository;
this.repoClonePath = repoClonePath;
+ this.resultService = resultService;
+ this.authCheckService = authCheckService;
}
/**
* Creates the export for all exercises the user participated in.
*
* @param workingDirectory the directory the export should be created in
- * @param userId the id of the user that requested the export
+ * @param user the user for which the export should be created
* @throws IOException if an error occurs while accessing the file system
*/
- public void createExercisesExport(Path workingDirectory, long userId) throws IOException {
+ public void createExercisesExport(Path workingDirectory, User user) throws IOException {
// retrieve all exercises as we cannot retrieve the exercises by course because a user might have participated in a course they are no longer a member of (they have
// unenrolled)
- var allExerciseParticipations = exerciseRepository.getAllExercisesUserParticipatedInWithEagerParticipationsSubmissionsResultsFeedbacksByUserId(userId);
+ var allExerciseParticipations = exerciseRepository.getAllExercisesUserParticipatedInWithEagerParticipationsSubmissionsResultsFeedbacksByUserId(user.getId());
var exerciseParticipationsPerCourse = allExerciseParticipations.stream().collect(Collectors.groupingBy(Exercise::getCourseViaExerciseGroupOrCourseMember));
for (var entry : exerciseParticipationsPerCourse.entrySet()) {
var course = entry.getKey();
Path courseDir = retrieveCourseDirPath(workingDirectory, course);
var exercises = entry.getValue();
+ Path exercisesDir = courseDir.resolve("exercises");
if (!exercises.isEmpty()) {
- Files.createDirectory(courseDir);
+ createDirectoryIfNotExistent(exercisesDir);
}
for (var exercise : exercises) {
if (exercise instanceof ProgrammingExercise programmingExercise) {
- createProgrammingExerciseExport(programmingExercise, courseDir, userId);
+ createProgrammingExerciseExport(programmingExercise, exercisesDir, user);
}
else {
- createNonProgrammingExerciseExport(exercise, courseDir, userId);
+ createNonProgrammingExerciseExport(exercise, exercisesDir, user);
}
}
}
}
/**
- * Creates an export for a given programming exercise. Includes submission information, the repository from the VCS and potential plagiarism cases.
+ * Creates an export for a given programming exercise.
+ *
+ * Includes submission information, the repository from the VCS and potential plagiarism cases.
*
* @param programmingExercise the programming exercise for which the export should be created
- * @param courseDir the directory that is used for the course the exercise belongs to
- * @param userId the id of the user that requested the export
+ * @param exercisesDir the directory where all exercises of a course should be stored
+ * @param user the user for which the export should be created
* @throws IOException if an error occurs while accessing the file system
*/
-
- public void createProgrammingExerciseExport(ProgrammingExercise programmingExercise, Path courseDir, long userId) throws IOException {
- Path exerciseDir = courseDir.resolve(programmingExercise.getSanitizedExerciseTitle());
+ public void createProgrammingExerciseExport(ProgrammingExercise programmingExercise, Path exercisesDir, User user) throws IOException {
+ Path exerciseDir = exercisesDir.resolve(EXERCISE_PREFIX + programmingExercise.getSanitizedExerciseTitle());
if (!Files.exists(exerciseDir)) {
Files.createDirectory(exerciseDir);
}
- createSubmissionsResultsExport(programmingExercise, exerciseDir);
+ createSubmissionsResultsExport(programmingExercise, exerciseDir, user);
RepositoryExportOptionsDTO repositoryExportOptions = new RepositoryExportOptionsDTO();
repositoryExportOptions.setExportAllParticipants(false);
repositoryExportOptions.setAnonymizeRepository(false);
@@ -141,7 +154,7 @@ public void createProgrammingExerciseExport(ProgrammingExercise programmingExerc
programmingExerciseExportService.exportStudentRepositories(programmingExercise, listOfProgrammingExerciseParticipations, repositoryExportOptions, tempRepoWorkingDir,
exerciseDir, Collections.synchronizedList(new ArrayList<>()));
- createPlagiarismCaseInfoExport(programmingExercise, exerciseDir, userId);
+ createPlagiarismCaseInfoExport(programmingExercise, exerciseDir, user.getId());
}
@@ -150,21 +163,35 @@ public void createProgrammingExerciseExport(ProgrammingExercise programmingExerc
*
* @param exercise the exercise for which the export should be created
* @param courseDir the directory that is used for the course the exercise belongs to
- * @param userId the id of the user that requested the export
+ * @param user the user for which the export should be created
* @throws IOException if an error occurs while accessing the file system
*/
- public void createNonProgrammingExerciseExport(Exercise exercise, Path courseDir, long userId) throws IOException {
- Path exercisePath = courseDir.resolve(exercise.getSanitizedExerciseTitle());
+ public void createNonProgrammingExerciseExport(Exercise exercise, Path courseDir, User user) throws IOException {
+ Path exercisePath = courseDir.resolve(EXERCISE_PREFIX + exercise.getSanitizedExerciseTitle());
if (!Files.exists(exercisePath)) {
Files.createDirectory(exercisePath);
}
- createSubmissionsResultsExport(exercise, exercisePath);
- createPlagiarismCaseInfoExport(exercise, exercisePath, userId);
+ createSubmissionsResultsExport(exercise, exercisePath, user);
+ createPlagiarismCaseInfoExport(exercise, exercisePath, user.getId());
}
- private void createSubmissionsResultsExport(Exercise exercise, Path exerciseDir) throws IOException {
- boolean includeResults = exercise.isExamExercise() && exercise.getExamViaExerciseGroupOrCourseMember().resultsPublished()
- || exercise.isCourseExercise() && ExerciseDateService.isAfterAssessmentDueDate(exercise);
+ /**
+ * Creates the export for the submission of the user to the given exercise.
+ *
+ * Includes the submission information and the submission content and the results if the results are
+ * published.
+ * For quiz exercises it delegates the creation of the export to {@link DataExportQuizExerciseCreationService}.
+ *
+ * @param exercise the exercise for which the export should be created
+ * @param exerciseDir the directory in which the export should be created
+ * @param user the user for which the export should be created
+ */
+ private void createSubmissionsResultsExport(Exercise exercise, Path exerciseDir, User user) throws IOException {
+ // quizzes do not have an assessment due date, so we need to check if they have ended according to their due date
+ boolean isInstructor = authCheckService.isAtLeastInstructorForExercise(exercise, user);
+ boolean includeResults = (exercise.isExamExercise() && exercise.getExamViaExerciseGroupOrCourseMember().resultsPublished())
+ || (exercise.isCourseExercise() && ExerciseDateService.isAfterAssessmentDueDate(exercise) && !(exercise instanceof QuizExercise))
+ || (exercise.isCourseExercise() && exercise instanceof QuizExercise quizExercise && quizExercise.isQuizEnded()) || isInstructor;
for (var participation : exercise.getStudentParticipations()) {
for (var submission : participation.getSubmissions()) {
createSubmissionCsvFile(submission, exerciseDir);
@@ -180,13 +207,22 @@ else if (submission instanceof ModelingSubmission modelingSubmission) {
else if (submission instanceof QuizSubmission) {
dataExportQuizExerciseCreationService.createQuizAnswersExport((QuizExercise) exercise, participation, exerciseDir, includeResults);
}
- if (includeResults) {
- createResultsAndComplaintFiles(submission, exerciseDir);
+ // for a programming exercise, we want to include the results that are visible before the assessment due date
+ if (includeResults || exercise instanceof ProgrammingExercise) {
+ boolean programmingExerciseBeforeAssessmentDueDate = exercise instanceof ProgrammingExercise && !ExerciseDateService.isAfterAssessmentDueDate(exercise);
+ createResultsAndComplaintFiles(submission, exerciseDir, user, programmingExerciseBeforeAssessmentDueDate, isInstructor);
}
}
}
}
+ /**
+ * Stores the modeling submission as pdf if the apollon profile is active and the apollon conversion service works, otherwise stores it as json file.
+ *
+ * @param modelingSubmission the modeling submission for which the content should be stored
+ * @param outputDir the directory in which the content should be stored
+ * @throws IOException if the file cannot be written
+ */
private void storeModelingSubmissionContent(ModelingSubmission modelingSubmission, Path outputDir) throws IOException {
if (modelingSubmission.getModel() == null) {
log.warn("Cannot include modeling submission content in data export because content is null for submission with id: {}", modelingSubmission.getId());
@@ -202,12 +238,22 @@ private void storeModelingSubmissionContent(ModelingSubmission modelingSubmissio
try (var modelAsPdf = apollonConversionService.get().convertModel(modelingSubmission.getModel())) {
Files.write(outputDir.resolve(fileName + PDF_FILE_EXTENSION), modelAsPdf.readAllBytes());
}
- catch (IOException e) {
+ catch (Exception e) {
log.warn("Failed to include the model as pdf, going to include it as plain JSON file.");
addModelJsonWithExplanationHowToView(modelingSubmission.getModel(), outputDir, fileName);
}
}
+ /**
+ * Stores the given model as json file and adds a markdown file with an explanation how to view the model.
+ *
+ * Used if the Apollon Conversion Service is not available or an error occurs while using it.
+ *
+ * @param model the model belonging to the submission as JSON string
+ * @param outputDir the directory in which the content should be stored
+ * @param fileName the file name of the JSON file
+ * @throws IOException if the file cannot be written
+ */
private void addModelJsonWithExplanationHowToView(String model, Path outputDir, String fileName) throws IOException {
Files.writeString(outputDir.resolve(fileName + ".json"), model);
String explanation = """
@@ -216,6 +262,13 @@ private void addModelJsonWithExplanationHowToView(String model, Path outputDir,
Files.writeString(outputDir.resolve("view_model.md"), explanation);
}
+ /**
+ * Stores the text submission content as txt file.
+ *
+ * @param textSubmission the text submission for which the content should be stored
+ * @param outputDir the directory in which the content should be stored
+ * @throws IOException if the file cannot be written
+ */
private void storeTextSubmissionContent(TextSubmission textSubmission, Path outputDir) throws IOException {
// text can be null which leads to an exception
if (textSubmission.getText() != null) {
@@ -226,32 +279,56 @@ private void storeTextSubmissionContent(TextSubmission textSubmission, Path outp
}
}
- private void createResultsAndComplaintFiles(Submission submission, Path outputDir) throws IOException {
+ /**
+ * Creates a txt file containing the results with the score, the number of passed test cases if it is a programming exercise
+ * and the feedbacks (both manual and automatic).
+ *
+ * @param submission the submission for which the results should be stored
+ * @param outputDir the directory in which the results should be stored
+ * @param user the user for which the export should be created
+ * @param programmingExerciseBeforeAssessmentDueDate whether the programming exercise is before the assessment due date
+ * @param isInstructor whether the user is an instructor in the course the exercise belongs to
+ * @throws IOException if the file cannot be written
+ */
+ private void createResultsAndComplaintFiles(Submission submission, Path outputDir, User user, boolean programmingExerciseBeforeAssessmentDueDate, boolean isInstructor)
+ throws IOException {
StringBuilder resultScoreAndFeedbacks = new StringBuilder();
for (var result : submission.getResults()) {
if (result != null) {
+ // Do not include the results if the assessment due date is in the future and the assessment is not automatic and the user is not an instructor
+ // We only consider programming exercises here because for other exercises this method is not called if the assessment due date is in the future
+ if (programmingExerciseBeforeAssessmentDueDate && result.getAssessmentType() != AssessmentType.AUTOMATIC && !isInstructor) {
+ continue;
+ }
+ resultService.filterSensitiveInformationIfNecessary(submission.getParticipation(), List.of(result), Optional.of(user));
var score = result.getScore();
if (score != null) {
resultScoreAndFeedbacks.append("Score of submission: ").append(score).append("%").append(" ")
- .append(score * submission.getParticipation().getExercise().getMaxPoints() / 100).append(" Points").append("\n");
+ .append(roundToNDecimalPlaces(score * submission.getParticipation().getExercise().getMaxPoints() / 100, 2)).append(" Points").append("\n");
}
if (submission instanceof ProgrammingSubmission && result.getPassedTestCaseCount() != null && result.getTestCaseCount() != null && result.getTestCaseCount() > 0) {
resultScoreAndFeedbacks.append("Passed test cases: ").append(result.getPassedTestCaseCount()).append("/").append(result.getTestCaseCount()).append("\n");
}
+ if (submission instanceof ProgrammingSubmission programmingSubmission && programmingSubmission.isBuildFailed()) {
+ resultScoreAndFeedbacks.append("Build failed").append("\n");
+ }
for (var feedback : result.getFeedbacks()) {
- resultScoreAndFeedbacks.append("- Feedback: ");
- // null if it's manual feedback
- if (feedback.getText() != null) {
- resultScoreAndFeedbacks.append(feedback.getText()).append("\t");
+ if (feedback != null) {
+ resultScoreAndFeedbacks.append("- Feedback: ");
+
+ // null if it's manual feedback
+ if (feedback.getText() != null) {
+ resultScoreAndFeedbacks.append(feedback.getText()).append("\t");
+ }
+ // null if the test case passes
+ if (feedback.getDetailText() != null) {
+ resultScoreAndFeedbacks.append(feedback.getDetailText()).append("\t");
+ }
+ if (feedback.getCredits() != null) {
+ resultScoreAndFeedbacks.append(feedback.getCredits());
+ }
+ resultScoreAndFeedbacks.append("\n");
}
- // null if the test case passes
- if (feedback.getDetailText() != null) {
- resultScoreAndFeedbacks.append(feedback.getDetailText()).append("\t");
- }
- if (feedback.getCredits() != null) {
- resultScoreAndFeedbacks.append(feedback.getCredits());
- }
- resultScoreAndFeedbacks.append("\n");
}
Files.writeString(outputDir.resolve("submission_" + submission.getId() + "_result_" + result.getId() + TXT_FILE_EXTENSION), resultScoreAndFeedbacks);
}
@@ -263,6 +340,15 @@ private void createResultsAndComplaintFiles(Submission submission, Path outputDi
}
}
+ /**
+ * Creates a CSV file containing the complaint data.
+ *
+ * Complaint can be either a complaint or a more feedback request.
+ *
+ * @param complaint the complaint for which the data should be stored
+ * @param outputDir the directory in which the data should be stored
+ * @throws IOException if the file cannot be written
+ */
private void addComplaintData(Complaint complaint, Path outputDir) throws IOException {
List headers = new ArrayList<>();
var dataStreamBuilder = Stream.builder();
@@ -292,6 +378,14 @@ private void addComplaintData(Complaint complaint, Path outputDir) throws IOExce
}
}
+ /**
+ * Creates a CSV file containing the plagiarism case information.
+ *
+ * @param exercise the exercise for which the plagiarism case information should be stored
+ * @param exercisePath the directory in which the plagiarism case information should be stored
+ * @param userId the id of the user that requested the export and that is involved in the plagiarism case
+ * @throws IOException if the file cannot be written
+ */
private void createPlagiarismCaseInfoExport(Exercise exercise, Path exercisePath, long userId) throws IOException {
var plagiarismCaseOptional = plagiarismCaseRepository.findByStudentIdAndExerciseIdWithPostAndAnswerPost(userId, exercise.getId());
List headers = new ArrayList<>();
@@ -329,6 +423,14 @@ else if (plagiarismCase.getVerdict() == PlagiarismVerdict.WARNING) {
}
}
+ /**
+ * Copies the file upload submission file to the data export working directory if it still exists.
+ *
+ * @param submissionFilePath the path to the file upload submission file
+ * @param outputDir the directory to which the file should be copied
+ * @param fileUploadSubmission the file upload submission for which the file should be copied
+ * @throws IOException if the file cannot be copied
+ */
private void copyFileUploadSubmissionFile(String submissionFilePath, Path outputDir, FileUploadSubmission fileUploadSubmission) throws IOException {
try {
FileUtils.copyDirectory(new File(submissionFilePath), outputDir.toFile());
@@ -339,12 +441,28 @@ private void copyFileUploadSubmissionFile(String submissionFilePath, Path output
}
}
+ /**
+ * Adds a markdown file to the data export working directory that informs the user that the file for the file upload submission no longer exists.
+ *
+ * @param outputDir the directory in which the file should be stored
+ * @param fileUploadSubmission the file upload submission for which the file should be stored
+ * @throws IOException if the file cannot be written
+ */
private void addInfoThatFileForFileUploadSubmissionNoLongerExists(Path outputDir, FileUploadSubmission fileUploadSubmission) throws IOException {
var exercise = fileUploadSubmission.getParticipation().getExercise();
Files.writeString(outputDir.resolve("submission_file_no_longer_exists.md"),
String.format("Your submitted file for the exercise %s no longer exists on the file system.", exercise));
}
+ /**
+ * Creates a CSV file containing the submission information.
+ *
+ * This includes the id, the submission date and the commit hash if it is a programming exercise.
+ *
+ * @param submission the submission for which the information should be stored
+ * @param outputPath the directory in which the information should be stored
+ * @throws IOException if the file cannot be written
+ */
private void createSubmissionCsvFile(Submission submission, Path outputPath) throws IOException {
List headers = new ArrayList<>(List.of("id", "submissionDate"));
if (submission instanceof ProgrammingSubmission) {
@@ -360,6 +478,14 @@ private void createSubmissionCsvFile(Submission submission, Path outputPath) thr
}
}
+ /**
+ * Returns a stream of the submission information that should be included in the CSV file.
+ *
+ * This includes the id, the submission date and the commit hash if it is a programming exercise.
+ *
+ * @param submission the submission for which the information should be stored
+ * @return a stream of the submission information that should be included in the CSV file
+ */
private Stream getSubmissionStreamToPrint(Submission submission) {
var builder = Stream.builder();
builder.add(submission.getId()).add(submission.getSubmissionDate());
diff --git a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportQuizExerciseCreationService.java b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportQuizExerciseCreationService.java
index bcc98cd6dedb..6b51e2ecf0ff 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportQuizExerciseCreationService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportQuizExerciseCreationService.java
@@ -16,7 +16,10 @@
import de.tum.in.www1.artemis.service.DragAndDropQuizAnswerConversionService;
/**
- * A service to create the data export for quiz exercise participations
+ * A service to create the data export for quiz exercise participations.
+ * This includes creating a pdf highlighting the submitted answers for drag and drop questions and
+ * txt files containing the submitted answers for multiple choice and short answer questions.
+ * Additionally, the results can be included in the export if the due date is over.
*/
@Service
public class DataExportQuizExerciseCreationService {
@@ -80,6 +83,13 @@ else if (submittedAnswer instanceof MultipleChoiceSubmittedAnswer multipleChoice
}
+ /**
+ * Creates a txt file containing the submitted answers for a multiple choice question and information if the answer was correct or not if includeResults is true.
+ *
+ * @param multipleChoiceSubmittedAnswer the submitted answer to a multiple choice question that should be included in the export
+ * @param includeResults true if the results should be included in the export (if the assessment due date or result publication date is over)
+ * @return the content for the txt file as a string
+ */
private String createExportForMultipleChoiceAnswerQuestion(MultipleChoiceSubmittedAnswer multipleChoiceSubmittedAnswer, boolean includeResults) {
StringBuilder stringBuilder = new StringBuilder();
MultipleChoiceQuestion question = (MultipleChoiceQuestion) multipleChoiceSubmittedAnswer.getQuizQuestion();
@@ -105,6 +115,15 @@ private String createExportForMultipleChoiceAnswerQuestion(MultipleChoiceSubmitt
return stringBuilder.toString();
}
+ /**
+ * Adds an explanation to the answer option if no result should be included in the export.
+ *
+ * The explanation contains information if the answer option was selected or not or if it is invalid.
+ *
+ * @param multipleChoiceSubmittedAnswer the submitted answer to a multiple choice question that should be included in the export
+ * @param stringBuilder the string builder user to create the txt file content
+ * @param answerOption the answer option for which the explanation should be added
+ */
private void addExplanationToAnswerOptionWithoutResult(MultipleChoiceSubmittedAnswer multipleChoiceSubmittedAnswer, StringBuilder stringBuilder, AnswerOption answerOption) {
if (answerOption.isInvalid()) {
stringBuilder.append("Invalid answer option: ");
@@ -117,6 +136,15 @@ else if (multipleChoiceSubmittedAnswer.getSelectedOptions().contains(answerOptio
}
}
+ /**
+ * Adds an explanation to the answer option if a result should be included in the export.
+ *
+ * The explanation contains information if the answer option was selected or not or it is invalid and if the answer option is correct or not.
+ *
+ * @param multipleChoiceSubmittedAnswer the submitted answer to a multiple choice question that should be included in the export
+ * @param stringBuilder the string builder user to create the txt file content
+ * @param answerOption the answer option for which the explanation should be added
+ */
private void addExplanationToAnswerOptionWithResult(MultipleChoiceSubmittedAnswer multipleChoiceSubmittedAnswer, StringBuilder stringBuilder, AnswerOption answerOption) {
if (answerOption.isInvalid()) {
stringBuilder.append("Invalid answer option: ");
@@ -135,6 +163,13 @@ else if (!answerOption.isIsCorrect() && !multipleChoiceSubmittedAnswer.getSelect
}
}
+ /**
+ * Creates a txt file containing the submitted answers for a short answer question and information if the answer was correct or not if includeResults is true.
+ *
+ * @param shortAnswerSubmittedAnswer the submitted answer to a short answer question that should be included in the export
+ * @param includeResults true if the results should be included in the export (if the assessment due date or result publication date is over)
+ * @return the content for the txt file as a string
+ */
private String createExportForShortAnswerQuestion(ShortAnswerSubmittedAnswer shortAnswerSubmittedAnswer, boolean includeResults) {
StringBuilder stringBuilder = new StringBuilder();
ShortAnswerQuestion question = (ShortAnswerQuestion) shortAnswerSubmittedAnswer.getQuizQuestion();
@@ -145,6 +180,14 @@ private String createExportForShortAnswerQuestion(ShortAnswerSubmittedAnswer sho
return replaceSpotWithSubmittedAnswer(shortAnswerSubmittedAnswer, stringBuilder, includeResults);
}
+ /**
+ * Replaces the spots (the gaps that indicate where an answer should be entered) in the text of a short answer question with the submitted answers.
+ *
+ * @param shortAnswerSubmittedAnswer the submitted answer to a short answer question that should be included in the export
+ * @param submittedAnswer the string builder user to create the txt file content
+ * @param includeResults true if the results should be included in the export (if the assessment due date or result publication date is over)
+ * @return the string containing the question text with the answers of the user
+ */
private String replaceSpotWithSubmittedAnswer(ShortAnswerSubmittedAnswer shortAnswerSubmittedAnswer, StringBuilder submittedAnswer, boolean includeResults) {
var spotToSubmittedTextMap = buildMapFromSpotsToSubmittedAnswers(shortAnswerSubmittedAnswer);
submittedAnswer.append("Your answer: ").append("\n");
@@ -160,30 +203,39 @@ private String replaceSpotWithSubmittedAnswer(ShortAnswerSubmittedAnswer shortAn
return submittedAnswer.toString();
}
+ /**
+ * Adds the submitted answer to the string builder if the answer is correct or incorrect.
+ *
+ * @param submittedAnswer the string builder user to create the txt file content
+ * @param includeResults true if the results should be included in the export (if the assessment due date or result publication date is over)
+ * @param submittedText the question text of a short answer question that should be included in the export
+ * @param pattern the pattern used to find the spot in the question text
+ * @param matcher the matcher used to find the spot in the question text
+ * @param replacement the string builder used to create the replacement (the submitted answer text) for the spot
+ * @return the matcher used to find the next spot in the question text
+ */
private Matcher addSubmittedAnswerWithResult(StringBuilder submittedAnswer, boolean includeResults, ShortAnswerSubmittedText submittedText, Pattern pattern, Matcher matcher,
StringBuilder replacement) {
int start = matcher.start();
int end = matcher.end();
- if (submittedText.isIsCorrect() != null && submittedText.isIsCorrect()) {
- replacement.append(submittedText.getText());
- if (includeResults) {
- replacement.append(" (Correct)");
- }
+ replacement.append(submittedText.getText());
+ if (submittedText.isIsCorrect() != null && submittedText.isIsCorrect() && includeResults) {
+ replacement.append(" (Correct)");
}
- else if (submittedText.isIsCorrect() != null && !submittedText.isIsCorrect()) {
- replacement.append(submittedText.getText());
- if (includeResults) {
- replacement.append(" (Incorrect)");
- }
- else {
- replacement.append(submittedText.getText());
- }
- submittedAnswer.replace(start, end, replacement.toString());
- matcher = pattern.matcher(submittedAnswer);
+ else if (submittedText.isIsCorrect() != null && !submittedText.isIsCorrect() && includeResults) {
+ replacement.append(" (Incorrect)");
}
+ submittedAnswer.replace(start, end, replacement.toString());
+ matcher = pattern.matcher(submittedAnswer);
return matcher;
}
+ /**
+ * Builds a map from the spots (the gaps that indicate where an answer should be entered) in the text of a short answer question to the submitted answers.
+ *
+ * @param shortAnswerSubmittedAnswer the submitted answer to a short answer question that should be included in the export
+ * @return a map from the spots (represented as the string in text) to the submitted answers
+ */
private Map buildMapFromSpotsToSubmittedAnswers(ShortAnswerSubmittedAnswer shortAnswerSubmittedAnswer) {
Map spotsToSubmittedAnswers = new HashMap<>();
for (var submittedText : shortAnswerSubmittedAnswer.getSubmittedTexts()) {
diff --git a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportService.java b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportService.java
index c23d99dc241f..a067a4eeb301 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportService.java
@@ -22,10 +22,13 @@
import de.tum.in.www1.artemis.service.FileService;
import de.tum.in.www1.artemis.web.rest.dto.DataExportDTO;
import de.tum.in.www1.artemis.web.rest.dto.RequestDataExportDTO;
+import de.tum.in.www1.artemis.web.rest.errors.AccessForbiddenException;
import de.tum.in.www1.artemis.web.rest.errors.InternalServerErrorException;
/**
* Service Implementation for managing the data export in accordance with Art. 15 GDPR.
+ * This service is responsible for downloading, deleting data exports and checking if a data export can be requested.
+ * For creating data exports, see {@link DataExportCreationService}.
*/
@Service
public class DataExportService {
@@ -109,8 +112,7 @@ public Resource downloadDataExport(DataExport dataExport) {
public DataExportDTO canDownloadAnyDataExport() {
var noDataExport = new DataExportDTO(null, null, null, null);
var user = userRepository.getUser();
- var dataExportsFromUser = dataExportRepository.findAllDataExportsByUserId(user.getId());
- Optional latestDataExport = dataExportsFromUser.stream().max(Comparator.comparing(DataExport::getCreatedDate));
+ var dataExportsFromUser = dataExportRepository.findAllDataExportsByUserIdOrderByRequestDateDesc(user.getId());
if (dataExportsFromUser.isEmpty()) {
return noDataExport;
}
@@ -120,10 +122,21 @@ public DataExportDTO canDownloadAnyDataExport() {
return new DataExportDTO(dataExport.getId(), dataExport.getDataExportState(), dataExport.getCreatedDate().atZone(ZoneId.systemDefault()), nextRequestDate);
}
}
- return new DataExportDTO(null, latestDataExport.get().getDataExportState(), latestDataExport.get().getCreatedDate().atZone(ZoneId.systemDefault()),
- retrieveNextRequestDate(latestDataExport.get()));
+ var latestDataExport = dataExportsFromUser.get(0);
+ return new DataExportDTO(null, latestDataExport.getDataExportState(), latestDataExport.getCreatedDate().atZone(ZoneId.systemDefault()),
+ retrieveNextRequestDate(latestDataExport));
}
+ /**
+ * Calculates the next date when the user can request a data export.
+ *
+ * This is the date when the last data export was requested (stored in the createdDate) + the constant DAYS_BETWEEN_DATA_EXPORTS.
+ * By default, DAYS_BETWEEN_DATA_EXPORTS is set to 14 days.
+ * This can be changed by setting the property artemis.data-export.days-between-data-exports in the application.yml file.
+ *
+ * @param dataExport the data export for which the next request date should be calculated
+ * @return the next date when the user can request a data export
+ */
@NotNull
private ZonedDateTime retrieveNextRequestDate(DataExport dataExport) {
return dataExport.getCreatedDate().atZone(ZoneId.systemDefault()).plusDays(DAYS_BETWEEN_DATA_EXPORTS);
@@ -148,4 +161,18 @@ public void deleteDataExportAndSetDataExportState(DataExport dataExport) {
dataExportRepository.save(dataExport);
}
+ /**
+ * Checks if the data export can be downloaded.
+ *
+ * The data export can be downloaded if its state is either EMAIL_SENT or DOWNLOADED.
+ *
+ * @param dataExport the data export to check
+ * @throws AccessForbiddenException if the data export is not in a downloadable state
+ */
+ public void checkDataExportCanBeDownloadedElseThrow(DataExport dataExport) {
+ if (!dataExport.getDataExportState().isDownloadable()) {
+ throw new AccessForbiddenException("Data export has either not been created or already been deleted");
+ }
+ }
+
}
diff --git a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportUtil.java b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportUtil.java
index 8c711062d08c..b275ec861b7e 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportUtil.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportUtil.java
@@ -17,12 +17,25 @@ private DataExportUtil() {
// Utility class
}
+ /**
+ * Creates the given directory if it does not exist yet.
+ *
+ * @param directory the directory to create
+ * @throws IOException if an error occurs while accessing the file system
+ */
static void createDirectoryIfNotExistent(Path directory) throws IOException {
if (!Files.exists(directory)) {
- Files.createDirectory(directory);
+ Files.createDirectories(directory);
}
}
+ /**
+ * Retrieves the path to the directory for the given course within the data export.
+ *
+ * @param workingDirectory the working directory where the data export is created
+ * @param course the course for which the directory should be retrieved
+ * @return the path to the directory for the given course
+ */
static Path retrieveCourseDirPath(Path workingDirectory, Course course) {
return workingDirectory.resolve(COURSE_DIRECTORY_PREFIX + course.getShortName());
}
diff --git a/src/main/java/de/tum/in/www1/artemis/service/exam/ExamUserService.java b/src/main/java/de/tum/in/www1/artemis/service/exam/ExamUserService.java
index f60e68c2a921..cd2aabaaad9f 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/exam/ExamUserService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/exam/ExamUserService.java
@@ -6,6 +6,7 @@
import java.util.List;
import java.util.Optional;
+import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripperByArea;
import org.slf4j.Logger;
@@ -51,7 +52,7 @@ public ExamUserService(FileService fileService, UserRepository userRepository, E
*/
public List parsePDF(MultipartFile file) {
- try (PDDocument document = PDDocument.load(file.getBytes())) {
+ try (PDDocument document = Loader.loadPDF(file.getBytes())) {
ImageExtractor imageExtractor = new ImageExtractor(document);
imageExtractor.process();
List images = imageExtractor.getImages();
@@ -129,6 +130,6 @@ public ExamUsersNotFoundDTO saveImages(long examId, MultipartFile file) {
/**
* Contains the information about an exam user with image
*/
- private record ExamUserWithImageDTO(String studentRegistrationNumber, ImageDTO image) {
+ record ExamUserWithImageDTO(String studentRegistrationNumber, ImageDTO image) {
}
}
diff --git a/src/main/java/de/tum/in/www1/artemis/service/exam/ImageExtractor.java b/src/main/java/de/tum/in/www1/artemis/service/exam/ImageExtractor.java
index 232c98a373a2..f7f9f210a2d0 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/exam/ImageExtractor.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/exam/ImageExtractor.java
@@ -46,12 +46,12 @@ public ImageExtractor(PDDocument document) {
this.images = new ArrayList<>();
this.pdfDocument = document;
- addOperator(new Concatenate());
- addOperator(new DrawObject());
- addOperator(new SetGraphicsStateParameters());
- addOperator(new Save());
- addOperator(new Restore());
- addOperator(new SetMatrix());
+ addOperator(new Concatenate(this));
+ addOperator(new DrawObject(this));
+ addOperator(new SetGraphicsStateParameters(this));
+ addOperator(new Save(this));
+ addOperator(new Restore(this));
+ addOperator(new SetMatrix(this));
}
/**
diff --git a/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseExportService.java b/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseExportService.java
index ac7bed156470..de1a21404714 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseExportService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/programming/ProgrammingExerciseExportService.java
@@ -50,7 +50,8 @@
import de.tum.in.www1.artemis.domain.*;
import de.tum.in.www1.artemis.domain.enumeration.ProgrammingLanguage;
import de.tum.in.www1.artemis.domain.enumeration.RepositoryType;
-import de.tum.in.www1.artemis.domain.participation.*;
+import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseStudentParticipation;
+import de.tum.in.www1.artemis.domain.participation.StudentParticipation;
import de.tum.in.www1.artemis.exception.GitException;
import de.tum.in.www1.artemis.repository.AuxiliaryRepositoryRepository;
import de.tum.in.www1.artemis.repository.ProgrammingExerciseRepository;
@@ -90,7 +91,9 @@ public class ProgrammingExerciseExportService {
public static final String EXPORTED_EXERCISE_PROBLEM_STATEMENT_FILE_PREFIX = "Problem-Statement";
- private static final String EMBEDDED_FILE_REGEX = "\\[.*] *\\(/api/files/markdown/.*\\)";
+ private static final String EMBEDDED_FILE_MARKDOWN_SYNTAX_REGEX = "\\[.*] *\\(/api/files/markdown/.*\\)";
+
+ private static final String EMBEDDED_FILE_HTML_SYNTAX_REGEX = "";
private static final String API_MARKDOWN_FILE_PATH = "/api/files/markdown/";
@@ -152,6 +155,14 @@ public Path exportProgrammingExerciseInstructorMaterial(ProgrammingExercise exer
return pathToZippedExercise;
}
+ /**
+ * Export problem statement and embedded files for a given programming exercise.
+ *
+ * @param exercise the programming exercise that is exported
+ * @param exportErrors List of failures that occurred during the export
+ * @param exportDir the directory where the content of the export is stored
+ * @param pathsToBeZipped the paths that should be included in the zip file
+ */
private void exportProblemStatementAndEmbeddedFiles(ProgrammingExercise exercise, List exportErrors, Path exportDir, List pathsToBeZipped) {
var problemStatementFileExtension = ".md";
String problemStatementFileName = EXPORTED_EXERCISE_PROBLEM_STATEMENT_FILE_PREFIX + "-" + exercise.getTitle() + problemStatementFileExtension;
@@ -168,15 +179,97 @@ private void exportProblemStatementAndEmbeddedFiles(ProgrammingExercise exercise
* @param outputDir the directory where the content of the export is stored
* @param pathsToBeZipped the paths that should be included in the zip file
*/
-
private void copyEmbeddedFiles(ProgrammingExercise exercise, Path outputDir, List pathsToBeZipped, List exportErrors) {
- Set embeddedFiles = new HashSet<>();
+ Set embeddedFilesWithMarkdownSyntax = new HashSet<>();
+ Set embeddedFilesWithHtmlSyntax = new HashSet<>();
+
+ Matcher matcherForMarkdownSyntax = Pattern.compile(EMBEDDED_FILE_MARKDOWN_SYNTAX_REGEX).matcher(exercise.getProblemStatement());
+ Matcher matcherForHtmlSyntax = Pattern.compile(EMBEDDED_FILE_HTML_SYNTAX_REGEX).matcher(exercise.getProblemStatement());
+ checkForMatchesInProblemStatementAndCreateDirectoryForFiles(outputDir, pathsToBeZipped, exportErrors, embeddedFilesWithMarkdownSyntax, matcherForMarkdownSyntax);
+ Path embeddedFilesDir = checkForMatchesInProblemStatementAndCreateDirectoryForFiles(outputDir, pathsToBeZipped, exportErrors, embeddedFilesWithHtmlSyntax,
+ matcherForHtmlSyntax);
+ // if the returned path is null the directory could not be created
+ if (embeddedFilesDir == null) {
+ return;
+ }
+ copyFilesEmbeddedWithMarkdownSyntax(exercise, exportErrors, embeddedFilesWithMarkdownSyntax, embeddedFilesDir);
+ copyFilesEmbeddedWithHtmlSyntax(exercise, exportErrors, embeddedFilesWithHtmlSyntax, embeddedFilesDir);
+
+ }
+
+ /**
+ * Copies the files that are embedded with Markdown syntax to the embedded files' directory.
+ *
+ * @param exercise the programming exercise that is exported
+ * @param exportErrors List of failures that occurred during the export
+ * @param embeddedFilesWithMarkdownSyntax the files that are embedded with Markdown syntax
+ * @param embeddedFilesDir the directory where the embedded files are stored
+ */
+ private void copyFilesEmbeddedWithMarkdownSyntax(ProgrammingExercise exercise, List exportErrors, Set embeddedFilesWithMarkdownSyntax, Path embeddedFilesDir) {
+ for (String embeddedFile : embeddedFilesWithMarkdownSyntax) {
+ // avoid matching other closing ] or () in the squared brackets by getting the index of the last ]
+ String lastPartOfMatchedString = embeddedFile.substring(embeddedFile.lastIndexOf("]") + 1);
+ String filePath = lastPartOfMatchedString.substring(lastPartOfMatchedString.indexOf("(") + 1, lastPartOfMatchedString.indexOf(")"));
+ constructFilenameAndCopyFile(exercise, exportErrors, embeddedFilesDir, filePath);
+ }
+ }
+
+ /**
+ * Copies the files that are embedded with html syntax to the embedded files' directory.
+ *
+ * @param exercise the programming exercise that is exported
+ * @param exportErrors List of failures that occurred during the export
+ * @param embeddedFilesWithHtmlSyntax the files that are embedded with html syntax
+ * @param embeddedFilesDir the directory where the embedded files are stored
+ */
+ private void copyFilesEmbeddedWithHtmlSyntax(ProgrammingExercise exercise, List exportErrors, Set embeddedFilesWithHtmlSyntax, Path embeddedFilesDir) {
+ for (String embeddedFile : embeddedFilesWithHtmlSyntax) {
+ int indexOfFirstQuotationMark = embeddedFile.indexOf('"');
+ String filePath = embeddedFile.substring(embeddedFile.indexOf("src=") + 5, embeddedFile.indexOf('"', indexOfFirstQuotationMark + 1));
+ constructFilenameAndCopyFile(exercise, exportErrors, embeddedFilesDir, filePath);
+ }
+ }
- Matcher matcher = Pattern.compile(EMBEDDED_FILE_REGEX).matcher(exercise.getProblemStatement());
+ /**
+ * Extracts the filename from the matched string and copies the file to the embedded files' directory.
+ *
+ * @param exercise the programming exercise that is exported
+ * @param exportErrors List of failures that occurred during the export
+ * @param embeddedFilesDir the directory where the embedded files are stored
+ * @param filePath the path of the file that should be copied
+ */
+ private void constructFilenameAndCopyFile(ProgrammingExercise exercise, List exportErrors, Path embeddedFilesDir, String filePath) {
+ String fileName = filePath.replace(API_MARKDOWN_FILE_PATH, "");
+ Path imageFilePath = Path.of(FilePathService.getMarkdownFilePath(), fileName);
+ Path imageExportPath = embeddedFilesDir.resolve(fileName);
+ // we need this check as it might be that the matched string is different and not filtered out above but the file is already copied
+ if (!Files.exists(imageExportPath)) {
+ try {
+ Files.copy(imageFilePath, imageExportPath);
+ }
+ catch (IOException e) {
+ exportErrors.add("Failed to copy embedded files: " + e.getMessage());
+ log.warn("Could not copy embedded file {} for exercise with id {}", fileName, exercise.getId());
+ }
+ }
+ }
+
+ /**
+ * Checks for matches in the problem statement and creates a directory for the embedded files.
+ *
+ * @param outputDir the directory where the content of the export is stored
+ * @param pathsToBeZipped the paths that should be included in the zip file
+ * @param exportErrors List of failures that occurred during the export
+ * @param embeddedFiles the files that are embedded in the problem statement
+ * @param matcher the matcher that is used to find the embedded files
+ * @return the path to the embedded files directory or null if the directory could not be created
+ */
+ private Path checkForMatchesInProblemStatementAndCreateDirectoryForFiles(Path outputDir, List pathsToBeZipped, List exportErrors, Set embeddedFiles,
+ Matcher matcher) {
while (matcher.find()) {
embeddedFiles.add(matcher.group());
}
- log.debug("Found embedded files:{} ", embeddedFiles);
+ log.debug("Found embedded files: {} ", embeddedFiles);
Path embeddedFilesDir = outputDir.resolve("files");
if (!embeddedFiles.isEmpty()) {
if (!Files.exists(embeddedFilesDir)) {
@@ -186,30 +279,12 @@ private void copyEmbeddedFiles(ProgrammingExercise exercise, Path outputDir, Lis
catch (IOException e) {
exportErrors.add("Could not create directory for embedded files: " + e.getMessage());
log.warn("Could not create directory for embedded files. Won't include embedded files: " + e.getMessage());
- return;
+ return null;
}
}
pathsToBeZipped.add(embeddedFilesDir);
}
- for (String embeddedFile : embeddedFiles) {
- // avoid matching other closing ] or () in the squared brackets by getting the index of the last ]
- String lastPartOfMatchedString = embeddedFile.substring(embeddedFile.lastIndexOf("]") + 1);
- String filePath = lastPartOfMatchedString.substring(lastPartOfMatchedString.indexOf("(") + 1, lastPartOfMatchedString.indexOf(")"));
- String fileName = filePath.replace(API_MARKDOWN_FILE_PATH, "");
- Path imageFilePath = Path.of(FilePathService.getMarkdownFilePath(), fileName);
- Path imageExportPath = embeddedFilesDir.resolve(fileName);
- // we need this check as it might be that the matched string is different and not filtered out above but the file is already copied
- if (!Files.exists(imageExportPath)) {
- try {
- Files.copy(imageFilePath, imageExportPath);
- }
- catch (IOException e) {
- exportErrors.add("Failed to copy embedded files: " + e.getMessage());
- log.warn("Could not copy embedded file {} for exercise with id {}", fileName, exercise.getId());
- }
- }
- }
-
+ return embeddedFilesDir;
}
/**
diff --git a/src/main/java/de/tum/in/www1/artemis/service/scheduled/DataExportScheduleService.java b/src/main/java/de/tum/in/www1/artemis/service/scheduled/DataExportScheduleService.java
index b856eac57890..8278387c1efe 100644
--- a/src/main/java/de/tum/in/www1/artemis/service/scheduled/DataExportScheduleService.java
+++ b/src/main/java/de/tum/in/www1/artemis/service/scheduled/DataExportScheduleService.java
@@ -1,8 +1,8 @@
package de.tum.in.www1.artemis.service.scheduled;
-import java.util.HashSet;
-import java.util.Optional;
-import java.util.Set;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,18 +57,19 @@ public DataExportScheduleService(DataExportRepository dataExportRepository, Data
* Deleted will be all data exports that have a creation date older than seven days
*/
@Scheduled(cron = "${artemis.scheduling.data-export-creation-time: 0 0 4 * * *}")
- public void createDataExportsAndDeleteOldOnes() {
+ public void createDataExportsAndDeleteOldOnes() throws InterruptedException {
if (profileService.isDev()) {
// do not execute this in a development environment
// NOTE: if you want to test this locally, please comment it out, but do not commit the changes
return;
}
-
checkSecurityUtils();
log.info("Creating data exports and deleting old ones");
- Set successfulDataExports = new HashSet<>();
+ Set successfulDataExports = Collections.synchronizedSet(new HashSet<>());
var dataExportsToBeCreated = dataExportRepository.findAllToBeCreated();
- dataExportsToBeCreated.forEach(dataExport -> createDataExport(dataExport, successfulDataExports));
+ ExecutorService executor = Executors.newFixedThreadPool(10);
+ dataExportsToBeCreated.forEach(dataExport -> executor.execute(() -> createDataExport(dataExport, successfulDataExports)));
+ executor.shutdown();
var dataExportsToBeDeleted = dataExportRepository.findAllToBeDeleted();
dataExportsToBeDeleted.forEach(this::deleteDataExport);
Optional admin = userService.findInternalAdminUser();
@@ -76,6 +77,12 @@ public void createDataExportsAndDeleteOldOnes() {
log.warn("No internal admin user found. Cannot send email to admin about successful creation of data exports.");
return;
}
+ // This job runs at 4 am by default and the next scheduled job runs at 5 am, so we should allow 60 minutes for the creation.
+ // If the creation doesn't finish within 60 minutes, all pending exports will be picked up when the job runs the next time.
+ if (!executor.awaitTermination(60, java.util.concurrent.TimeUnit.MINUTES)) {
+ log.info("Not all pending data exports could be created within 60 minutes.");
+ executor.shutdownNow();
+ }
if (!successfulDataExports.isEmpty()) {
mailService.sendSuccessfulDataExportsEmailToAdmin(admin.get(), successfulDataExports);
}
@@ -87,6 +94,7 @@ public void createDataExportsAndDeleteOldOnes() {
* @param dataExport the data export to be created
*/
private void createDataExport(DataExport dataExport, Set successfulDataExports) {
+ checkSecurityUtils();
log.info("Creating data export for {}", dataExport.getUser().getLogin());
var successful = dataExportCreationService.createDataExport(dataExport);
if (successful) {
diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/DataExportResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/DataExportResource.java
index be14fd560b6e..366a915199ef 100644
--- a/src/main/java/de/tum/in/www1/artemis/web/rest/DataExportResource.java
+++ b/src/main/java/de/tum/in/www1/artemis/web/rest/DataExportResource.java
@@ -5,14 +5,12 @@
import java.time.Duration;
import java.time.ZoneId;
import java.time.ZonedDateTime;
-import java.util.Comparator;
import javax.validation.constraints.NotNull;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
+import org.springframework.http.*;
import org.springframework.web.bind.annotation.*;
import de.tum.in.www1.artemis.domain.DataExport;
@@ -27,7 +25,8 @@
import de.tum.in.www1.artemis.web.rest.errors.EntityNotFoundException;
/**
- * REST controller for data exports
+ * REST controller for data exports.
+ * It contains the REST endpoints for requesting, downloading data exports and checking if a data export can be requested or downloaded.
*/
@RestController
@RequestMapping("api/")
@@ -50,9 +49,9 @@ public DataExportResource(@Value("${artemis.data-export.days-between-data-export
}
/**
- * Request a data export for the given user
+ * POST /data-exports: Request a data export for the currently logged-in user.
*
- * @return the data export object
+ * @return a DTO containing the id of the data export that was created, its state and when it was requested
*/
@PostMapping("data-exports")
@EnforceAtLeastStudent
@@ -65,16 +64,19 @@ public RequestDataExportDTO requestDataExport() {
/**
* Checks if the user can request a new data export.
+ *
+ * This is the case if the user has not requested a data export yet or if the last data export was created more than DAYS_BETWEEN_DATA_EXPORTS days ago.
*
* @return true if the user can request a new data export, false otherwise
*/
private boolean canRequestDataExport() {
var user = userRepository.getUser();
- var dataExports = dataExportRepository.findAllDataExportsByUserId(user.getId());
+ var dataExports = dataExportRepository.findAllDataExportsByUserIdOrderByRequestDateDesc(user.getId());
if (dataExports.isEmpty()) {
return true;
}
- var latestDataExport = dataExports.stream().max(Comparator.comparing(DataExport::getCreatedDate)).get();
+ // because we order by request date desc, the first data export is the latest one
+ var latestDataExport = dataExports.get(0);
var olderThanDaysBetweenDataExports = Duration.between(latestDataExport.getCreatedDate().atZone(ZoneId.systemDefault()), ZonedDateTime.now())
.toDays() >= DAYS_BETWEEN_DATA_EXPORTS;
@@ -82,7 +84,12 @@ private boolean canRequestDataExport() {
}
/**
- * Download the data export for the given user
+ * GET /data-exports/{dataExportId}: Download the data export for the given id.
+ *
+ * We check if the user is the owner of the data export and if the data export can be downloaded.
+ * If this is the case, we return a resource containing the data export zip file.
+ * The file name is set to the name of the zip file.
+ * The content disposition header is set to attachment so that the browser will download the file instead of displaying it.
*
* @param dataExportId the id of the data export to download
* @return A resource containing the data export zip file
@@ -92,20 +99,18 @@ private boolean canRequestDataExport() {
public ResponseEntity downloadDataExport(@PathVariable long dataExportId) {
DataExport dataExport = dataExportRepository.findByIdElseThrow(dataExportId);
currentlyLoggedInUserIsOwnerOfDataExportElseThrow(dataExport);
- checkDataExportCanBeDownloaded(dataExport);
+ dataExportService.checkDataExportCanBeDownloadedElseThrow(dataExport);
Resource resource = dataExportService.downloadDataExport(dataExport);
File finalZipFile = Path.of(dataExport.getFilePath()).toFile();
- return ResponseEntity.ok().contentLength(finalZipFile.length()).contentType(MediaType.APPLICATION_OCTET_STREAM).header("filename", finalZipFile.getName()).body(resource);
- }
-
- private void checkDataExportCanBeDownloaded(DataExport dataExport) {
- if (!dataExport.getDataExportState().isDownloadable()) {
- throw new AccessForbiddenException("Data export has either not been created or already been deleted");
- }
+ ContentDisposition contentDisposition = ContentDisposition.builder("attachment").filename(finalZipFile.getName()).build();
+ HttpHeaders headers = new HttpHeaders();
+ headers.setContentDisposition(contentDisposition);
+ return ResponseEntity.ok().contentLength(finalZipFile.length()).headers(headers).contentType(MediaType.APPLICATION_OCTET_STREAM).header("filename", finalZipFile.getName())
+ .body(resource);
}
/**
- * checks if the currently logged-in user is the owner of the given data export
+ * Checks if the currently logged-in user is the owner of the given data export.
*
* @param dataExport the data export that needs to be checked
* @throws AccessForbiddenException if logged-in user isn't the owner of the data export
@@ -117,7 +122,7 @@ private void currentlyLoggedInUserIsOwnerOfDataExportElseThrow(@NotNull DataExpo
}
/**
- * checks if the currently logged-in user is owner of the given data export
+ * Checks if the currently logged-in user is owner of the given data export.
*
* @param dataExport the data export that needs to be checked
* @return true if the user is the owner of the data export, false otherwise
@@ -132,7 +137,7 @@ private boolean currentlyLoggedInUserIsOwnerOfDataExport(DataExport dataExport)
}
/**
- * Check if the user can request a data export
+ * GET /data-exports/can-request: Check if the logged-in user can request a data export.
*
* @return true if the user can request a data export, false otherwise
*/
@@ -143,9 +148,9 @@ public boolean canRequestExport() {
}
/**
- * Check if the user can download any data export
+ * GET /data-exports/can-download: Check if the logged-in user can download any data export.
*
- * @return a data export DTO with the id of the export that can be downloaded or a DTO with a id of null if no export can be downloaded
+ * @return a data export DTO with the id of the export that can be downloaded or a DTO with an id of null if no export can be downloaded
*/
@GetMapping("data-exports/can-download")
@EnforceAtLeastStudent
@@ -154,7 +159,7 @@ public DataExportDTO canDownloadAnyExport() {
}
/**
- * Check if the user can download a specific data export
+ * GET /data-exports/{dataExportId}/can-download: Check if the logged-in user can download the data export with the given id.
*
* @param dataExportId the id of the data export that should be checked
* @return true if the user can download the data export, false otherwise
@@ -170,8 +175,8 @@ public boolean canDownloadSpecificExport(@PathVariable long dataExportId) {
*
* @param dataExportId the id of the data export to check
* @return true if the data export can be downloaded, false otherwise
- * @throws de.tum.in.www1.artemis.web.rest.errors.EntityNotFoundException if the data export or the user could not be found
- * @throws de.tum.in.www1.artemis.web.rest.errors.AccessForbiddenException if the user is not allowed to download the data export
+ * @throws EntityNotFoundException if the data export or the user could not be found
+ * @throws AccessForbiddenException if the user is not allowed to download the data export
*/
private boolean canDownloadSpecificDataExport(long dataExportId) throws EntityNotFoundException, AccessForbiddenException {
var dataExport = dataExportRepository.findByIdElseThrow(dataExportId);
diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/ProgrammingExerciseParticipationResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/ProgrammingExerciseParticipationResource.java
index 19b31e24c79a..e7a545e69e6d 100644
--- a/src/main/java/de/tum/in/www1/artemis/web/rest/ProgrammingExerciseParticipationResource.java
+++ b/src/main/java/de/tum/in/www1/artemis/web/rest/ProgrammingExerciseParticipationResource.java
@@ -83,7 +83,7 @@ public ResponseEntity getParticipationW
participationAuthCheckService.checkCanAccessParticipationElseThrow(participation);
// hide details that should not be shown to the students
- resultService.filterSensitiveInformationIfNecessary(participation, participation.getResults());
+ resultService.filterSensitiveInformationIfNecessary(participation, participation.getResults(), Optional.empty());
return ResponseEntity.ok(participation);
}
diff --git a/src/main/webapp/app/core/legal/data-export/data-export.component.html b/src/main/webapp/app/core/legal/data-export/data-export.component.html
index 182a57931dd1..693f7d2fe7d6 100644
--- a/src/main/webapp/app/core/legal/data-export/data-export.component.html
+++ b/src/main/webapp/app/core/legal/data-export/data-export.component.html
@@ -24,7 +24,8 @@