Development: Fix issues in the user data export

src/main/java/de/tum/in/www1/artemis/domain/enumeration/DataExportState.java

@@ -8,10 +8,26 @@ public enum DataExportState {
 
     REQUESTED, IN_CREATION, EMAIL_SENT, DOWNLOADED, DOWNLOADED_DELETED, DELETED, FAILED;
 
+    /**
+     * Checks if the data export can be downloaded.
+     * The data export can be downloaded if its state is either EMAIL_SENT or DOWNLOADED.
+     * The state is EMAIL_SENT if the data export has been created and the user has been notified via email.
+     * The state is DOWNLOADED if the user has downloaded the data export at least once.
+     *
+     * @return true if the data export can be downloaded, false otherwise
+     */
     public boolean isDownloadable() {
         return this == DOWNLOADED || this == EMAIL_SENT;
     }
 
+    /**
+     * Checks if the data export has been downloaded.
+     * The data export has been downloaded if its state is either DOWNLOADED or DOWNLOADED_DELETED.
+     * The state is DOWNLOADED if the user has downloaded the data export at least once, but it has not been deleted yet.
+     * The state is DOWNLOADED_DELETED if the user has downloaded the data export at least once, and it has been deleted.
+     *
+     * @return true if the data export has been downloaded, false otherwise
+     */
     public boolean hasBeenDownloaded() {
         return this == DOWNLOADED || this == DOWNLOADED_DELETED;
     }

src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCommunicationDataService.java

@@ -25,6 +25,8 @@
 
 /**
  * A service to create the communication data export for users
+ * This includes messages (posts), thread replies (answer posts) and reactions to posts and answer posts
+ * All communication data is exported per course and stored in a CSV file
  */
 @Service
 public class DataExportCommunicationDataService {
@@ -67,6 +69,12 @@ public void createCommunicationDataExport(long userId, Path workingDirectory) th
         createCommunicationDataExportIfReactionsToAnswerPostsExist(workingDirectory, reactionsToAnswerPostsPerCourse);
     }
 
+    /**
+     * Creates the communication data export for a course if only reactions to answer posts exist
+     *
+     * @param workingDirectory                the directory where the export is stored
+     * @param reactionsToAnswerPostsPerCourse the reactions to answer posts grouped by course
+     */
     private void createCommunicationDataExportIfReactionsToAnswerPostsExist(Path workingDirectory, Map<Course, List<Reaction>> reactionsToAnswerPostsPerCourse) throws IOException {
         // it can happen that only answer post reactions exist in a course but neither posts, nor answer posts nor reactions to posts
         for (var entry : reactionsToAnswerPostsPerCourse.entrySet()) {
@@ -78,6 +86,13 @@ private void createCommunicationDataExportIfReactionsToAnswerPostsExist(Path wor
         }
     }
 
+    /**
+     * Creates the communication data export for a course if only reactions to posts (and potentially to answer posts) exist
+     *
+     * @param workingDirectory                the directory where the export is stored
+     * @param reactionsToPostsPerCourse       the reactions to posts grouped by course
+     * @param reactionsToAnswerPostsPerCourse the reactions to answer posts grouped by course
+     */
     private void createCommunicationDataExportIfReactionsToPostsExist(Path workingDirectory, Map<Course, List<Reaction>> reactionsToPostsPerCourse,
             Map<Course, List<Reaction>> reactionsToAnswerPostsPerCourse) throws IOException {
         // it can happen that only reactions exist in a course but no post or answer post
@@ -91,6 +106,14 @@ private void createCommunicationDataExportIfReactionsToPostsExist(Path workingDi
         }
     }
 
+    /**
+     * Creates the communication data export for a course if only answer posts (and potentially reactions to post and answer posts) exist
+     *
+     * @param workingDirectory                the directory where the export is stored
+     * @param answerPostsPerCourse            the answer posts grouped by course
+     * @param reactionsToPostsPerCourse       the reactions to posts grouped by course
+     * @param reactionsToAnswerPostsPerCourse the reactions to answer posts grouped by course
+     */
     private void createCommunicationDataExportIfAnswerPostsExist(Path workingDirectory, Map<Course, List<AnswerPost>> answerPostsPerCourse,
             Map<Course, List<Reaction>> reactionsToPostsPerCourse, Map<Course, List<Reaction>> reactionsToAnswerPostsPerCourse) throws IOException {
         // it can happen that an answer post and reactions exist in a course but no post
@@ -105,6 +128,15 @@ private void createCommunicationDataExportIfAnswerPostsExist(Path workingDirecto
         }
     }
 
+    /**
+     * Creates the communication data export for a course if posts exist
+     *
+     * @param workingDirectory                the directory where the export is stored
+     * @param postsPerCourse                  the posts grouped by course
+     * @param answerPostsPerCourse            the answer posts grouped by course
+     * @param reactionsToPostsPerCourse       the reactions to posts grouped by course
+     * @param reactionsToAnswerPostsPerCourse the reactions to answer posts grouped by course
+     */
     private void createCommunicationDataExportIfPostsExist(Path workingDirectory, Map<Course, List<Post>> postsPerCourse, Map<Course, List<AnswerPost>> answerPostsPerCourse,
             Map<Course, List<Reaction>> reactionsToPostsPerCourse, Map<Course, List<Reaction>> reactionsToAnswerPostsPerCourse) throws IOException {
         // this covers all cases where at least one post in a course exists
@@ -121,6 +153,15 @@ private void createCommunicationDataExportIfPostsExist(Path workingDirectory, Ma
         }
     }
 
+    /**
+     * Creates the actual CSV file containing the communication data for a course
+     *
+     * @param courseDir                   the directory where the CSV file is stored
+     * @param postsInCourse               the posts in the course
+     * @param answerPostsInCourse         the answer posts in the course
+     * @param postReactionsInCourse       the reactions to posts in the course
+     * @param answerPostReactionsInCourse the reactions to answer posts in the course
+     */
     private void createCommunicationDataCsvFile(Path courseDir, List<Post> postsInCourse, List<AnswerPost> answerPostsInCourse, List<Reaction> postReactionsInCourse,
             List<Reaction> answerPostReactionsInCourse) throws IOException {
 

src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCreationService.java

@@ -30,6 +30,8 @@
 
 /**
  * A service to create data exports for users
+ * This service is responsible for creating the data export, delegating most tasks to the {@link DataExportExerciseCreationService} and {@link DataExportExamCreationService}
+ * and notifying the user about the creation.
  */
 @Service
 public class DataExportCreationService {
@@ -105,6 +107,7 @@ private void addReadmeFile(Path workingDirectory) throws IOException, URISyntaxE
 
     /**
      * Creates the data export for the given user.
+     * This includes creation of the export and notifying the user about the creation.
      *
      * @param dataExport the data export to be created
      * @return true if the export was successful, false otherwise
@@ -129,6 +132,14 @@ public boolean createDataExport(DataExport dataExport) {
         return true;
     }
 
+    /**
+     * Handles the case of a failed data export creation.
+     * This includes setting the state of the data export to failed, notifying the user about the failure and sending an email to the admin with the exception why the export
+     * failed.
+     *
+     * @param dataExport the data export that failed to be created
+     * @param exception  the exception that occurred during the creation
+     */
     private void handleCreationFailure(DataExport dataExport, Exception exception) {
         dataExport.setDataExportState(DataExportState.FAILED);
         dataExport = dataExportRepository.save(dataExport);
@@ -141,6 +152,13 @@ private void handleCreationFailure(DataExport dataExport, Exception exception) {
         mailService.sendDataExportFailedEmailToAdmin(admin.get(), dataExport, exception);
     }
 
+    /**
+     * Finishes the creation of the data export by setting the file path to the zip file, the state to EMAIL_SENT and the creation finished date.
+     *
+     * @param dataExport     the data export whose creation is finished
+     * @param dataExportPath the path to the zip file containing the data export
+     * @return the updated data export from the database
+     */
     private DataExport finishDataExportCreation(DataExport dataExport, Path dataExportPath) {
         dataExport.setFilePath(dataExportPath.toString());
         dataExport.setCreationFinishedDate(ZonedDateTime.now());
@@ -149,6 +167,14 @@ private DataExport finishDataExportCreation(DataExport dataExport, Path dataExpo
         return dataExportRepository.save(dataExport);
     }
 
+    /**
+     * Prepares the data export by creating the working directory, scheduling it for deletion and setting the state to IN_CREATION.
+     * If the path where the data exports are stored does not exist yet, it will be created.
+     *
+     * @param dataExport the data export to be prepared
+     * @return the path to the working directory
+     * @throws IOException if the working directory could not be created
+     */
     private Path prepareDataExport(DataExport dataExport) throws IOException {
         if (!Files.exists(dataExportsPath)) {
             Files.createDirectories(dataExportsPath);
@@ -161,6 +187,13 @@ private Path prepareDataExport(DataExport dataExport) throws IOException {
         return workingDirectory;
     }
 
+    /**
+     * Adds the general user information to the data export.
+     * This includes the login, name, email and registration number (matriculation number).
+     *
+     * @param user             the user for which the information should be added
+     * @param workingDirectory the directory in which the information should be stored
+     */
     private void addGeneralUserInformation(User user, Path workingDirectory) throws IOException {
         String[] headers = { "login", "name", "email", "registration number" };
         CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(headers).build();
@@ -171,11 +204,18 @@ private void addGeneralUserInformation(User user, Path workingDirectory) throws
         }
     }
 
+    /**
+     * Creates the zip file containing the data export.
+     *
+     * @param userLogin        the login of the user for which the data export was created
+     * @param workingDirectory the directory containing the data export
+     * @return the path to the zip file
+     * @throws IOException if the zip file could not be created
+     */
     private Path createDataExportZipFile(String userLogin, Path workingDirectory) throws IOException {
         // There should actually never exist more than one data export for a user at a time (once the feature is fully implemented), but to be sure the name is unique, we add the
         // current timestamp
         return zipFileService.createZipFileWithFolderContent(dataExportsPath.resolve("data-export_" + userLogin + ZonedDateTime.now().toEpochSecond() + ZIP_FILE_EXTENSION),
                 workingDirectory, null);
-
     }
 }

src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExamCreationService.java

@@ -7,9 +7,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -18,14 +16,18 @@
 import org.springframework.stereotype.Service;
 
 import de.tum.in.www1.artemis.domain.Course;
+import de.tum.in.www1.artemis.domain.GradingScale;
 import de.tum.in.www1.artemis.domain.ProgrammingExercise;
 import de.tum.in.www1.artemis.domain.exam.StudentExam;
+import de.tum.in.www1.artemis.repository.GradingScaleRepository;
 import de.tum.in.www1.artemis.repository.StudentExamRepository;
 import de.tum.in.www1.artemis.service.exam.ExamService;
 import de.tum.in.www1.artemis.web.rest.dto.ExamScoresDTO;
 
 /**
  * A service to create the data export for exams the user has participated in.
+ * This includes exercise participations and general information such as working time.
+ * If the results are published, the results are also included.
  */
 @Service
 public class DataExportExamCreationService {
@@ -38,11 +40,14 @@ public class DataExportExamCreationService {
 
     private final ExamService examService;
 
-    public DataExportExamCreationService(StudentExamRepository studentExamRepository, DataExportExerciseCreationService dataExportExerciseCreationService,
-            ExamService examService) {
+    private final GradingScaleRepository gradingScaleRepository;
+
+    public DataExportExamCreationService(StudentExamRepository studentExamRepository, DataExportExerciseCreationService dataExportExerciseCreationService, ExamService examService,
+            GradingScaleRepository gradingScaleRepository) {
         this.studentExamRepository = studentExamRepository;
         this.dataExportExerciseCreationService = dataExportExerciseCreationService;
         this.examService = examService;
+        this.gradingScaleRepository = gradingScaleRepository;
     }
 
     /**
@@ -61,14 +66,22 @@ public void createExportForExams(long userId, Path workingDirectory) throws IOEx
                 var exam = studentExam.getExam();
                 var examTitle = exam.getSanitizedExamTitle();
                 var courseDirPath = retrieveCourseDirPath(workingDirectory, exam.getCourse());
-                createDirectoryIfNotExistent(courseDirPath);
+                var examsDirPath = courseDirPath.resolve("exams");
+                createDirectoryIfNotExistent(examsDirPath);
                 var examDirectoryName = EXAM_DIRECTORY_PREFIX + examTitle + "_" + studentExam.getId();
-                var examWorkingDir = Files.createDirectories(courseDirPath.resolve(examDirectoryName));
+                var examWorkingDir = Files.createDirectories(examsDirPath.resolve(examDirectoryName));
                 createStudentExamExport(studentExam, examWorkingDir);
             }
         }
     }
 
+    /**
+     * Creates the data export for the given student exam.
+     * This includes extracting all exercise participations, general exam information such as working time and the results if the results are published.
+     *
+     * @param studentExam    the student exam belonging to the user for which the data export should be created
+     * @param examWorkingDir the directory in which the information about the exam should be stored
+     */
     private void createStudentExamExport(StudentExam studentExam, Path examWorkingDir) throws IOException {
         for (var exercise : studentExam.getExercises()) {
             // since the behavior is undefined if multiple student exams for the same exam and student combination exist, the exercise can be null
@@ -89,11 +102,18 @@ private void createStudentExamExport(StudentExam studentExam, Path examWorkingDi
         addGeneralExamInformation(studentExam, examWorkingDir);
     }
 
+    /**
+     * Adds the results of the student to the data export.
+     *
+     * @param studentExam    the student exam for which the results should be added
+     * @param examWorkingDir the directory in which the results should be stored
+     */
     private void addExamScores(StudentExam studentExam, Path examWorkingDir) throws IOException {
         var studentExamGrade = examService.getStudentExamGradeForDataExport(studentExam);
         var studentResult = studentExamGrade.studentResult();
+        var gradingScale = gradingScaleRepository.findByExamId(studentExam.getExam().getId());
         List<String> headers = new ArrayList<>();
-        var examResults = getExamResultsStreamToPrint(studentResult, headers);
+        var examResults = getExamResultsStreamToPrint(studentResult, headers, gradingScale);
         CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(headers.toArray(new String[0])).build();
         try (final CSVPrinter printer = new CSVPrinter(
                 Files.newBufferedWriter(examWorkingDir.resolve(EXAM_DIRECTORY_PREFIX + studentExam.getId() + "_result" + CSV_FILE_EXTENSION)), csvFormat)) {
@@ -102,21 +122,29 @@ private void addExamScores(StudentExam studentExam, Path examWorkingDir) throws
         }
     }
 
-    private Stream<?> getExamResultsStreamToPrint(ExamScoresDTO.StudentResult studentResult, List<String> headers) {
+    /**
+     * Returns a stream of the exam results that should be included in the exam results CSV file.
+     *
+     * @param studentResult
+     * @param headers              a list containing the column headers that should be included in the CSV file
+     * @param gradingScaleOptional the optional grading scale of the exam
+     * @return a stream of information that should be included in the exam results CSV file
+     */
+    private Stream<?> getExamResultsStreamToPrint(ExamScoresDTO.StudentResult studentResult, List<String> headers, Optional<GradingScale> gradingScaleOptional) {
         var builder = Stream.builder();
         if (studentResult.overallPointsAchieved() != null) {
             builder.add(studentResult.overallPointsAchieved());
             headers.add("overall points");
         }
-        if (studentResult.hasPassed() != null) {
+        if (studentResult.hasPassed() != null && gradingScaleOptional.isPresent()) {
             builder.add(studentResult.hasPassed());
             headers.add("passed");
         }
-        if (studentResult.overallGrade() != null) {
+        if (studentResult.overallGrade() != null && gradingScaleOptional.isPresent()) {
             builder.add(studentResult.overallGrade());
             headers.add("overall grade");
         }
-        if (studentResult.gradeWithBonus() != null) {
+        if (studentResult.gradeWithBonus() != null && gradingScaleOptional.isPresent()) {
             builder.add(studentResult.gradeWithBonus());
             headers.add("grade with bonus");
         }
@@ -127,6 +155,14 @@ private Stream<?> getExamResultsStreamToPrint(ExamScoresDTO.StudentResult studen
         return builder.build();
     }
 
+    /**
+     * Adds general information about the student exam to the data export.
+     * This includes information such as if the exam was started, if it is a test exam, when it was started, if it was submitted, when it was submitted, the working time and the
+     * individual end of the working time.
+     *
+     * @param studentExam    the student exam for which the information should be added
+     * @param examWorkingDir the directory in which the information should be stored
+     */
     private void addGeneralExamInformation(StudentExam studentExam, Path examWorkingDir) throws IOException {
         String[] headers = { "started", "testExam", "started at", "submitted", "submitted at", "working time (in minutes)", "individual end date" };
         CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(headers).build();