Development: Fix issues in the user data export

src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportCreationService.java

@@ -176,6 +176,5 @@ private Path createDataExportZipFile(String userLogin, Path workingDirectory) th
         // current timestamp
         return zipFileService.createZipFileWithFolderContent(dataExportsPath.resolve("data-export_" + userLogin + ZonedDateTime.now().toEpochSecond() + ZIP_FILE_EXTENSION),
                 workingDirectory, null);
-
     }
 }

src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExamCreationService.java

@@ -7,9 +7,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -18,8 +16,10 @@
 import org.springframework.stereotype.Service;
 
 import de.tum.in.www1.artemis.domain.Course;
+import de.tum.in.www1.artemis.domain.GradingScale;
 import de.tum.in.www1.artemis.domain.ProgrammingExercise;
 import de.tum.in.www1.artemis.domain.exam.StudentExam;
+import de.tum.in.www1.artemis.repository.GradingScaleRepository;
 import de.tum.in.www1.artemis.repository.StudentExamRepository;
 import de.tum.in.www1.artemis.service.exam.ExamService;
 import de.tum.in.www1.artemis.web.rest.dto.ExamScoresDTO;
@@ -38,11 +38,14 @@ public class DataExportExamCreationService {
 
     private final ExamService examService;
 
-    public DataExportExamCreationService(StudentExamRepository studentExamRepository, DataExportExerciseCreationService dataExportExerciseCreationService,
-            ExamService examService) {
+    private final GradingScaleRepository gradingScaleRepository;
+
+    public DataExportExamCreationService(StudentExamRepository studentExamRepository, DataExportExerciseCreationService dataExportExerciseCreationService, ExamService examService,
+            GradingScaleRepository gradingScaleRepository) {
         this.studentExamRepository = studentExamRepository;
         this.dataExportExerciseCreationService = dataExportExerciseCreationService;
         this.examService = examService;
+        this.gradingScaleRepository = gradingScaleRepository;
     }
 
     /**
@@ -61,9 +64,10 @@ public void createExportForExams(long userId, Path workingDirectory) throws IOEx
                 var exam = studentExam.getExam();
                 var examTitle = exam.getSanitizedExamTitle();
                 var courseDirPath = retrieveCourseDirPath(workingDirectory, exam.getCourse());
-                createDirectoryIfNotExistent(courseDirPath);
+                var examsDirPath = courseDirPath.resolve("exams");
+                createDirectoryIfNotExistent(examsDirPath);
                 var examDirectoryName = EXAM_DIRECTORY_PREFIX + examTitle + "_" + studentExam.getId();
-                var examWorkingDir = Files.createDirectories(courseDirPath.resolve(examDirectoryName));
+                var examWorkingDir = Files.createDirectories(examsDirPath.resolve(examDirectoryName));
                 createStudentExamExport(studentExam, examWorkingDir);
             }
         }
@@ -92,8 +96,9 @@ private void createStudentExamExport(StudentExam studentExam, Path examWorkingDi
     private void addExamScores(StudentExam studentExam, Path examWorkingDir) throws IOException {
         var studentExamGrade = examService.getStudentExamGradeForDataExport(studentExam);
         var studentResult = studentExamGrade.studentResult();
+        var gradingScale = gradingScaleRepository.findByExamId(studentExam.getExam().getId());
         List<String> headers = new ArrayList<>();
-        var examResults = getExamResultsStreamToPrint(studentResult, headers);
+        var examResults = getExamResultsStreamToPrint(studentResult, headers, gradingScale);
         CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(headers.toArray(new String[0])).build();
         try (final CSVPrinter printer = new CSVPrinter(
                 Files.newBufferedWriter(examWorkingDir.resolve(EXAM_DIRECTORY_PREFIX + studentExam.getId() + "_result" + CSV_FILE_EXTENSION)), csvFormat)) {
@@ -102,21 +107,21 @@ private void addExamScores(StudentExam studentExam, Path examWorkingDir) throws
         }
     }
 
-    private Stream<?> getExamResultsStreamToPrint(ExamScoresDTO.StudentResult studentResult, List<String> headers) {
+    private Stream<?> getExamResultsStreamToPrint(ExamScoresDTO.StudentResult studentResult, List<String> headers, Optional<GradingScale> gradingScaleOptional) {
         var builder = Stream.builder();
         if (studentResult.overallPointsAchieved() != null) {
             builder.add(studentResult.overallPointsAchieved());
             headers.add("overall points");
         }
-        if (studentResult.hasPassed() != null) {
+        if (studentResult.hasPassed() != null && gradingScaleOptional.isPresent()) {
             builder.add(studentResult.hasPassed());
             headers.add("passed");
         }
-        if (studentResult.overallGrade() != null) {
+        if (studentResult.overallGrade() != null && gradingScaleOptional.isPresent()) {
             builder.add(studentResult.overallGrade());
             headers.add("overall grade");
         }
-        if (studentResult.gradeWithBonus() != null) {
+        if (studentResult.gradeWithBonus() != null && gradingScaleOptional.isPresent()) {
             builder.add(studentResult.gradeWithBonus());
             headers.add("grade with bonus");
         }

src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportExerciseCreationService.java

@@ -1,6 +1,7 @@
 package de.tum.in.www1.artemis.service.dataexport;
 
 import static de.tum.in.www1.artemis.service.dataexport.DataExportQuizExerciseCreationService.TXT_FILE_EXTENSION;
+import static de.tum.in.www1.artemis.service.dataexport.DataExportUtil.createDirectoryIfNotExistent;
 import static de.tum.in.www1.artemis.service.dataexport.DataExportUtil.retrieveCourseDirPath;
 
 import java.io.File;
@@ -45,6 +46,8 @@ public class DataExportExerciseCreationService {
 
     private static final String PDF_FILE_EXTENSION = ".pdf";
 
+    private static final String EXERCISE_PREFIX = "exercise_";
+
     static final String CSV_FILE_EXTENSION = ".csv";
 
     private final Path repoClonePath;
@@ -96,15 +99,16 @@ public void createExercisesExport(Path workingDirectory, long userId) throws IOE
             var course = entry.getKey();
             Path courseDir = retrieveCourseDirPath(workingDirectory, course);
             var exercises = entry.getValue();
+            Path exercisesDir = courseDir.resolve("exercises");
             if (!exercises.isEmpty()) {
-                Files.createDirectory(courseDir);
+                createDirectoryIfNotExistent(exercisesDir);
             }
             for (var exercise : exercises) {
                 if (exercise instanceof ProgrammingExercise programmingExercise) {
-                    createProgrammingExerciseExport(programmingExercise, courseDir, userId);
+                    createProgrammingExerciseExport(programmingExercise, exercisesDir, userId);
                 }
                 else {
-                    createNonProgrammingExerciseExport(exercise, courseDir, userId);
+                    createNonProgrammingExerciseExport(exercise, exercisesDir, userId);
                 }
             }
         }
@@ -114,13 +118,12 @@ public void createExercisesExport(Path workingDirectory, long userId) throws IOE
      * Creates an export for a given programming exercise. Includes submission information, the repository from the VCS and potential plagiarism cases.
      *
      * @param programmingExercise the programming exercise for which the export should be created
-     * @param courseDir           the directory that is used for the course the exercise belongs to
+     * @param exercisesDir        the directory that is used for to store the exercises of the course the exercise belongs to
      * @param userId              the id of the user that requested the export
      * @throws IOException if an error occurs while accessing the file system
      */
-
-    public void createProgrammingExerciseExport(ProgrammingExercise programmingExercise, Path courseDir, long userId) throws IOException {
-        Path exerciseDir = courseDir.resolve(programmingExercise.getSanitizedExerciseTitle());
+    public void createProgrammingExerciseExport(ProgrammingExercise programmingExercise, Path exercisesDir, long userId) throws IOException {
+        Path exerciseDir = exercisesDir.resolve(EXERCISE_PREFIX + programmingExercise.getSanitizedExerciseTitle());
         if (!Files.exists(exerciseDir)) {
             Files.createDirectory(exerciseDir);
         }
@@ -156,7 +159,7 @@ public void createProgrammingExerciseExport(ProgrammingExercise programmingExerc
      * @throws IOException if an error occurs while accessing the file system
      */
     public void createNonProgrammingExerciseExport(Exercise exercise, Path courseDir, long userId) throws IOException {
-        Path exercisePath = courseDir.resolve(exercise.getSanitizedExerciseTitle());
+        Path exercisePath = courseDir.resolve(EXERCISE_PREFIX + exercise.getSanitizedExerciseTitle());
         if (!Files.exists(exercisePath)) {
             Files.createDirectory(exercisePath);
         }

src/main/java/de/tum/in/www1/artemis/service/dataexport/DataExportUtil.java

@@ -19,7 +19,7 @@ private DataExportUtil() {
 
     static void createDirectoryIfNotExistent(Path directory) throws IOException {
         if (!Files.exists(directory)) {
-            Files.createDirectory(directory);
+            Files.createDirectories(directory);
         }
     }
 

src/main/java/de/tum/in/www1/artemis/service/scheduled/DataExportScheduleService.java

@@ -56,7 +56,9 @@ public DataExportScheduleService(DataExportRepository dataExportRepository, Data
      * Created will be all data exports that are in the state REQUESTED OR IN_CREATION
      * Deleted will be all data exports that have a creation date older than seven days
      */
-    @Scheduled(cron = "${artemis.scheduling.data-export-creation-time: 0 0 4 * * *}")
+    // TODO change again after testing. As long as we do not use this cron expression one server test always fails
+    // @Scheduled(cron = "${artemis.scheduling.data-export-creation-time: 0 0 4 * * *}")
+    @Scheduled(cron = "0 */5 * * * *")
     public void createDataExportsAndDeleteOldOnes() {
         if (profileService.isDev()) {
             // do not execute this in a development environment

src/main/java/de/tum/in/www1/artemis/web/rest/DataExportResource.java

@@ -11,8 +11,7 @@
 
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.Resource;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
+import org.springframework.http.*;
 import org.springframework.web.bind.annotation.*;
 
 import de.tum.in.www1.artemis.domain.DataExport;
@@ -96,7 +95,11 @@ public ResponseEntity<Resource> downloadDataExport(@PathVariable long dataExport
         checkDataExportCanBeDownloaded(dataExport);
         Resource resource = dataExportService.downloadDataExport(dataExport);
         File finalZipFile = Path.of(dataExport.getFilePath()).toFile();
-        return ResponseEntity.ok().contentLength(finalZipFile.length()).contentType(MediaType.APPLICATION_OCTET_STREAM).header("filename", finalZipFile.getName()).body(resource);
+        ContentDisposition contentDisposition = ContentDisposition.builder("attachment").filename(finalZipFile.getName()).build();
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentDisposition(contentDisposition);
+        return ResponseEntity.ok().contentLength(finalZipFile.length()).headers(headers).contentType(MediaType.APPLICATION_OCTET_STREAM).header("filename", finalZipFile.getName())
+                .body(resource);
     }
 
     private void checkDataExportCanBeDownloaded(DataExport dataExport) {

src/main/webapp/app/core/legal/data-export/data-export.component.ts

@@ -4,9 +4,8 @@ import { Subject } from 'rxjs';
 import { ButtonSize, ButtonType } from 'app/shared/components/button.component';
 import { DataExportService } from 'app/core/legal/data-export/data-export.service';
 import { AccountService } from 'app/core/auth/account.service';
-import { HttpErrorResponse, HttpResponse } from '@angular/common/http';
+import { HttpErrorResponse } from '@angular/common/http';
 import { AlertService } from 'app/core/util/alert.service';
-import { downloadZipFileFromResponse } from 'app/shared/util/download.util';
 import { DataExport, DataExportState } from 'app/entities/data-export.model';
 import { ActivatedRoute } from '@angular/router';
 import { convertDateFromServer } from 'app/utils/date.utils';
@@ -94,9 +93,6 @@ export class DataExportComponent implements OnInit {
     }
 
     downloadDataExport() {
-        this.dataExportService.downloadDataExport(this.dataExportId).subscribe((response: HttpResponse<Blob>) => {
-            downloadZipFileFromResponse(response);
-            this.alertService.success('artemisApp.dataExport.downloadSuccess');
-        });
+        this.dataExportService.downloadDataExport(this.dataExportId);
     }
 }

src/main/webapp/app/core/legal/data-export/data-export.service.ts

@@ -1,4 +1,4 @@
-import { HttpClient, HttpResponse } from '@angular/common/http';
+import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
 import { Observable } from 'rxjs';
 import { DataExport } from 'app/entities/data-export.model';
@@ -11,11 +11,9 @@ export class DataExportService {
         return this.http.post<DataExport>(`api/data-exports`, {});
     }
 
-    downloadDataExport(dataExportId: number): Observable<HttpResponse<Blob>> {
-        return this.http.get(`api/data-exports/${dataExportId}`, {
-            observe: 'response',
-            responseType: 'blob',
-        });
+    downloadDataExport(dataExportId: number) {
+        const url = `api/data-exports/${dataExportId}`;
+        window.open(url, '_blank');
     }
 
     canRequestDataExport(): Observable<boolean> {

src/test/java/de/tum/in/www1/artemis/assessment/GradingScaleUtilService.java

@@ -8,19 +8,25 @@
 
 import javax.validation.constraints.NotNull;
 
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.ResourceUtils;
 
 import com.opencsv.CSVReader;
 
 import de.tum.in.www1.artemis.domain.*;
+import de.tum.in.www1.artemis.domain.exam.Exam;
+import de.tum.in.www1.artemis.repository.GradingScaleRepository;
 
 /**
  * Service responsible for initializing the database with specific testdata related to grading for use in integration tests.
  */
 @Service
 public class GradingScaleUtilService {
 
+    @Autowired
+    private GradingScaleRepository gradingScaleRepository;
+
     @NotNull
     public Set<GradeStep> generateGradeStepSet(GradingScale gradingScale, boolean valid) {
         GradeStep gradeStep1 = new GradeStep();
@@ -64,6 +70,13 @@ public GradingScale generateGradingScale(int gradeStepCount, double[] intervals,
         return gradingScale;
     }
 
+    public GradingScale generateAndSaveGradingScale(int gradeStepCount, double[] intervals, boolean lowerBoundInclusivity, int firstPassingIndex, Optional<String[]> gradeNames,
+            Exam exam) {
+        GradingScale gradingScale = generateGradingScale(gradeStepCount, intervals, lowerBoundInclusivity, firstPassingIndex, gradeNames);
+        gradingScale.setExam(exam);
+        return gradingScaleRepository.save(gradingScale);
+    }
+
     public GradingScale generateGradingScale(int gradeStepCount, double[] intervals, boolean lowerBoundInclusivity, int firstPassingIndex, Optional<String[]> gradeNames) {
         if (gradeStepCount != intervals.length - 1 || firstPassingIndex >= gradeStepCount || firstPassingIndex < 0) {
             fail("Invalid grading scale parameters");

src/test/java/de/tum/in/www1/artemis/course/CourseUtilService.java

@@ -11,6 +11,7 @@
 import org.springframework.stereotype.Service;
 
 import de.tum.in.www1.artemis.assessment.ComplaintUtilService;
+import de.tum.in.www1.artemis.assessment.GradingScaleUtilService;
 import de.tum.in.www1.artemis.competency.CompetencyUtilService;
 import de.tum.in.www1.artemis.domain.*;
 import de.tum.in.www1.artemis.domain.enumeration.*;
@@ -138,6 +139,9 @@ public class CourseUtilService {
     @Autowired
     private ComplaintUtilService complaintUtilService;
 
+    @Autowired
+    private GradingScaleUtilService gradingScaleUtilService;
+
     public Course createCourse() {
         return createCourse(null);
     }
@@ -878,11 +882,12 @@ public void updateCourseGroups(String userPrefix, Course course, String suffix)
         courseRepo.save(course);
     }
 
-    public Course createCourseWithCustomStudentUserGroupWithExamAndExerciseGroupAndExercises(User user, String studentGroupName, String shortName, boolean withProgrammingExercise,
-            boolean withAllQuizQuestionTypes) {
+    public Course createCourseWithCustomStudentUserGroupWithExamAndExerciseGroupAndExercisesAndGradingScale(User user, String studentGroupName, String shortName,
+            boolean withProgrammingExercise, boolean withAllQuizQuestionTypes) {
         Course course = createCourseWithCustomStudentGroupName(studentGroupName, shortName);
         Exam exam = examUtilService.addExam(course, user, ZonedDateTime.now().minusMinutes(10), ZonedDateTime.now().minusMinutes(5), ZonedDateTime.now().minusMinutes(2),
                 ZonedDateTime.now().minusMinutes(1));
+        gradingScaleUtilService.generateAndSaveGradingScale(2, new double[] { 0, 50, 100 }, true, 1, Optional.empty(), exam);
         course.addExam(exam);
         examUtilService.addExerciseGroupsAndExercisesToExam(exam, withProgrammingExercise, withAllQuizQuestionTypes);
         return courseRepo.save(course);