Overhauled the end to end tests and added the test code from progpedia

core/src/main/java/de/jplag/GreedyStringTiling.java

@@ -23,8 +23,8 @@ public class GreedyStringTiling {
 
     private final int minimumMatchLength;
     private final int minimumNeighborLength;
-    private JPlagOptions options;
-    private ConcurrentMap<TokenType, Integer> tokenTypeValues;
+    private final JPlagOptions options;
+    private final ConcurrentMap<TokenType, Integer> tokenTypeValues;
     private final Map<Submission, Set<Token>> baseCodeMarkings = new IdentityHashMap<>();
 
     private final Map<Submission, int[]> cachedTokenValueLists = new IdentityHashMap<>();
@@ -81,9 +81,17 @@ public final JPlagComparison compare(Submission firstSubmission, Submission seco
         if (firstSubmission.getTokenList().size() > secondSubmission.getTokenList().size()) {
             smallerSubmission = secondSubmission;
             largerSubmission = firstSubmission;
-        } else {
+        } else if (firstSubmission.getTokenList().size() < secondSubmission.getTokenList().size()) {
             smallerSubmission = firstSubmission;
             largerSubmission = secondSubmission;
+        } else {
+            if (firstSubmission.getName().compareTo(secondSubmission.getName()) < 0) {
+                smallerSubmission = firstSubmission;
+                largerSubmission = secondSubmission;
+            } else {
+                smallerSubmission = secondSubmission;
+                largerSubmission = firstSubmission;
+            }
         }
         return compareInternal(smallerSubmission, largerSubmission);
     }

endtoend-testing/pom.xml

@@ -18,7 +18,6 @@
             <groupId>de.jplag</groupId>
             <artifactId>cli</artifactId>
             <version>${revision}</version>
-            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>de.jplag</groupId>

endtoend-testing/src/main/java/de/jplag/endtoend/constants/TestDirectoryConstants.java

@@ -12,18 +12,17 @@ private TestDirectoryConstants() {
     }
 
     /**
-     * Base path to the saved results
+     * Base path to the resources directory
      */
-    public static final Path BASE_PATH_TO_RESULT_JSON = Path.of("src", "test", "resources", "results");
+    public static final Path BASE_PATH_TO_RESOURCES = Path.of("src", "test", "resources");
 
     /**
-     * Base path to the endToEnd testing resources
+     * Base path to the saved results
      */
-    public static final Path BASE_PATH_TO_LANGUAGE_RESOURCES = Path.of("src", "test", "resources", "languageTestFiles");
+    public static final Path BASE_PATH_TO_RESULT_JSON = BASE_PATH_TO_RESOURCES.resolve(Path.of("results"));
 
     /**
-     * Create the complete path to the submission files. Here the temporary system path is extended with the
-     * "SUBMISSION_DIRECTORY_NAME", which is predefined in this class.
+     * Base path to the data set descriptors
      */
-    public static final Path TEMPORARY_SUBMISSION_DIRECTORY_NAME = Path.of("target", "testing-directory-submission");
+    public static final Path BASE_PATH_TO_DATA_SET_DESCRIPTORS = BASE_PATH_TO_RESOURCES.resolve(Path.of("dataSets"));
 }

endtoend-testing/src/main/java/de/jplag/endtoend/helper/AverageCalculator.java

@@ -0,0 +1,34 @@
+package de.jplag.endtoend.helper;
+
+/**
+ * Helper to calculate the average of multiple numbers.
+ */
+public class AverageCalculator {
+    private double sum;
+    private int count;
+
+    /**
+     * Creates a new empty calculator
+     */
+    public AverageCalculator() {
+        this.sum = 0;
+        this.count = 0;
+    }
+
+    /**
+     * Adds a new value to the calculation
+     * @param value The value
+     */
+    public void add(double value) {
+        this.sum += value;
+        this.count++;
+    }
+
+    /**
+     * Calculates and returns the average of all added values.
+     * @return The result
+     */
+    public double calculate() {
+        return this.sum / count;
+    }
+}

endtoend-testing/src/main/java/de/jplag/endtoend/helper/LanguageDeserializer.java

@@ -0,0 +1,21 @@
+package de.jplag.endtoend.helper;
+
+import java.io.IOException;
+
+import de.jplag.Language;
+import de.jplag.cli.LanguageLoader;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+
+/**
+ * Deserialized a language from a json file
+ */
+public class LanguageDeserializer extends JsonDeserializer<Language> {
+    @Override
+    public Language deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
+        String name = jsonParser.getText();
+        return LanguageLoader.getLanguage(name).orElseThrow(() -> new IllegalStateException(String.format("Language %s not found.", name)));
+    }
+}

endtoend-testing/src/main/java/de/jplag/endtoend/helper/TestSuiteHelper.java

@@ -1,13 +1,10 @@
 package de.jplag.endtoend.helper;
 
-import java.io.File;
-import java.util.List;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import de.jplag.JPlagComparison;
-import de.jplag.Language;
 import de.jplag.Submission;
-import de.jplag.endtoend.constants.TestDirectoryConstants;
 
 /**
  * Helper class to perform all necessary additional functions for the endToEnd tests.
@@ -28,30 +25,8 @@ private TestSuiteHelper() {
      * @return unique identifier for test case recognition
      */
     public static String getTestIdentifier(JPlagComparison jPlagComparison) {
-        return List.of(jPlagComparison.firstSubmission(), jPlagComparison.secondSubmission()).stream().map(Submission::getRoot)
+        return Stream.of(jPlagComparison.firstSubmission(), jPlagComparison.secondSubmission()).map(Submission::getRoot)
                 .map(FileHelper::getFileNameWithoutFileExtension).sorted().collect(Collectors.joining("-"));
 
     }
-
-    /**
-     * Returns the file pointing to the directory of the submissions for the given language and result json. The result
-     * json's name is expected to be equal to the test suite identifier.
-     * @param language is the language for the tests
-     * @param resultJSON is the json containing the expected values
-     * @return returns the directory of the submissions
-     */
-    public static File getSubmissionDirectory(Language language, File resultJSON) {
-        return getSubmissionDirectory(language, FileHelper.getFileNameWithoutFileExtension(resultJSON));
-    }
-
-    /**
-     * Returns the file pointing to the directory of the submissions for the given language and test suite identifier as
-     * described in the Readme.md.
-     * @param language is the langauge for the tests
-     * @param testSuiteIdentifier is the test suite identifier of the tests
-     * @return returns the directory of the submissions
-     */
-    public static File getSubmissionDirectory(Language language, String testSuiteIdentifier) {
-        return TestDirectoryConstants.BASE_PATH_TO_LANGUAGE_RESOURCES.resolve(language.getIdentifier()).resolve(testSuiteIdentifier).toFile();
-    }
 }

endtoend-testing/src/main/java/de/jplag/endtoend/model/ComparisonIdentifier.java

@@ -0,0 +1,52 @@
+package de.jplag.endtoend.model;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.util.HashSet;
+import java.util.Scanner;
+import java.util.Set;
+
+/**
+ * Identifier for a comparison. The order of the names does not matter
+ * @param firstName The first name
+ * @param secondName The second name
+ */
+public record ComparisonIdentifier(String firstName, String secondName) {
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof ComparisonIdentifier other)) {
+            return false;
+        }
+
+        return (firstName.equals(other.firstName) && secondName.equals(other.secondName))
+                || (secondName.equals(other.firstName) && firstName.equals(other.secondName));
+    }
+
+    @Override
+    public int hashCode() {
+        return firstName.hashCode() + secondName.hashCode();
+    }
+
+    /**
+     * Loads the identifiers stored in a csv (semicolon separated) file.
+     * @param file The file to load
+     * @return The comparisons in the file
+     */
+    public static Set<ComparisonIdentifier> loadIdentifiersFromFile(File file) {
+        try (Scanner scanner = new Scanner(file)) {
+            Set<ComparisonIdentifier> identifiers = new HashSet<>();
+            while (scanner.hasNextLine()) {
+                String[] parts = scanner.nextLine().split(";");
+                identifiers.add(new ComparisonIdentifier(parts[0], parts[1]));
+            }
+            return identifiers;
+        } catch (FileNotFoundException e) {
+            throw new IllegalStateException(String.format("Comparisons could not be loaded for %s.", file.getName()), e);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return firstName + " - " + secondName;
+    }
+}

endtoend-testing/src/main/java/de/jplag/endtoend/model/DataSet.java

@@ -0,0 +1,83 @@
+package de.jplag.endtoend.model;
+
+import java.io.File;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import de.jplag.Language;
+import de.jplag.endtoend.constants.TestDirectoryConstants;
+import de.jplag.endtoend.helper.LanguageDeserializer;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+/**
+ * Represents a data set for the endToEnd test suite.
+ * @param name The name of the data set
+ * @param language The language
+ * @param format The format
+ * @param sourceDirectory The source directory, may be null
+ * @param resultFile The result file name, may be null
+ * @param goldStandardFile The gold standard file, may be null
+ * @param options The options for the jplag runs, may be null
+ */
+public record DataSet(@JsonProperty(required = true) String name,
+        @JsonDeserialize(using = LanguageDeserializer.class) @JsonProperty(required = true) Language language,
+        @JsonProperty(required = true) DataSetFormat format, @JsonProperty String sourceDirectory, @JsonProperty String resultFile,
+        @JsonProperty String goldStandardFile, @JsonProperty Options options) {
+
+    /**
+     * Gets the source directories
+     * @return The source directories
+     */
+    public Set<File> getSourceDirectories() {
+        return format.getSourceDirectories(this).stream()
+                .map(file -> new File(TestDirectoryConstants.BASE_PATH_TO_RESOURCES.toFile(), file.getPath())).collect(Collectors.toSet());
+    }
+
+    /**
+     * Helper function replacing null by the default value
+     * @return The source directory
+     */
+    String actualSourceDirectory() {
+        if (sourceDirectory == null) {
+            return "data/" + name;
+        }
+        return sourceDirectory;
+    }
+
+    /**
+     * Helper function replacing null by the default value
+     * @return The result file
+     */
+    public File getResultFile() {
+        if (resultFile == null) {
+            return new File(TestDirectoryConstants.BASE_PATH_TO_RESULT_JSON.toFile(), name + ".json");
+        } else {
+            return new File(TestDirectoryConstants.BASE_PATH_TO_RESULT_JSON.toFile(), resultFile);
+        }
+    }
+
+    /**
+     * @return The gold standard file. Can be null.
+     */
+    public File getGoldStandardFile() {
+        if (goldStandardFile == null) {
+            return null;
+        }
+
+        return new File(TestDirectoryConstants.BASE_PATH_TO_RESOURCES.toFile(), goldStandardFile);
+    }
+
+    /**
+     * Helper function replacing null by the default value.
+     * @return The options
+     */
+    public Options getOptions() {
+        if (this.options != null) {
+            return this.options;
+        } else {
+            return new Options(null, null);
+        }
+    }
+}

endtoend-testing/src/main/java/de/jplag/endtoend/model/DataSetFormat.java

@@ -0,0 +1,55 @@
+package de.jplag.endtoend.model;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * The available formats for data sets.
+ */
+@SuppressWarnings("unused") // The formats only referred to from the data set configuration
+public enum DataSetFormat {
+    /**
+     * The progpedia format
+     */
+    PROGPEDIA {
+        @Override
+        public Set<File> getSourceDirectories(DataSet dataSet) {
+            return Collections.singleton(new File(dataSet.actualSourceDirectory(), "ACCEPTED"));
+        }
+
+        @Override
+        public File getBaseCodeDirectory(DataSet dataSet, String directoryName) {
+            return new File(dataSet.actualSourceDirectory(), directoryName);
+        }
+    },
+    /**
+     * Plain format where the submissions are flat within the source directory.
+     */
+    PLAIN {
+        @Override
+        public Set<File> getSourceDirectories(DataSet dataSet) {
+            return Collections.singleton(new File(dataSet.actualSourceDirectory()));
+        }
+
+        @Override
+        public File getBaseCodeDirectory(DataSet dataSet, String directoryName) {
+            throw new IllegalStateException("Plain formatted data sets cannot include base code.");
+        }
+    };
+
+    /**
+     * Resolved the source directories for jplag
+     * @param dataSet The data set
+     * @return The source directories
+     */
+    public abstract Set<File> getSourceDirectories(DataSet dataSet);
+
+    /**
+     * Resolves the base code directory
+     * @param dataSet The data set
+     * @param directoryName The name of the base code directory
+     * @return The base code directory
+     */
+    public abstract File getBaseCodeDirectory(DataSet dataSet, String directoryName);
+}