diff --git a/.github/workflows/files/progpedia.zip b/.github/workflows/files/progpedia.zip index 310d196fa..7af75298c 100644 Binary files a/.github/workflows/files/progpedia.zip and b/.github/workflows/files/progpedia.zip differ diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 443f3c50f..e83f58f23 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,7 +1,7 @@ name: Build on: - push: + push: paths: - ".github/workflows/maven.yml" - "**/pom.xml" @@ -14,7 +14,7 @@ on: - "**/pom.xml" - "**.java" - "**.g4" - + # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -43,17 +43,21 @@ jobs: with: java-version: 21 distribution: 'temurin' - + + - uses: actions/setup-node@v4 + with: + node-version: "18" + - name: Run Tests run: mvn verify -B -U - + - name: Build Assembly - run: mvn clean package assembly:single - + run: mvn -Pwith-report-viewer clean package assembly:single + - name: Upload Assembly uses: actions/upload-artifact@v4 with: name: "JPlag" path: "jplag.cli/target/jplag-*-jar-with-dependencies.jar" - + diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5a31d8afe..b4ea875e3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,7 +13,7 @@ jobs: java-version: '21' distribution: 'temurin' - name: Set maven settings.xml - uses: whelk-io/maven-settings-xml-action@v21 + uses: whelk-io/maven-settings-xml-action@v22 with: servers: '[{ "id": "ossrh", "username": "jplag", "password": "${{ secrets.OSSRH_TOKEN }}" }]' - name: Import GPG key @@ -34,8 +34,12 @@ jobs: with: java-version: '21' distribution: 'temurin' + - uses: actions/setup-node@v4 + with: + node-version: "18" + - name: Build JPlag - run: mvn -U -B clean package assembly:single + run: mvn -Pwith-report-viewer -U -B clean package assembly:single - name: Attach CLI to Release on GitHub uses: softprops/action-gh-release@v1 diff --git a/.github/workflows/sonarcloud-branch.yml b/.github/workflows/sonarcloud-branch.yml index d47c15001..486fb0777 100644 --- a/.github/workflows/sonarcloud-branch.yml +++ b/.github/workflows/sonarcloud-branch.yml @@ -42,7 +42,7 @@ jobs: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 - + - name: Build and analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sonarcloud-pr.yml b/.github/workflows/sonarcloud-pr.yml index ca89871fc..5c3cfc300 100644 --- a/.github/workflows/sonarcloud-pr.yml +++ b/.github/workflows/sonarcloud-pr.yml @@ -65,7 +65,7 @@ jobs: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 - + - name: Build and analyze (PR) env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/spotless.yml b/.github/workflows/spotless.yml index 0357fd108..51058bb69 100644 --- a/.github/workflows/spotless.yml +++ b/.github/workflows/spotless.yml @@ -14,7 +14,7 @@ on: - "**/pom.xml" - "**.java" - "**.g4" - + # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -43,9 +43,9 @@ jobs: with: java-version: 21 distribution: 'temurin' - + - name: Check with Spotless run: mvn clean spotless:check - - + + diff --git a/README.md b/README.md index aaa5ed368..303ce1a57 100644 --- a/README.md +++ b/README.md @@ -10,34 +10,42 @@ [![GitHub commit activity](https://img.shields.io/github/commit-activity/y/jplag/JPlag)](https://github.com/jplag/JPlag/pulse) [![SonarCloud Coverage](https://sonarcloud.io/api/project_badges/measure?project=jplag_JPlag&metric=coverage)](https://sonarcloud.io/component_measures/metric/coverage/list?id=jplag_JPlag) [![Report Viewer](https://img.shields.io/badge/report%20viewer-online-b80025)](https://jplag.github.io/JPlag/) -[![Java Version](https://img.shields.io/badge/java-SE%2017-yellowgreen)](#download-and-installation) +[![Java Version](https://img.shields.io/badge/java-SE%2021-yellowgreen)](#download-and-installation) -JPlag is a system that finds similarities among multiple sets of source code files. This way it can detect software plagiarism and collusion in software development. JPlag currently supports various programming languages, EMF metamodels, and natural language text. +JPlag finds pairwise similarities among a set of multiple programs. It can reliably detect software plagiarism and collusion in software development, even when obfuscated. All similarities are calculated locally, and no source code or plagiarism results are ever uploaded to the internet. JPlag supports a large number of programming and modeling languages. + +* 📈 [JPlag Demo](https://jplag.github.io/Demo/) + +* 🏛️ [JPlag on Helmholtz RSD](https://helmholtz.software/software/jplag) + +* 🤩 [Give us Feedback in a **short (<5 min) survey**](https://docs.google.com/forms/d/e/1FAIpQLSckqUlXhIlJ-H2jtu2VmGf_mJt4hcnHXaDlwhpUL3XG1I8UYw/viewform?usp=sf_link) + ## Supported Languages -In the following, a list of all supported languages with their supported language version is provided. A language can be selected from the command line using subcommands (jplag [jplag options] [language options]). Alternatively you can use the legacy "-l" argument. +All supported languages and their supported versions are listed below. | Language | Version | CLI Argument Name | [state](https://github.com/jplag/JPlag/wiki/2.-Supported-Languages) | parser | |--------------------------------------------------------|---------------------------------------------------------------------------------------:|-------------------|:-------------------------------------------------------------------:|:---------:| | [Java](https://www.java.com) | 21 | java | mature | JavaC | -| [C/C++](https://isocpp.org) | 11 | cpp | legacy | JavaCC | -| [C/C++](https://isocpp.org) | 14 | cpp2 | beta | ANTLR 4 | -| [C#](https://docs.microsoft.com/en-us/dotnet/csharp/) | 6 | csharp | beta | ANTLR 4 | +| [C](https://isocpp.org) | 11 | c | legacy | JavaCC | +| [C++](https://isocpp.org) | 14 | cpp | beta | ANTLR 4 | +| [C#](https://docs.microsoft.com/en-us/dotnet/csharp/) | 6 | csharp | mature | ANTLR 4 | +| [Python](https://www.python.org) | 3.6 | python3 | legacy | ANTLR 4 | +| [JavaScript](https://www.javascript.com/) | ES6 | javascript | beta | ANTLR 4 | +| [TypeScript](https://www.typescriptlang.org/) | [~5](https://github.com/antlr/grammars-v4/tree/master/javascript/typescript/README.md) | typescript | beta | ANTLR 4 | | [Go](https://go.dev) | 1.17 | golang | beta | ANTLR 4 | | [Kotlin](https://kotlinlang.org) | 1.3 | kotlin | beta | ANTLR 4 | -| [Python](https://www.python.org) | 3.6 | python3 | legacy | ANTLR 4 | | [R](https://www.r-project.org/) | 3.5.0 | rlang | beta | ANTLR 4 | | [Rust](https://www.rust-lang.org/) | 1.60.0 | rust | beta | ANTLR 4 | -| [Scala](https://www.scala-lang.org) | 2.13.8 | scala | beta | Scalameta | -| [Scheme](http://www.scheme-reports.org) | ? | scheme | unknown | JavaCC | | [Swift](https://www.swift.org) | 5.4 | swift | beta | ANTLR 4 | +| [Scala](https://www.scala-lang.org) | 2.13.8 | scala | beta | Scalameta | +| [LLVM IR](https://llvm.org) | 15 | llvmir | beta | ANTLR 4 | +| [Scheme](http://www.scheme-reports.org) | ? | scheme | legacy | JavaCC | | [EMF Metamodel](https://www.eclipse.org/modeling/emf/) | 2.25.0 | emf | beta | EMF | | [EMF Model](https://www.eclipse.org/modeling/emf/) | 2.25.0 | emf-model | alpha | EMF | -| [LLVM IR](https://llvm.org) | 15 | llvmir | beta | ANTLR 4 | -| [TypeScript](https://www.typescriptlang.org/) | [~5](https://github.com/antlr/grammars-v4/tree/master/javascript/typescript/README.md) | typescript | beta | ANTLR 4 | -| JavaScript | ES6 | javascript | beta | ANTLR 4 | +| [SCXML](https://www.w3.org/TR/scxml/) | 1.0 | scxml | alpha | XML | | Text (naive) | - | text | legacy | CoreNLP | ## Download and Installation @@ -53,6 +61,7 @@ JPlag is released on [Maven Central](https://search.maven.org/search?q=de.jplag) de.jplag jplag + ``` @@ -60,94 +69,70 @@ JPlag is released on [Maven Central](https://search.maven.org/search?q=de.jplag) 1. Download or clone the code from this repository. 2. Run `mvn clean package` from the root of the repository to compile and build all submodules. Run `mvn clean package assembly:single` instead if you need the full jar which includes all dependencies. -5. You will find the generated JARs in the subdirectory `cli/target`. + Run `mvn -P with-report-viewer clean package assembly:single` to build the full jar with the report viewer. In this case, you'll need [Node.js](https://nodejs.org/en/download) installed. +3. You will find the generated JARs in the subdirectory `cli/target`. ## Usage JPlag can either be used via the CLI or directly via its Java API. For more information, see the [usage information in the wiki](https://github.com/jplag/JPlag/wiki/1.-How-to-Use-JPlag). If you are using the CLI, you can display your results via [jplag.github.io](https://jplag.github.io/JPlag/). No data will leave your computer! ### CLI *Note that the [legacy CLI](https://github.com/jplag/jplag/blob/legacy/README.md) is varying slightly.* - -The language can either be set with the -l parameter or as a subcommand. If both a subcommand and the -l option are specified, the subcommand will take priority. -When using the subcommand language specific arguments can be set. -A list of language specific options can be obtained by requesting the help page of a subcommand (e.g. "jplag java -h"). +The language can either be set with the -l parameter or as a subcommand (`jplag [jplag options] [language options]`). A subcommand takes priority over the -l option. +When using the subcommand, language-specific arguments can be set. A list of language-specific options can be obtained by requesting the help page of a subcommand (e.g. `jplag java -h`). ``` -Usage: jplag [OPTIONS] [root-dirs[,root-dirs...]...] [COMMAND] - +Parameter descriptions: [root-dirs[,root-dirs...]...] - Root-directory with submissions to check for plagiarism - + Root-directory with submissions to check for plagiarism. -bc, --bc, --base-code= - Path of the directory containing the base code - (common framework used in all submissions) - - -h, --help display this help and exit - -l, --language= - Select the language to parse the submissions (default: - java). The language names are the same as the - subcommands. - - -n, --shown-comparisons= - The maximum number of comparisons that will be shown - in the generated report, if set to -1 all comparisons - will be shown (default: 100) - + Path to the base code directory (common framework used in all submissions). + -l, --language= + Select the language of the submissions (default: java). See subcommands below. + -M, --mode=<{RUN, VIEW, RUN_AND_VIEW}> + The mode of JPlag: either only run analysis, only open the viewer, or do both (default: null) + -n, --shown-comparisons= + The maximum number of comparisons that will be shown in the generated report, if set to -1 all comparisons will be shown (default: 500) -new, --new=[,...] - Root-directory with submissions to check for plagiarism - (same as the root directory) - + Root-directories with submissions to check for plagiarism (same as root). + --normalize Activate the normalization of tokens. Supported for languages: Java, C++. -old, --old=[,...] - Root-directory with prior submissions to compare against - - -r, --result-directory= - Name of the directory in which the comparison results - will be stored (default: result) - - -t, --min-tokens= - Tunes the comparison sensitivity by adjusting the - minimum token required to be counted as a matching - section. A smaller increases the sensitivity but - might lead to more false-positives + Root-directories with prior submissions to compare against. + -r, --result-file= + Name of the file in which the comparison results will be stored (default: results). Missing .zip endings will be automatically added. + -t, --min-tokens= + Tunes the comparison sensitivity by adjusting the minimum token required to be counted as a matching section. A smaller value increases the sensitivity but might lead to more + false-positives. Advanced - -d, --debug Debug parser. Non-parsable files will be stored - (default: false) - - -m, --similarity-threshold= - Comparison similarity threshold [0.0-1.0]: All - comparisons above this threshold will be saved - (default: 0.0) - - -p, --suffixes=[,...] - comma-separated list of all filename suffixes that are - included - - -s, --subdirectory= - Look in directories /*/ for programs - - -x, --exclusion-file= - All files named in this file will be ignored in the - comparison (line-separated list) + --csv-export Export pairwise similarity values as a CSV file. + -d, --debug Store on-parsable files in error folder. + -m, --similarity-threshold= + Comparison similarity threshold [0.0-1.0]: All comparisons above this threshold will be saved (default: 0.0). + -p, --suffixes=[,...] + comma-separated list of all filename suffixes that are included. + -P, --port= The port used for the internal report viewer (default: 1996). + -s, --subdirectory= + Look in directories /*/ for programs. + -x, --exclusion-file= + All files named in this file will be ignored in the comparison (line-separated list). Clustering - --cluster-alg, --cluster-algorithm= - Which clustering algorithm to use. Agglomerative merges - similar submissions bottom up. Spectral clustering is - combined with Bayesian Optimization to execute - the k-Means clustering algorithm multiple times, - hopefully finding a "good" clustering - automatically. (default: spectral) - - --cluster-metric= - The metric used for clustering. AVG is intersection - over union, MAX can expose some attempts of - obfuscation. (default: MAX) - - --cluster-skip Skips the clustering (default: false) -Commands: + --cluster-alg, --cluster-algorithm=<{AGGLOMERATIVE, SPECTRAL}> + Specifies the clustering algorithm (default: spectral). + --cluster-metric=<{AVG, MIN, MAX, INTERSECTION}> + The similarity metric used for clustering (default: average similarity). + --cluster-skip Skips the cluster calculation. + +Subsequence Match Merging + --gap-size= + Maximal gap between neighboring matches to be merged (between 1 and minTokenMatch, default: 6). + --match-merging Enables merging of neighboring matches to counteract obfuscation attempts. + --neighbor-length= + Minimal length of neighboring matches to be merged (between 1 and minTokenMatch, default: 2). + +Subcommands (supported languages): + c cpp - cpp2 csharp emf emf-model @@ -174,20 +159,21 @@ The new API makes it easy to integrate JPlag's plagiarism detection into externa ```java -JavaLanguage language = new JavaLanguage(); -language.getOptions(); //Use the object returned by this to set language options(same as language specific arguments above). +Language language = new JavaLanguage(); Set submissionDirectories = Set.of(new File("/path/to/rootDir")); File baseCode = new File("/path/to/baseCode"); JPlagOptions options = new JPlagOptions(language, submissionDirectories, Set.of()).withBaseCodeSubmissionDirectory(baseCode); try { JPlagResult result = JPlag.run(options); - + // Optional - ReportObjectFactory reportObjectFactory = new ReportObjectFactory(); - reportObjectFactory.createAndSaveReport(result, "/path/to/output"); + ReportObjectFactory reportObjectFactory = new ReportObjectFactory(new File("/path/to/output")); + reportObjectFactory.createAndSaveReport(result); } catch (ExitException e) { // error handling here +} catch (FileNotFoundException e) { + // handle IO exception here } ``` diff --git a/cli/pom.xml b/cli/pom.xml index b8851012e..459daf3eb 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -7,6 +7,7 @@ ${revision} cli + @@ -44,12 +45,12 @@ de.jplag - cpp + c ${revision} de.jplag - cpp2 + cpp ${revision} @@ -128,6 +129,12 @@ picocli 4.7.5 + + + me.tongfei + progressbar + 0.10.0 + @@ -161,4 +168,56 @@ + + + + with-report-viewer + + + + report-viewer + ../report-viewer/dist + + + + + org.codehaus.mojo + exec-maven-plugin + 1.3.2 + + + npm install + + exec + + generate-resources + + npm + ../report-viewer + + install + + + + + npm build + + exec + + generate-resources + + npm + ../report-viewer + + run + build + + + + + + + + + diff --git a/cli/src/main/java/de/jplag/cli/CLI.java b/cli/src/main/java/de/jplag/cli/CLI.java index 029e6455d..ac79e68c0 100644 --- a/cli/src/main/java/de/jplag/cli/CLI.java +++ b/cli/src/main/java/de/jplag/cli/CLI.java @@ -4,7 +4,11 @@ import static picocli.CommandLine.Model.UsageMessageSpec.SECTION_KEY_OPTION_LIST; import static picocli.CommandLine.Model.UsageMessageSpec.SECTION_KEY_SYNOPSIS; +import java.awt.Desktop; import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; import java.security.SecureRandom; import java.util.Arrays; import java.util.HashSet; @@ -21,9 +25,12 @@ import de.jplag.JPlagResult; import de.jplag.Language; import de.jplag.cli.logger.CollectedLoggerFactory; +import de.jplag.cli.logger.TongfeiProgressBarProvider; +import de.jplag.cli.server.ReportViewer; import de.jplag.clustering.ClusteringOptions; import de.jplag.clustering.Preprocessing; import de.jplag.exceptions.ExitException; +import de.jplag.logging.ProgressBarLogger; import de.jplag.merging.MergingOptions; import de.jplag.options.JPlagOptions; import de.jplag.options.LanguageOption; @@ -45,12 +52,12 @@ public final class CLI { private static final Random RANDOM = new SecureRandom(); - private static final String CREDITS = "Created by IPD Tichy, Guido Malpohl, and others. JPlag logo designed by Sandro Koch. Currently maintained by Sebastian Hahner and Timur Saglam."; + private static final String CREDITS = "Created by IPD Tichy, Guido Malpohl, and others. Maintained by Timur Saglam and Sebastian Hahner. Logo by Sandro Koch."; private static final String[] DESCRIPTIONS = {"Detecting Software Plagiarism", "Software-Archaeological Playground", "Since 1996", "Scientifically Published", "Maintained by SDQ", "RIP Structure and Table", "What else?", "You have been warned!", "Since Java 1.0", - "More Abstract than Tree", "Students Nightmare", "No, changing variable names does not work", "The tech is out there!", - "Developed by plagiarism experts."}; + "More Abstract than Tree", "Students Nightmare", "No, changing variable names does not work...", "The tech is out there!", + "Developed by plagiarism experts.", "State of the Art Obfuscation Resilience", "www.helmholtz.software/software/jplag"}; private static final String OPTION_LIST_HEADING = "Parameter descriptions: "; @@ -63,6 +70,8 @@ public final class CLI { private static final String DESCRIPTION_PATTERN = "%nJPlag - %s%n%s%n%n"; + private static final String DEFAULT_FILE_ENDING = ".zip"; + /** * Main class for using JPlag via the CLI. * @param args are the CLI arguments that will be passed to JPlag. @@ -76,15 +85,20 @@ public static void main(String[] args) { ParseResult parseResult = cli.parseOptions(args); if (!parseResult.isUsageHelpRequested() && !(parseResult.subcommand() != null && parseResult.subcommand().isUsageHelpRequested())) { - JPlagOptions options = cli.buildOptionsFromArguments(parseResult); - JPlagResult result = JPlag.run(options); - ReportObjectFactory reportObjectFactory = new ReportObjectFactory(); - reportObjectFactory.createAndSaveReport(result, cli.getResultFolder()); - - OutputFileGenerator.generateCsvOutput(result, new File(cli.getResultFolder()), cli.options); + ProgressBarLogger.setProgressBarProvider(new TongfeiProgressBarProvider()); + switch (cli.options.mode) { + case RUN -> cli.runJPlag(parseResult); + case VIEW -> cli.runViewer(null); + case RUN_AND_VIEW -> cli.runViewer(cli.runJPlag(parseResult)); + } + } + } catch (ExitException | IOException exception) { // do not pass exceptions here to keep log clean + if (exception.getCause() != null) { + logger.error("{} - {}", exception.getMessage(), exception.getCause().getMessage()); + } else { + logger.error(exception.getMessage()); } - } catch (ExitException exception) { - logger.error(exception.getMessage()); // do not pass exception here to keep log clean + finalizeLogger(); System.exit(1); } @@ -102,9 +116,8 @@ public CLI() { this.commandLine.getHelpSectionMap().put(SECTION_KEY_OPTION_LIST, help -> help.optionList().lines().map(it -> { if (it.startsWith(" -")) { return " " + it; - } else { - return it; } + return it; }).collect(Collectors.joining(System.lineSeparator()))); buildSubcommands().forEach(commandLine::addSubcommand); @@ -114,6 +127,29 @@ public CLI() { this.commandLine.setAllowSubcommandsAsOptionParameters(true); } + public File runJPlag(ParseResult parseResult) throws ExitException, FileNotFoundException { + JPlagOptions jplagOptions = buildOptionsFromArguments(parseResult); + JPlagResult result = JPlag.run(jplagOptions); + File target = new File(getResultFilePath()); + ReportObjectFactory reportObjectFactory = new ReportObjectFactory(target); + reportObjectFactory.createAndSaveReport(result); + logger.info("Successfully written the result: {}", target.getPath()); + logger.info("View the result using --mode or at: https://jplag.github.io/JPlag/"); + OutputFileGenerator.generateCsvOutput(result, new File(getResultFileBaseName()), this.options); + return target; + } + + public void runViewer(File zipFile) throws IOException { + ReportViewer reportViewer = new ReportViewer(zipFile, this.options.advanced.port); + int port = reportViewer.start(); + logger.info("ReportViewer started on port http://localhost:{}", port); + Desktop.getDesktop().browse(URI.create("http://localhost:" + port + "/")); + + System.out.println("Press Enter key to exit..."); + System.in.read(); + reportViewer.stop(); + } + private List buildSubcommands() { return LanguageLoader.getAllAvailableLanguages().values().stream().map(language -> { CommandSpec command = CommandSpec.create().name(language.getIdentifier()); @@ -143,7 +179,7 @@ public ParseResult parseOptions(String... args) throws CliException { } return result; } catch (CommandLine.ParameterException e) { - if (e.getArgSpec().isOption() && Arrays.asList(((OptionSpec) e.getArgSpec()).names()).contains("-l")) { + if (e.getArgSpec() != null && e.getArgSpec().isOption() && Arrays.asList(((OptionSpec) e.getArgSpec()).names()).contains("-l")) { throw new CliException(String.format(UNKOWN_LANGAUGE_EXCEPTION, e.getValue(), String.join(", ", LanguageLoader.getAllAvailableLanguageIdentifiers()))); } @@ -181,33 +217,31 @@ public JPlagOptions buildOptionsFromArguments(ParseResult parseResult) throws Cl JPlagOptions jPlagOptions = new JPlagOptions(loadLanguage(parseResult), this.options.minTokenMatch, submissionDirectories, oldSubmissionDirectories, null, this.options.advanced.subdirectory, suffixes, this.options.advanced.exclusionFileName, JPlagOptions.DEFAULT_SIMILARITY_METRIC, this.options.advanced.similarityThreshold, this.options.shownComparisons, clusteringOptions, - this.options.advanced.debug, mergingOptions); + this.options.advanced.debug, mergingOptions, this.options.normalize); String baseCodePath = this.options.baseCode; File baseCodeDirectory = baseCodePath == null ? null : new File(baseCodePath); if (baseCodeDirectory == null || baseCodeDirectory.exists()) { return jPlagOptions.withBaseCodeSubmissionDirectory(baseCodeDirectory); - } else { - logger.warn("Using legacy partial base code API. Please migrate to new full path base code API."); - return jPlagOptions.withBaseCodeSubmissionName(baseCodePath); } + logger.warn("Using legacy partial base code API. Please migrate to new full path base code API."); + return jPlagOptions.withBaseCodeSubmissionName(baseCodePath); } private Language loadLanguage(ParseResult result) throws CliException { - if (result.subcommand() != null) { - ParseResult subcommandResult = result.subcommand(); - Language language = LanguageLoader.getLanguage(subcommandResult.commandSpec().name()) - .orElseThrow(() -> new CliException(IMPOSSIBLE_EXCEPTION)); - LanguageOptions languageOptions = language.getOptions(); - languageOptions.getOptionsAsList().forEach(option -> { - if (subcommandResult.hasMatchedOption(option.getNameAsUnixParameter())) { - option.setValue(subcommandResult.matchedOptionValue(option.getNameAsUnixParameter(), null)); - } - }); - return language; - } else { + if (result.subcommand() == null) { return this.options.language; } + ParseResult subcommandResult = result.subcommand(); + Language language = LanguageLoader.getLanguage(subcommandResult.commandSpec().name()) + .orElseThrow(() -> new CliException(IMPOSSIBLE_EXCEPTION)); + LanguageOptions languageOptions = language.getOptions(); + languageOptions.getOptionsAsList().forEach(option -> { + if (subcommandResult.hasMatchedOption(option.getNameAsUnixParameter())) { + option.setValue(subcommandResult.matchedOptionValue(option.getNameAsUnixParameter(), null)); + } + }); + return language; } private static ClusteringOptions getClusteringOptions(CliOptions options) { @@ -249,7 +283,16 @@ private String generateDescription() { return String.format(DESCRIPTION_PATTERN, randomDescription, CREDITS); } - public String getResultFolder() { - return this.options.resultFolder; + private String getResultFilePath() { + String optionValue = this.options.resultFile; + if (optionValue.endsWith(DEFAULT_FILE_ENDING)) { + return optionValue; + } + return optionValue + DEFAULT_FILE_ENDING; + } + + private String getResultFileBaseName() { + String defaultOutputFile = getResultFilePath(); + return defaultOutputFile.substring(0, defaultOutputFile.length() - DEFAULT_FILE_ENDING.length()); } } diff --git a/cli/src/main/java/de/jplag/cli/CliOptions.java b/cli/src/main/java/de/jplag/cli/CliOptions.java index da249342c..384a2d41c 100644 --- a/cli/src/main/java/de/jplag/cli/CliOptions.java +++ b/cli/src/main/java/de/jplag/cli/CliOptions.java @@ -7,6 +7,7 @@ import de.jplag.clustering.ClusteringOptions; import de.jplag.clustering.algorithm.InterClusterSimilarity; import de.jplag.java.JavaLanguage; +import de.jplag.merging.MergingOptions; import de.jplag.options.JPlagOptions; import de.jplag.options.SimilarityMetric; @@ -19,47 +20,51 @@ public class CliOptions implements Runnable { public static final Language defaultLanguage = new JavaLanguage(); - @Parameters(paramLabel = "root-dirs", description = "Root-directory with submissions to check for plagiarism%n", split = ",") + @Parameters(paramLabel = "root-dirs", description = "Root-directory with submissions to check for plagiarism.", split = ",") public File[] rootDirectory = new File[0]; - @Option(names = {"--new", - "-new"}, split = ",", description = "Root-directory with submissions to check for plagiarism (same as the root directory)%n") + @Option(names = {"--new", "-new"}, split = ",", description = "Root-directories with submissions to check for plagiarism (same as root).") public File[] newDirectories = new File[0]; - @Option(names = {"--old", "-old"}, split = ",", description = "Root-directory with prior submissions to compare against%n") + @Option(names = {"--old", "-old"}, split = ",", description = "Root-directories with prior submissions to compare against.") public File[] oldDirectories = new File[0]; @Option(names = {"--language", - "-l"}, arity = "1", converter = LanguageConverter.class, completionCandidates = LanguageCandidates.class, description = "Select the language to parse the submissions (default: ${DEFAULT-VALUE}). The language names are the same as the subcommands.%n") + "-l"}, arity = "1", converter = LanguageConverter.class, completionCandidates = LanguageCandidates.class, description = "Select the language of the submissions (default: ${DEFAULT-VALUE}). See subcommands below.") public Language language = defaultLanguage; - @Option(names = {"-bc", "--bc", - "--base-code"}, description = "Path of the directory containing the base code (common framework used in all submissions)%n") + @Option(names = {"-bc", "--bc", "--base-code"}, description = "Path to the base code directory (common framework used in all submissions).") public String baseCode; - @Option(names = {"-t", "--min-tokens"}, description = "Tunes the comparison sensitivity by adjusting the minimum token required to be counted " - + "as a matching section. A smaller increases the sensitivity but might lead to more " + "false-positives%n") + @Option(names = {"-t", + "--min-tokens"}, description = "Tunes the comparison sensitivity by adjusting the minimum token required to be counted as a matching section. A smaller value increases the sensitivity but might lead to more false-positives.") public Integer minTokenMatch = null; - @Option(names = {"-h", "--help"}, usageHelp = true, description = "display this help and exit") + @Option(names = {"-h", "--help"}, usageHelp = true, description = "Display this help text", hidden = true) public boolean help; @Option(names = {"-n", - "--shown-comparisons"}, description = "The maximum number of comparisons that will be shown in the generated report, if set " - + "to -1 all comparisons will be shown (default: ${DEFAULT-VALUE})%n") + "--shown-comparisons"}, description = "The maximum number of comparisons that will be shown in the generated report, if set to -1 all comparisons will be shown (default: ${DEFAULT-VALUE})") public int shownComparisons = JPlagOptions.DEFAULT_SHOWN_COMPARISONS; @Option(names = {"-r", - "--result-directory"}, description = "Name of the directory in which the comparison results will be stored (default: ${DEFAULT-VALUE})%n") - public String resultFolder = "results"; + "--result-file"}, description = "Name of the file in which the comparison results will be stored (default: ${DEFAULT-VALUE}). Missing .zip endings will be automatically added.") + public String resultFile = "results"; - @ArgGroup(heading = "Advanced%n", exclusive = false) + @Option(names = {"-M", + "--mode"}, description = "The mode of JPlag: either only run analysis, only open the viewer, or do both (default: ${DEFAULT_VALUE})") + public JPlagMode mode = JPlagMode.RUN; + + @Option(names = {"--normalize"}, description = "Activate the normalization of tokens. Supported for languages: Java, C++.") + public boolean normalize = false; + + @ArgGroup(heading = "%nAdvanced%n", exclusive = false) public Advanced advanced = new Advanced(); - @ArgGroup(validate = false, heading = "Clustering%n") + @ArgGroup(validate = false, heading = "%nClustering%n") public Clustering clustering = new Clustering(); - @ArgGroup(validate = false, heading = "Merging of neighboring matches to increase the similarity of concealed plagiarism:%n") + @ArgGroup(validate = false, heading = "%nSubsequence Match Merging%n") public Merging merging = new Merging(); /** @@ -71,59 +76,58 @@ public void run() { } public static class Advanced { - @Option(names = {"-d", "--debug"}, description = "Debug parser. Non-parsable files will be stored (default: ${DEFAULT-VALUE})%n") + @Option(names = {"-d", "--debug"}, description = "Store on-parsable files in error folder.") public boolean debug; - @Option(names = {"-s", "--subdirectory"}, description = "Look in directories /*/ for programs%n") + @Option(names = {"-s", "--subdirectory"}, description = "Look in directories /*/ for programs.") public String subdirectory; - @Option(names = {"-p", "--suffixes"}, split = ",", description = "comma-separated list of all filename suffixes that are included%n") + @Option(names = {"-p", "--suffixes"}, split = ",", description = "comma-separated list of all filename suffixes that are included.") public String[] suffixes = new String[0]; @Option(names = {"-x", - "--exclusion-file"}, description = "All files named in this file will be ignored in the comparison (line-separated list)%n") + "--exclusion-file"}, description = "All files named in this file will be ignored in the comparison (line-separated list).") public String exclusionFileName; @Option(names = {"-m", - "--similarity-threshold"}, description = "Comparison similarity threshold [0.0-1.0]: All comparisons above this threshold will " - + "be saved (default: ${DEFAULT-VALUE})%n") + "--similarity-threshold"}, description = "Comparison similarity threshold [0.0-1.0]: All comparisons above this threshold will " + + "be saved (default: ${DEFAULT-VALUE}).") public double similarityThreshold = JPlagOptions.DEFAULT_SIMILARITY_THRESHOLD; - @Option(names = "--csv-export", description = "If present, a csv export will be generated in addition to the zip file.") + @Option(names = {"-P", "--port"}, description = "The port used for the internal report viewer (default: ${DEFAULT-VALUE}).") + public int port = 1996; + + @Option(names = "--csv-export", description = "Export pairwise similarity values as a CSV file.") public boolean csvExport = false; } public static class Clustering { - @Option(names = {"--cluster-skip"}, description = "Skips the clustering (default: ${DEFAULT-VALUE})%n") + @Option(names = {"--cluster-skip"}, description = "Skips the cluster calculation.") public boolean disable; @ArgGroup public ClusteringEnabled enabled = new ClusteringEnabled(); public static class ClusteringEnabled { - @Option(names = {"--cluster-alg", - "--cluster-algorithm"}, description = "Which clustering algorithm to use. Agglomerative merges similar submissions bottom up. " - + "Spectral clustering is combined with Bayesian Optimization to execute the k-Means " - + "clustering algorithm multiple times, hopefully finding a \"good\" clustering " - + "automatically. (default: ${DEFAULT-VALUE})%n") + @Option(names = {"--cluster-alg", "--cluster-algorithm"}, description = "Specifies the clustering algorithm (default: ${DEFAULT-VALUE}).") public ClusteringAlgorithm algorithm = new ClusteringOptions().algorithm(); - @Option(names = { - "--cluster-metric"}, description = "The metric used for clustering. AVG is intersection over union, MAX can expose some " - + "attempts of obfuscation. (default: ${DEFAULT-VALUE})%n") + @Option(names = {"--cluster-metric"}, description = "The similarity metric used for clustering (default: ${DEFAULT-VALUE}).") public SimilarityMetric metric = new ClusteringOptions().similarityMetric(); } } public static class Merging { - @Option(names = {"--match-merging"}, description = "Enables match merging (default: false)%n") - public boolean enabled; + @Option(names = {"--match-merging"}, description = "Enables merging of neighboring matches to counteract obfuscation attempts.") + public boolean enabled = MergingOptions.DEFAULT_ENABLED; - @Option(names = {"--neighbor-length"}, description = "Defines how short a match can be, to be considered (default: 2)%n") - public int minimumNeighborLength; + @Option(names = { + "--neighbor-length"}, description = "Minimal length of neighboring matches to be merged (between 1 and minTokenMatch, default: ${DEFAULT-VALUE}).%n") + public int minimumNeighborLength = MergingOptions.DEFAULT_NEIGHBOR_LENGTH; - @Option(names = {"--gap-size"}, description = "Defines how many token there can be between two neighboring matches (default: 6)%n") - public int maximumGapSize; + @Option(names = { + "--gap-size"}, description = "Maximal gap between neighboring matches to be merged (between 1 and minTokenMatch, default: ${DEFAULT-VALUE}).") + public int maximumGapSize = MergingOptions.DEFAULT_GAP_SIZE; } diff --git a/cli/src/main/java/de/jplag/cli/JPlagMode.java b/cli/src/main/java/de/jplag/cli/JPlagMode.java new file mode 100644 index 000000000..402a18b58 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/JPlagMode.java @@ -0,0 +1,19 @@ +package de.jplag.cli; + +/** + * The mode JPlag runs in. This influences which steps JPlag will execute. + */ +public enum JPlagMode { + /** + * Only run JPlag and create a results.zip + */ + RUN, + /** + * Only start the report viewer + */ + VIEW, + /** + * Run JPlag and open the result in report viewer + */ + RUN_AND_VIEW +} diff --git a/cli/src/main/java/de/jplag/cli/LanguageLoader.java b/cli/src/main/java/de/jplag/cli/LanguageLoader.java index 77c2de673..2ee1c815b 100644 --- a/cli/src/main/java/de/jplag/cli/LanguageLoader.java +++ b/cli/src/main/java/de/jplag/cli/LanguageLoader.java @@ -32,8 +32,9 @@ private LanguageLoader() { * @return the languages as unmodifiable map from identifier to language instance. */ public static synchronized Map getAllAvailableLanguages() { - if (cachedLanguageInstances != null) + if (cachedLanguageInstances != null) { return cachedLanguageInstances; + } Map languages = new TreeMap<>(); @@ -44,10 +45,10 @@ public static synchronized Map getAllAvailableLanguages() { languages.remove(languageIdentifier); continue; } - logger.debug("Loading Language Module '{}'", language.getName()); + logger.trace("Loading Language Module '{}'", language.getName()); languages.put(languageIdentifier, language); } - logger.info("Available languages: '{}'", languages.values().stream().map(Language::getName).toList()); + logger.debug("Available languages: '{}'", languages.values().stream().map(Language::getName).toList()); cachedLanguageInstances = Collections.unmodifiableMap(languages); return cachedLanguageInstances; @@ -61,8 +62,9 @@ public static synchronized Map getAllAvailableLanguages() { */ public static Optional getLanguage(String identifier) { var language = getAllAvailableLanguages().get(identifier); - if (language == null) + if (language == null) { logger.warn("Attempt to load Language {} was not successful", identifier); + } return Optional.ofNullable(language); } diff --git a/cli/src/main/java/de/jplag/cli/logger/CollectedLogger.java b/cli/src/main/java/de/jplag/cli/logger/CollectedLogger.java index 80d94a74a..3be42c8cd 100644 --- a/cli/src/main/java/de/jplag/cli/logger/CollectedLogger.java +++ b/cli/src/main/java/de/jplag/cli/logger/CollectedLogger.java @@ -77,8 +77,9 @@ private void log(int level, String message, Throwable throwable, Date timeOfErro builder.append('[').append(renderLevel(level)).append(']').append(' '); // Append the name of the log instance - if (shortLogName == null) + if (shortLogName == null) { shortLogName = computeShortName(); + } builder.append(shortLogName).append(" - "); // Append the message builder.append(message); @@ -90,8 +91,9 @@ void printAllErrorsForLogger() { this.isFinalizing = true; // Copy errors to prevent infinite recursion var errors = new ArrayList<>(this.allErrors); - if (errors.isEmpty()) + if (errors.isEmpty()) { return; + } this.allErrors.removeAll(errors); diff --git a/cli/src/main/java/de/jplag/cli/logger/CollectedLoggerFactory.java b/cli/src/main/java/de/jplag/cli/logger/CollectedLoggerFactory.java index ffba004b9..e356763e3 100644 --- a/cli/src/main/java/de/jplag/cli/logger/CollectedLoggerFactory.java +++ b/cli/src/main/java/de/jplag/cli/logger/CollectedLoggerFactory.java @@ -28,11 +28,10 @@ public Logger getLogger(String name) { CollectedLogger simpleLogger = loggerMap.get(name); if (simpleLogger != null) { return simpleLogger; - } else { - CollectedLogger newInstance = new CollectedLogger(name); - Logger oldInstance = loggerMap.putIfAbsent(name, newInstance); - return oldInstance == null ? newInstance : oldInstance; } + CollectedLogger newInstance = new CollectedLogger(name); + Logger oldInstance = loggerMap.putIfAbsent(name, newInstance); + return oldInstance == null ? newInstance : oldInstance; } /** diff --git a/cli/src/main/java/de/jplag/cli/logger/TongfeiProgressBar.java b/cli/src/main/java/de/jplag/cli/logger/TongfeiProgressBar.java new file mode 100644 index 000000000..4305a497e --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/logger/TongfeiProgressBar.java @@ -0,0 +1,24 @@ +package de.jplag.cli.logger; + +import de.jplag.logging.ProgressBar; + +/** + * A ProgressBar, that used the tongfei progress bar library underneath, to show progress bars on the cli. + */ +public class TongfeiProgressBar implements ProgressBar { + private final me.tongfei.progressbar.ProgressBar progressBar; + + public TongfeiProgressBar(me.tongfei.progressbar.ProgressBar progressBar) { + this.progressBar = progressBar; + } + + @Override + public void step(int number) { + this.progressBar.stepBy(number); + } + + @Override + public void dispose() { + this.progressBar.close(); + } +} diff --git a/cli/src/main/java/de/jplag/cli/logger/TongfeiProgressBarProvider.java b/cli/src/main/java/de/jplag/cli/logger/TongfeiProgressBarProvider.java new file mode 100644 index 000000000..da09fff33 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/logger/TongfeiProgressBarProvider.java @@ -0,0 +1,20 @@ +package de.jplag.cli.logger; + +import de.jplag.logging.ProgressBar; +import de.jplag.logging.ProgressBarProvider; +import de.jplag.logging.ProgressBarType; + +import me.tongfei.progressbar.ProgressBarBuilder; +import me.tongfei.progressbar.ProgressBarStyle; + +/** + * A ProgressBar provider, that used the tongfei progress bar library underneath, to show progress bars on the cli. + */ +public class TongfeiProgressBarProvider implements ProgressBarProvider { + @Override + public ProgressBar initProgressBar(ProgressBarType type, int totalSteps) { + me.tongfei.progressbar.ProgressBar progressBar = new ProgressBarBuilder().setTaskName(type.getDefaultText()).setInitialMax(totalSteps) + .setStyle(ProgressBarStyle.ASCII).build(); + return new TongfeiProgressBar(progressBar); + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/ContentType.java b/cli/src/main/java/de/jplag/cli/server/ContentType.java new file mode 100644 index 000000000..cf90673b3 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/ContentType.java @@ -0,0 +1,41 @@ +package de.jplag.cli.server; + +/** + * Data types used by JPlag in the context of http. Contains the according mime type. + */ +public enum ContentType { + HTML("text/html; charset=utf-8", ".html"), + JS("application/javascript; charset=utf-8", ".js"), + CSS("text/css; charset=utf-8", ".css"), + PNG("image/png", ".png"), + PLAIN("text/plain; charset=utf-8", null), + ZIP("application/zip", ".zip"); + + private final String value; + + private final String nameSuffix; + + ContentType(String value, String nameSuffix) { + this.value = value; + this.nameSuffix = nameSuffix; + } + + public String getValue() { + return value; + } + + /** + * Guesses the type from the given path using the suffix after the last '.'. + * @param path The path to guess from + * @return The guessed type + */ + public static ContentType fromPath(String path) { + String suffix = path.substring(path.lastIndexOf('.')); + for (ContentType value : ContentType.values()) { + if (suffix.equals(value.nameSuffix)) { + return value; + } + } + return ContentType.PLAIN; + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/HttpRequestMethod.java b/cli/src/main/java/de/jplag/cli/server/HttpRequestMethod.java new file mode 100644 index 000000000..acbf4a5b9 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/HttpRequestMethod.java @@ -0,0 +1,32 @@ +package de.jplag.cli.server; + +/** + * Wraps the http request methods used by JPlag. Request methods determine the capabilities of a http request. + */ +public enum HttpRequestMethod { + GET("GET"), + POST("POST"); + + private final String name; + + /** + * @param name The name of the request method + */ + HttpRequestMethod(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public static HttpRequestMethod fromName(String name) { + for (HttpRequestMethod value : HttpRequestMethod.values()) { + if (value.name.equals(name)) { + return value; + } + } + + return null; + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/ReportViewer.java b/cli/src/main/java/de/jplag/cli/server/ReportViewer.java new file mode 100644 index 000000000..6e861c926 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/ReportViewer.java @@ -0,0 +1,133 @@ +package de.jplag.cli.server; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.BindException; +import java.net.InetAddress; +import java.net.InetSocketAddress; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +/** + * Manages the internal report viewer. Serves the static files for the report viewer and the results.zip. + */ +public class ReportViewer implements HttpHandler { + private static final String REPORT_VIEWER_RESOURCE_PREFIX = "report-viewer"; + private static final String INDEX_PATH = "index.html"; + private static final String RESULT_PATH = "results.zip"; + + private static final Logger logger = LoggerFactory.getLogger(ReportViewer.class); + private static final int SUCCESS_RESPONSE = 200; + private static final int NOT_FOUND_RESPONSE = 404; + private static final int MAX_PORT_LOOKUPS = 4; + + private final RoutingTree routingTree; + private final int port; + + private HttpServer server; + + /** + * @param zipFile The zip file to use for the report viewer + * @param port The port to use for the server. You can use 0 to use any free port. + * @throws IOException If the zip file cannot be read + */ + public ReportViewer(File zipFile, int port) throws IOException { + this.routingTree = new RoutingTree(); + + this.routingTree.insertRouting("", new RoutingResources(REPORT_VIEWER_RESOURCE_PREFIX).or(new RoutingAlias(INDEX_PATH))); + this.routingTree.insertRouting(RESULT_PATH, new RoutingStaticFile(zipFile, ContentType.ZIP)); + this.port = port; + } + + /** + * Starts the server and serves the internal report viewer. If available, the result.zip is also exposed. If the given + * port is already in use, the next free port will be used. + * @return The port the server runs at + * @throws IOException If the server cannot be started + */ + public int start() throws IOException { + if (server != null) { + throw new IllegalStateException("Server already started"); + } + + int currentPort = this.port; + int remainingLookups = MAX_PORT_LOOKUPS; + BindException lastException = new BindException("Could not create server. Probably due to no free port found."); + while (server == null && remainingLookups-- > 0) { + try { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), currentPort), 0); + } catch (BindException e) { + logger.info("Port {} is not available. Trying to find a different one.", currentPort); + lastException = e; + currentPort++; + } + } + if (server == null) { + throw lastException; + } + server.createContext("/", this); + server.setExecutor(null); + server.start(); + + return server.getAddress().getPort(); + } + + /** + * Stops the server + */ + public void stop() { + server.stop(0); + } + + /** + * Do not call manually. Called by the running web server. + * @param exchange The http reqest + * @throws IOException If the IO handling goes wrong + */ + @Override + public void handle(HttpExchange exchange) throws IOException { + RoutingPath path = new RoutingPath(exchange.getRequestURI().getPath()); + Pair resolved = this.routingTree.resolveRouting(path); + HttpRequestMethod method = HttpRequestMethod.fromName(exchange.getRequestMethod()); + + if (resolved == null || !ArrayUtils.contains(resolved.getRight().allowedMethods(), method)) { + exchange.sendResponseHeaders(NOT_FOUND_RESPONSE, 0); + exchange.close(); + return; + } + + logger.debug("Serving {}", path); + + ResponseData responseData = resolved.getRight().fetchData(resolved.getLeft(), exchange, this); + if (responseData == null) { + logger.warn("No response data found for path: {}", path.asPath()); + exchange.sendResponseHeaders(NOT_FOUND_RESPONSE, 0); + exchange.close(); + return; + } + + InputStream inputStream = responseData.stream(); + + if (responseData.contentType() != null) { + exchange.getResponseHeaders().set("Content-Type", responseData.contentType().getValue()); + } + exchange.sendResponseHeaders(SUCCESS_RESPONSE, responseData.size()); + + inputStream.transferTo(exchange.getResponseBody()); + exchange.getResponseBody().flush(); + exchange.getResponseBody().close(); + inputStream.close(); + } + + RoutingTree getRoutingTree() { + return routingTree; + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/ResponseData.java b/cli/src/main/java/de/jplag/cli/server/ResponseData.java new file mode 100644 index 000000000..26e9d037d --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/ResponseData.java @@ -0,0 +1,46 @@ +package de.jplag.cli.server; + +import java.io.InputStream; + +/** + * Data for a http response + * @param stream The stream containing the binary data + * @param contentType The type of data + * @param size The total size of the data + */ +public record ResponseData(InputStream stream, ContentType contentType, int size) { + /** + * Constructor with unknown type and size. Type will be set to PLAIN. + * @param data The binary data to respond with + */ + public ResponseData(InputStream data) { + this(data, ContentType.PLAIN, 0); + } + + /** + * Constructor with unknown size + * @param data The binary data + * @param contentType The type of content + */ + public ResponseData(InputStream data, ContentType contentType) { + this(data, contentType, 0); + } + + /** + * Creates a new instance for a given resource url. + * @param url The resource url + * @return The new response data + */ + public static ResponseData fromResourceUrl(String url) { + if (url.endsWith("/")) { + return null; + } + + InputStream inputStream = ResponseData.class.getResourceAsStream(url); + + if (inputStream != null) { + return new ResponseData(inputStream, ContentType.fromPath(url)); + } + return null; + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/Routing.java b/cli/src/main/java/de/jplag/cli/server/Routing.java new file mode 100644 index 000000000..a6152a031 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/Routing.java @@ -0,0 +1,33 @@ +package de.jplag.cli.server; + +import com.sun.net.httpserver.HttpExchange; + +/** + * Handles the data for a url prefix. + */ +public interface Routing { + /** + * @return The methods, that this routing can be used for. + */ + default HttpRequestMethod[] allowedMethods() { + return new HttpRequestMethod[] {HttpRequestMethod.GET}; + } + + /** + * Gets the data for the given url + * @param subPath The remaining suffix of the url, that is not jet interpreted + * @param request The original http request + * @param viewer The current report viewer + * @return The data to respond with + */ + ResponseData fetchData(RoutingPath subPath, HttpExchange request, ReportViewer viewer); + + /** + * Use the other routing if this routing does not find any data. + * @param other The other routing + * @return The combined routing + */ + default Routing or(Routing other) { + return new RoutingFallback(this, other); + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/RoutingAlias.java b/cli/src/main/java/de/jplag/cli/server/RoutingAlias.java new file mode 100644 index 000000000..70c4aefe1 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/RoutingAlias.java @@ -0,0 +1,36 @@ +package de.jplag.cli.server; + +import org.apache.commons.lang3.tuple.Pair; + +import com.sun.net.httpserver.HttpExchange; + +/** + * An alias routing, that will respond with the response for a different path + */ +public class RoutingAlias implements Routing { + private final RoutingPath path; + + /** + * @param path The path to actually use + */ + public RoutingAlias(RoutingPath path) { + this.path = path; + } + + /** + * @param path The path to actually use + */ + public RoutingAlias(String path) { + this(new RoutingPath(path)); + } + + @Override + public ResponseData fetchData(RoutingPath subPath, HttpExchange request, ReportViewer viewer) { + Pair redirect = viewer.getRoutingTree().resolveRouting(path); + if (redirect == null) { + return null; + } + + return redirect.getValue().fetchData(redirect.getLeft(), request, viewer); + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/RoutingFallback.java b/cli/src/main/java/de/jplag/cli/server/RoutingFallback.java new file mode 100644 index 000000000..13de6d6d2 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/RoutingFallback.java @@ -0,0 +1,30 @@ +package de.jplag.cli.server; + +import com.sun.net.httpserver.HttpExchange; + +/** + * Responds with the first given routing, unless that would respond with null, in that case the second one is used. + */ +public class RoutingFallback implements Routing { + private final Routing first; + private final Routing second; + + /** + * @param first The first routing + * @param second The second routing + */ + public RoutingFallback(Routing first, Routing second) { + this.first = first; + this.second = second; + } + + @Override + public ResponseData fetchData(RoutingPath subPath, HttpExchange request, ReportViewer viewer) { + ResponseData attempt = this.first.fetchData(subPath, request, viewer); + if (attempt != null) { + return attempt; + } + + return this.second.fetchData(subPath, request, viewer); + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/RoutingPath.java b/cli/src/main/java/de/jplag/cli/server/RoutingPath.java new file mode 100644 index 000000000..e9263c8e8 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/RoutingPath.java @@ -0,0 +1,72 @@ +package de.jplag.cli.server; + +import java.util.Arrays; + +/** + * A path used for routing. Can be used like a linked list. + */ +public class RoutingPath { + private final String[] components; + private final int offset; + + /** + * @param path The full path + */ + public RoutingPath(String path) { + this.components = Arrays.stream(path.split("/", 0)).filter(it -> !it.isBlank()).toArray(String[]::new); + this.offset = 0; + } + + private RoutingPath(String[] components, int offset) { + this.components = components; + this.offset = offset; + } + + /** + * @return The first path segment + */ + public String head() { + return components[offset]; + } + + /** + * @return All path segments except the first + */ + public RoutingPath tail() { + if (!hasTail()) { + throw new IllegalStateException("Routing path is done."); + } + + return new RoutingPath(this.components, this.offset + 1); + } + + /** + * @return True, if the tail has at least 0 elements + */ + public boolean hasTail() { + return this.components.length > this.offset; + } + + /** + * @return True, if there are no segments in this path + */ + public boolean isEmpty() { + return this.offset == this.components.length; + } + + /** + * @return The remaining path as a string + */ + public String asPath() { + StringBuilder builder = new StringBuilder(); + + for (int i = this.offset; i < this.components.length; i++) { + if (i > this.offset) { + builder.append("/"); + } + builder.append(this.components[i]); + } + + return builder.toString(); + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/RoutingResources.java b/cli/src/main/java/de/jplag/cli/server/RoutingResources.java new file mode 100644 index 000000000..d46a8fe43 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/RoutingResources.java @@ -0,0 +1,31 @@ +package de.jplag.cli.server; + +import com.sun.net.httpserver.HttpExchange; + +/** + * Responds with data from the resources + */ +public class RoutingResources implements Routing { + private String prefix; + + /** + * @param prefix The prefix to use within the resources + */ + public RoutingResources(String prefix) { + this.prefix = prefix; + + if (!this.prefix.startsWith("/")) { + this.prefix = "/" + this.prefix; + } + + if (!this.prefix.endsWith("/")) { + this.prefix = this.prefix + "/"; + } + } + + @Override + public ResponseData fetchData(RoutingPath subPath, HttpExchange request, ReportViewer viewer) { + String fullPath = this.prefix + subPath.asPath(); + return ResponseData.fromResourceUrl(fullPath); + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/RoutingStaticFile.java b/cli/src/main/java/de/jplag/cli/server/RoutingStaticFile.java new file mode 100644 index 000000000..3ce5bed1a --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/RoutingStaticFile.java @@ -0,0 +1,42 @@ +package de.jplag.cli.server; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; + +import com.sun.net.httpserver.HttpExchange; + +/** + * Responds with a given file + */ +public class RoutingStaticFile implements Routing { + private final byte[] data; + private final ContentType contentType; + + /** + * @param file The file to use + * @param contentType The type of content in the file + * @throws IOException If the file cannot be read + */ + public RoutingStaticFile(File file, ContentType contentType) throws IOException { + if (file != null) { + try (FileInputStream inputStream = new FileInputStream(file)) { + this.data = inputStream.readAllBytes(); + + this.contentType = contentType; + } + } else { + this.data = null; + this.contentType = contentType; + } + } + + @Override + public ResponseData fetchData(RoutingPath subPath, HttpExchange request, ReportViewer viewer) { + if (this.data != null) { + return new ResponseData(new ByteArrayInputStream(this.data), contentType, this.data.length); + } + return null; + } +} diff --git a/cli/src/main/java/de/jplag/cli/server/RoutingTree.java b/cli/src/main/java/de/jplag/cli/server/RoutingTree.java new file mode 100644 index 000000000..83bc4dff3 --- /dev/null +++ b/cli/src/main/java/de/jplag/cli/server/RoutingTree.java @@ -0,0 +1,87 @@ +package de.jplag.cli.server; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang3.tuple.Pair; + +/** + * Manages the tree of paths handled by the web server + */ +public class RoutingTree { + private final RoutingTreeNode root; + + /** + * Creates an empty tree + */ + public RoutingTree() { + this.root = new RoutingTreeNode(); + } + + /** + * Adds a new routing to the tree + * @param path The path to use the routing for + * @param routing The routing + */ + public void insertRouting(RoutingPath path, Routing routing) { + this.root.buildRouting(path, routing); + } + + /** + * Adds a new routing to the tree + * @param path The path to use the routing for + * @param routing The routing + */ + public void insertRouting(String path, Routing routing) { + this.insertRouting(new RoutingPath(path), routing); + } + + /** + * Gets the routing for a given path + * @param path The path to look up + * @return The remaining path to be handled by the routing and the found routing + */ + public Pair resolveRouting(RoutingPath path) { + return this.root.resolve(path); + } + + private static class RoutingTreeNode { + private final Map children; + private Routing routing; + + public RoutingTreeNode(RoutingPath building, Routing routing) { + this(); + this.buildRouting(building, routing); + } + + public RoutingTreeNode() { + this.children = new HashMap<>(); + } + + public void buildRouting(RoutingPath building, Routing routing) { + if (building.isEmpty()) { + this.routing = routing; + } else if (this.children.containsKey(building.head())) { + this.children.get(building.head()).buildRouting(building.tail(), routing); + } else { + this.children.put(building.head(), new RoutingTreeNode(building.tail(), routing)); + } + } + + public Pair resolve(RoutingPath path) { + if ((path.isEmpty() || !this.children.containsKey(path.head())) && this.routing != null) { + return Pair.of(path, this.routing); + } + + if (this.children.containsKey(path.head()) && !path.isEmpty()) { + Pair childResolved = this.children.get(path.head()).resolve(path.tail()); + if (childResolved == null && this.routing != null) { + return Pair.of(path, this.routing); + } + return childResolved; + } + + return null; + } + } +} diff --git a/cli/src/main/resources/README.md b/cli/src/main/resources/README.md new file mode 100644 index 000000000..5355d847c --- /dev/null +++ b/cli/src/main/resources/README.md @@ -0,0 +1 @@ +Copy ReportViewer to a directory called `JPlag` \ No newline at end of file diff --git a/cli/src/test/java/de/jplag/cli/ClusteringTest.java b/cli/src/test/java/de/jplag/cli/ClusteringTest.java index bf591726e..75ee12124 100644 --- a/cli/src/test/java/de/jplag/cli/ClusteringTest.java +++ b/cli/src/test/java/de/jplag/cli/ClusteringTest.java @@ -1,6 +1,8 @@ package de.jplag.cli; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; diff --git a/cli/src/test/java/de/jplag/cli/MergingOptionsTest.java b/cli/src/test/java/de/jplag/cli/MergingOptionsTest.java new file mode 100644 index 000000000..bf2b642c7 --- /dev/null +++ b/cli/src/test/java/de/jplag/cli/MergingOptionsTest.java @@ -0,0 +1,25 @@ +package de.jplag.cli; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import de.jplag.merging.MergingOptions; + +/** + * Test cases for the options of the match merging mechanism. + */ +class MergingOptionsTest extends CommandLineInterfaceTest { + + @Test + @DisplayName("Test if default values are used when creating merging options from CLI") + void testMergingDefault() throws CliException { + buildOptionsFromCLI(defaultArguments()); + assertNotNull(options.mergingOptions()); + assertEquals(MergingOptions.DEFAULT_ENABLED, options.mergingOptions().enabled()); + assertEquals(MergingOptions.DEFAULT_NEIGHBOR_LENGTH, options.mergingOptions().minimumNeighborLength()); + assertEquals(MergingOptions.DEFAULT_GAP_SIZE, options.mergingOptions().maximumGapSize()); + } +} diff --git a/cli/src/test/java/de/jplag/cli/ReportViewerTest.java b/cli/src/test/java/de/jplag/cli/ReportViewerTest.java new file mode 100644 index 000000000..cc5f592ce --- /dev/null +++ b/cli/src/test/java/de/jplag/cli/ReportViewerTest.java @@ -0,0 +1,33 @@ +package de.jplag.cli; + +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.awt.Desktop; +import java.net.URI; +import java.util.concurrent.TimeUnit; + +import javax.swing.JOptionPane; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import de.jplag.cli.server.ReportViewer; + +@Timeout(value = 5, unit = TimeUnit.MINUTES) +class ReportViewerTest { + @Test + @Disabled("Starts the internal server for manual testing. Does not terminal automatically.") + void testStartViewer() throws Exception { + assumeTrue(Desktop.isDesktopSupported()); + ReportViewer viewer = new ReportViewer(null, 0); + + int port = viewer.start(); + Desktop.getDesktop().browse(URI.create("http://localhost:" + port)); + + // Open Dialog to keep the test running + JOptionPane.showMessageDialog(null, "Press OK to stop the server"); + viewer.stop(); + } + +} \ No newline at end of file diff --git a/cli/src/test/java/de/jplag/cli/server/RoutingFallbackTest.java b/cli/src/test/java/de/jplag/cli/server/RoutingFallbackTest.java new file mode 100644 index 000000000..6dc08d057 --- /dev/null +++ b/cli/src/test/java/de/jplag/cli/server/RoutingFallbackTest.java @@ -0,0 +1,44 @@ +package de.jplag.cli.server; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.File; +import java.io.IOException; + +import org.junit.jupiter.api.Test; + +class RoutingFallbackTest { + private final Routing nullRouting; + private final Routing contentRouting; + + RoutingFallbackTest() throws IOException { + File testFile = File.createTempFile("content", ".any"); + this.nullRouting = new RoutingStaticFile(null, ContentType.PLAIN); + this.contentRouting = new RoutingStaticFile(testFile, ContentType.PLAIN); + } + + @Test + void testSecondNull() { + Routing routing = this.nullRouting.or(this.contentRouting); + assertNotNull(routing.fetchData(null, null, null)); + } + + @Test + void testFirstNull() { + Routing routing = this.contentRouting.or(this.nullRouting); + assertNotNull(routing.fetchData(null, null, null)); + } + + @Test + void testNeitherNull() { + Routing routing = this.contentRouting.or(this.contentRouting); + assertNotNull(routing.fetchData(null, null, null)); + } + + @Test + void testBothNull() { + Routing routing = this.nullRouting.or(this.nullRouting); + assertNull(routing.fetchData(null, null, null)); + } +} \ No newline at end of file diff --git a/cli/src/test/java/de/jplag/cli/server/RoutingPathTest.java b/cli/src/test/java/de/jplag/cli/server/RoutingPathTest.java new file mode 100644 index 000000000..e4c2ebff0 --- /dev/null +++ b/cli/src/test/java/de/jplag/cli/server/RoutingPathTest.java @@ -0,0 +1,46 @@ +package de.jplag.cli.server; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class RoutingPathTest { + private static final String TEST_PATH = "some/path/to/index.html"; + private static final String TEST_PATH_WITH_BEGINNING_SLASH = "/some/path/to/index.html"; + private static final String TEST_PATH_WITH_ADDITIONAL_SLASHES = "///some/path////to/index.html"; + + private static final String[] TEST_PATH_PARTS = new String[] {"some", "path", "to", "index.html"}; + + @ParameterizedTest + @ValueSource(strings = {TEST_PATH_WITH_BEGINNING_SLASH, TEST_PATH, TEST_PATH_WITH_ADDITIONAL_SLASHES}) + void testAsPath(String path) { + RoutingPath routingPath = new RoutingPath(path); + assertEquals(TEST_PATH, routingPath.asPath()); + } + + @Test + void testIterating() { + RoutingPath routingPath = new RoutingPath(TEST_PATH); + for (String expectedPart : TEST_PATH_PARTS) { + String currentPart = routingPath.head(); + routingPath = routingPath.tail(); + assertEquals(expectedPart, currentPart); + } + + assertFalse(routingPath.hasTail()); + assertTrue(routingPath.isEmpty()); + } + + @Test + void testErrorWithEmptyTail() { + assertThrowsExactly(IllegalStateException.class, () -> { + RoutingPath routingPath = new RoutingPath(""); + routingPath.tail(); + }); + } +} \ No newline at end of file diff --git a/cli/src/test/java/de/jplag/cli/server/RoutingResourcesTest.java b/cli/src/test/java/de/jplag/cli/server/RoutingResourcesTest.java new file mode 100644 index 000000000..56df92140 --- /dev/null +++ b/cli/src/test/java/de/jplag/cli/server/RoutingResourcesTest.java @@ -0,0 +1,20 @@ +package de.jplag.cli.server; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +class RoutingResourcesTest { + private static final RoutingResources routing = new RoutingResources("/"); + + @Test + void testExistingFile() { + assertNotNull(routing.fetchData(new RoutingPath("testResource.txt"), null, null)); + } + + @Test + void testNotExistingFile() { + assertNull(routing.fetchData(new RoutingPath("otherFile.txt"), null, null)); + } +} \ No newline at end of file diff --git a/cli/src/test/java/de/jplag/cli/server/RoutingStaticFileTest.java b/cli/src/test/java/de/jplag/cli/server/RoutingStaticFileTest.java new file mode 100644 index 000000000..8e9549d00 --- /dev/null +++ b/cli/src/test/java/de/jplag/cli/server/RoutingStaticFileTest.java @@ -0,0 +1,43 @@ +package de.jplag.cli.server; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class RoutingStaticFileTest { + private static final String TEST_FILE_CONTENT = "some test content."; + private static final ContentType TEST_CONTENT_TYPE = ContentType.PLAIN; + private static RoutingStaticFile routing; + + @BeforeAll + static void setUp() throws IOException { + File testFile = File.createTempFile("testFile", ".txt"); + try (FileWriter writer = new FileWriter(testFile)) { + writer.write(TEST_FILE_CONTENT); + } + routing = new RoutingStaticFile(testFile, TEST_CONTENT_TYPE); + } + + @Test + void testRespondsWithFileContent() throws IOException { + ResponseData responseData = routing.fetchData(null, null, null); + assertEquals(TEST_CONTENT_TYPE, responseData.contentType()); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseData.stream()))) { + assertEquals(TEST_FILE_CONTENT, reader.readLine()); + } + } + + @Test + void testWithNullFile() throws IOException { + RoutingStaticFile nullRouting = new RoutingStaticFile(null, TEST_CONTENT_TYPE); + assertNull(nullRouting.fetchData(null, null, null)); + } +} \ No newline at end of file diff --git a/cli/src/test/java/de/jplag/cli/server/RoutingTreeTest.java b/cli/src/test/java/de/jplag/cli/server/RoutingTreeTest.java new file mode 100644 index 000000000..318c00db2 --- /dev/null +++ b/cli/src/test/java/de/jplag/cli/server/RoutingTreeTest.java @@ -0,0 +1,80 @@ +package de.jplag.cli.server; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.sun.net.httpserver.HttpExchange; + +class RoutingTreeTest { + private static final String firstRoutingPath = "/content/image.png"; + private static final String secondRoutingPath = "/index.html"; + private RoutingTree routingTree; + + @BeforeEach + void setUp() { + this.routingTree = new RoutingTree(); + this.routingTree.insertRouting(firstRoutingPath, new TestRouting(firstRoutingPath)); + this.routingTree.insertRouting(secondRoutingPath, new TestRouting(secondRoutingPath)); + } + + @Test + void testAccessRoutingTree() { + Pair firstRouting = this.routingTree.resolveRouting(new RoutingPath(firstRoutingPath)); + Pair secondRouting = this.routingTree.resolveRouting(new RoutingPath(secondRoutingPath + "/suffix")); + + assertTrue(firstRouting.getLeft().isEmpty()); + assertFalse(secondRouting.getLeft().isEmpty()); + assertEquals("suffix", secondRouting.getLeft().asPath()); + + assertInstanceOf(TestRouting.class, firstRouting.getRight()); + assertInstanceOf(TestRouting.class, secondRouting.getRight()); + + assertEquals(firstRoutingPath, ((TestRouting) firstRouting.getRight()).path); + assertEquals(secondRoutingPath, ((TestRouting) secondRouting.getRight()).path); + } + + @Test + void testUnknownPath() { + assertNull(this.routingTree.resolveRouting(new RoutingPath("/unknown.html"))); + } + + @Test + void testPartialPathRoute() { + RoutingTree routingTree = new RoutingTree(); + routingTree.insertRouting("/path/", new TestRouting("")); + assertNotNull(routingTree.resolveRouting(new RoutingPath("/path/index.html"))); + } + + @Test + void testPartialPathRouteWithSubpath() { + RoutingTree routingTree = new RoutingTree(); + routingTree.insertRouting("/path/", new TestRouting("/path/")); + routingTree.insertRouting("/path/subPath/a.html", new TestRouting("")); + + Pair result = routingTree.resolveRouting(new RoutingPath("/path/subPath/b.html")); + assertNotNull(result); + assertInstanceOf(TestRouting.class, result.getRight()); + assertEquals("/path/", ((TestRouting) result.getRight()).path); + } + + private static class TestRouting implements Routing { + private final String path; + + public TestRouting(String path) { + this.path = path; + } + + @Override + public ResponseData fetchData(RoutingPath subPath, HttpExchange request, ReportViewer viewer) { + return null; + } + } +} \ No newline at end of file diff --git a/cli/src/test/resources/testResource.txt b/cli/src/test/resources/testResource.txt new file mode 100644 index 000000000..c29d01b9b --- /dev/null +++ b/cli/src/test/resources/testResource.txt @@ -0,0 +1 @@ +Test resource file for RoutingResourcesTest \ No newline at end of file diff --git a/core/src/main/java/de/jplag/JPlag.java b/core/src/main/java/de/jplag/JPlag.java index 822dc959c..41f1c08c8 100644 --- a/core/src/main/java/de/jplag/JPlag.java +++ b/core/src/main/java/de/jplag/JPlag.java @@ -71,9 +71,13 @@ public static JPlagResult run(JPlagOptions options) throws ExitException { // Parse and validate submissions. SubmissionSetBuilder builder = new SubmissionSetBuilder(options); SubmissionSet submissionSet = builder.buildSubmissionSet(); + if (options.normalize() && options.language().supportsNormalization() && options.language().requiresCoreNormalization()) { + submissionSet.normalizeSubmissions(); + } int submissionCount = submissionSet.numberOfSubmissions(); - if (submissionCount < 2) + if (submissionCount < 2) { throw new SubmissionException("Not enough valid submissions! (found " + submissionCount + " valid submissions)"); + } // Compare valid submissions. JPlagResult result = comparisonStrategy.compareSubmissions(submissionSet); @@ -83,8 +87,9 @@ public static JPlagResult run(JPlagOptions options) throws ExitException { result = new MatchMerging(options).mergeMatchesOf(result); } - if (logger.isInfoEnabled()) + if (logger.isInfoEnabled()) { logger.info("Total time for comparing submissions: {}", TimeUtil.formatDuration(result.getDuration())); + } result.setClusteringResult(ClusteringFactory.getClusterings(result.getAllComparisons(), options.clusteringOptions())); logSkippedSubmissions(submissionSet, options); @@ -103,6 +108,10 @@ private static void logSkippedSubmissions(SubmissionSet submissionSet, JPlagOpti } private static void checkForConfigurationConsistency(JPlagOptions options) throws RootDirectoryException { + if (options.normalize() && !options.language().supportsNormalization()) { + logger.error(String.format("The language %s cannot be used with normalization.", options.language().getName())); + } + List duplicateNames = getDuplicateSubmissionFolderNames(options); if (duplicateNames.size() > 0) { throw new RootDirectoryException(String.format("Duplicate root directory names found: %s", String.join(", ", duplicateNames))); diff --git a/core/src/main/java/de/jplag/JPlagResult.java b/core/src/main/java/de/jplag/JPlagResult.java index 2b1aabd32..581079b87 100644 --- a/core/src/main/java/de/jplag/JPlagResult.java +++ b/core/src/main/java/de/jplag/JPlagResult.java @@ -1,5 +1,6 @@ package de.jplag; +import java.util.Comparator; import java.util.List; import java.util.function.ToDoubleFunction; @@ -27,7 +28,7 @@ public class JPlagResult { public JPlagResult(List comparisons, SubmissionSet submissions, long durationInMillis, JPlagOptions options) { // sort by similarity (descending) - this.comparisons = comparisons.stream().sorted((first, second) -> Double.compare(second.similarity(), first.similarity())).toList(); + this.comparisons = comparisons.stream().sorted(Comparator.comparing(JPlagComparison::similarity).reversed()).toList(); this.submissions = submissions; this.durationInMillis = durationInMillis; this.options = options; diff --git a/core/src/main/java/de/jplag/Match.java b/core/src/main/java/de/jplag/Match.java index 0350dff37..1e94315a7 100644 --- a/core/src/main/java/de/jplag/Match.java +++ b/core/src/main/java/de/jplag/Match.java @@ -22,9 +22,8 @@ public boolean overlaps(Match other) { if (startOfSecond < other.startOfSecond) { return (other.startOfSecond - startOfSecond) < length; - } else { - return (startOfSecond - other.startOfSecond) < other.length; } + return (startOfSecond - other.startOfSecond) < other.length; } /** diff --git a/core/src/main/java/de/jplag/NumberOfArgumentValues.java b/core/src/main/java/de/jplag/NumberOfArgumentValues.java deleted file mode 100644 index 8a4987c2a..000000000 --- a/core/src/main/java/de/jplag/NumberOfArgumentValues.java +++ /dev/null @@ -1,21 +0,0 @@ -package de.jplag; - -/** - * Allowed number of values of a command-line argument in the CLI. - */ -public enum NumberOfArgumentValues { - SINGLE_VALUE(""), - ONE_OR_MORE_VALUES("+"), - ZERO_OR_MORE_VALUES("*"); - - private final String representation; - - NumberOfArgumentValues(String representation) { - this.representation = representation; - } - - @Override - public String toString() { - return representation; - } -} \ No newline at end of file diff --git a/core/src/main/java/de/jplag/Submission.java b/core/src/main/java/de/jplag/Submission.java index 0fec4e0b3..0fe8db33f 100644 --- a/core/src/main/java/de/jplag/Submission.java +++ b/core/src/main/java/de/jplag/Submission.java @@ -7,8 +7,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; @@ -61,6 +63,8 @@ public class Submission implements Comparable { private final Language language; + private Map fileTokenCount; + /** * Creates a submission. * @param name Identification of the submission (directory or filename). @@ -235,9 +239,11 @@ private static File createErrorDirectory(String... subdirectoryNames) { /** * Parse files of the submission. + * @param debugParser specifies if the submission should be copied upon parsing errors. + * @param normalize specifies if the tokens sequences should be normalized. * @return Whether parsing was successful. */ - /* package-private */ boolean parse(boolean debugParser) { + /* package-private */ boolean parse(boolean debugParser, boolean normalize) { if (files == null || files.isEmpty()) { logger.error("ERROR: nothing to parse for submission \"{}\"", name); tokenList = null; @@ -246,14 +252,15 @@ private static File createErrorDirectory(String... subdirectoryNames) { } try { - tokenList = language.parse(new HashSet<>(files)); + tokenList = language.parse(new HashSet<>(files), normalize); if (logger.isDebugEnabled()) { for (Token token : tokenList) { logger.debug(String.join(" | ", token.getType().toString(), Integer.toString(token.getLine()), token.getSemantics().toString())); } } } catch (ParsingException e) { - logger.warn("Failed to parse submission {} with error {}", this, e.getMessage(), e); + String shortenedMessage = e.getMessage().replace(submissionRootFile.toString(), name); + logger.warn("Failed to parse submission {}:{}{}", name, System.lineSeparator(), shortenedMessage); tokenList = null; hasErrors = true; if (debugParser) { @@ -272,7 +279,7 @@ private static File createErrorDirectory(String... subdirectoryNames) { } /** - * Perform token string normalization, which makes the token string invariant to dead code insertion and independent + * Perform token sequence normalization, which makes the token sequence invariant to dead code insertion and independent * statement reordering. */ void normalize() { @@ -309,4 +316,24 @@ public Submission copy() { copy.setBaseCodeComparison(baseCodeComparison); return copy; } + + /** + * @return A mapping of each file in the submission to the number of tokens in the file + */ + public Map getTokenCountPerFile() { + if (this.tokenList == null) { + return Collections.emptyMap(); + } + + if (fileTokenCount == null) { + fileTokenCount = new HashMap<>(); + for (File file : this.files) { + fileTokenCount.put(file, 0); + } + for (Token token : this.tokenList) { + fileTokenCount.put(token.getFile(), fileTokenCount.get(token.getFile()) + 1); + } + } + return fileTokenCount; + } } diff --git a/core/src/main/java/de/jplag/SubmissionFileData.java b/core/src/main/java/de/jplag/SubmissionFileData.java new file mode 100644 index 000000000..91eb3edda --- /dev/null +++ b/core/src/main/java/de/jplag/SubmissionFileData.java @@ -0,0 +1,13 @@ +package de.jplag; + +import java.io.File; + +/** + * Contains the information about a single file in a submission. For single file submissions the submission file is the + * same as the root. + * @param submissionFile The file, that is part of a submission + * @param root The root of the submission + * @param isNew Indicates weather this follows the new or the old syntax + */ +public record SubmissionFileData(File submissionFile, File root, boolean isNew) { +} diff --git a/core/src/main/java/de/jplag/SubmissionSet.java b/core/src/main/java/de/jplag/SubmissionSet.java index 166a151fb..907687974 100644 --- a/core/src/main/java/de/jplag/SubmissionSet.java +++ b/core/src/main/java/de/jplag/SubmissionSet.java @@ -10,6 +10,9 @@ import de.jplag.exceptions.BasecodeException; import de.jplag.exceptions.ExitException; import de.jplag.exceptions.SubmissionException; +import de.jplag.logging.ProgressBar; +import de.jplag.logging.ProgressBarLogger; +import de.jplag.logging.ProgressBarType; import de.jplag.options.JPlagOptions; /** @@ -37,6 +40,7 @@ public class SubmissionSet { /** * @param submissions Submissions to check for plagiarism. * @param baseCode Base code submission if it exists or {@code null}. + * @param options The JPlag options */ public SubmissionSet(List submissions, Submission baseCode, JPlagOptions options) throws ExitException { this.allSubmissions = submissions; @@ -119,9 +123,10 @@ private void parseAllSubmissions() throws ExitException { private void parseBaseCodeSubmission(Submission baseCode) throws BasecodeException { long startTime = System.currentTimeMillis(); logger.trace("----- Parsing basecode submission: " + baseCode.getName()); - if (!baseCode.parse(options.debugParser())) { + if (!baseCode.parse(options.debugParser(), options.normalize())) { throw new BasecodeException("Could not successfully parse basecode submission!"); - } else if (baseCode.getNumberOfTokens() < options.minimumTokenMatch()) { + } + if (baseCode.getNumberOfTokens() < options.minimumTokenMatch()) { throw new BasecodeException(String.format("Basecode submission contains %d token(s), which is less than the minimum match length (%d)!", baseCode.getNumberOfTokens(), options.minimumTokenMatch())); } @@ -133,6 +138,7 @@ private void parseBaseCodeSubmission(Submission baseCode) throws BasecodeExcepti /** * Parse all given submissions. + * @param submissions The list of submissions */ private void parseSubmissions(List submissions) { if (submissions.isEmpty()) { @@ -143,14 +149,14 @@ private void parseSubmissions(List submissions) { long startTime = System.currentTimeMillis(); int tooShort = 0; + ProgressBar progressBar = ProgressBarLogger.createProgressBar(ProgressBarType.PARSING, submissions.size()); for (Submission submission : submissions) { - logger.info("Parsing submission {}", submission.getName()); boolean ok; logger.trace("------ Parsing submission: " + submission.getName()); currentSubmissionName = submission.getName(); - if (!(ok = submission.parse(options.debugParser()))) { + if (!(ok = submission.parse(options.debugParser(), options.normalize()))) { errors++; } @@ -168,7 +174,9 @@ private void parseSubmissions(List submissions) { } else { logger.error("ERROR -> Submission {} removed", currentSubmissionName); } + progressBar.step(); } + progressBar.dispose(); int validSubmissions = submissions.size() - errors - tooShort; logger.trace(validSubmissions + " submissions parsed successfully!"); diff --git a/core/src/main/java/de/jplag/SubmissionSetBuilder.java b/core/src/main/java/de/jplag/SubmissionSetBuilder.java index 4d93c0d44..e0cf584e4 100644 --- a/core/src/main/java/de/jplag/SubmissionSetBuilder.java +++ b/core/src/main/java/de/jplag/SubmissionSetBuilder.java @@ -6,6 +6,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -21,6 +22,9 @@ import de.jplag.exceptions.ExitException; import de.jplag.exceptions.RootDirectoryException; import de.jplag.exceptions.SubmissionException; +import de.jplag.logging.ProgressBar; +import de.jplag.logging.ProgressBarLogger; +import de.jplag.logging.ProgressBarType; import de.jplag.options.JPlagOptions; /** @@ -35,9 +39,9 @@ public class SubmissionSetBuilder { /** * Creates a builder for submission sets. - * @deprecated in favor of {@link #SubmissionSetBuilder(JPlagOptions)}. * @param language is the language of the submissions. * @param options are the configured options. + * @deprecated in favor of {@link #SubmissionSetBuilder(JPlagOptions)}. */ @Deprecated(since = "4.3.0") public SubmissionSetBuilder(Language language, JPlagOptions options) { @@ -67,14 +71,21 @@ public SubmissionSet buildSubmissionSet() throws ExitException { int numberOfRootDirectories = submissionDirectories.size() + oldSubmissionDirectories.size(); boolean multipleRoots = (numberOfRootDirectories > 1); - // Collect valid looking entries from the root directories. - Map foundSubmissions = new HashMap<>(); - for (File directory : submissionDirectories) { - processRootDirectoryEntries(directory, multipleRoots, foundSubmissions, true); + List submissionFiles = new ArrayList<>(); + for (File submissionDirectory : submissionDirectories) { + submissionFiles.addAll(listSubmissionFiles(submissionDirectory, true)); } - for (File oldDirectory : oldSubmissionDirectories) { - processRootDirectoryEntries(oldDirectory, multipleRoots, foundSubmissions, false); + for (File submissionDirectory : oldSubmissionDirectories) { + submissionFiles.addAll(listSubmissionFiles(submissionDirectory, false)); + } + + ProgressBar progressBar = ProgressBarLogger.createProgressBar(ProgressBarType.LOADING, submissionFiles.size()); + Map foundSubmissions = new HashMap<>(); + for (SubmissionFileData submissionFile : submissionFiles) { + processSubmissionFile(submissionFile, multipleRoots, foundSubmissions); + progressBar.step(); } + progressBar.dispose(); Optional baseCodeSubmission = loadBaseCode(); baseCodeSubmission.ifPresent(baseSubmission -> foundSubmissions.remove(baseSubmission.getRoot())); @@ -84,7 +95,7 @@ public SubmissionSet buildSubmissionSet() throws ExitException { // Some languages expect a certain order, which is ensured here: if (options.language().expectsSubmissionOrder()) { - List rootFiles = foundSubmissions.values().stream().map(it -> it.getRoot()).toList(); + List rootFiles = foundSubmissions.values().stream().map(Submission::getRoot).toList(); rootFiles = options.language().customizeSubmissionOrder(rootFiles); submissions = new ArrayList<>(rootFiles.stream().map(foundSubmissions::get).toList()); } @@ -155,31 +166,25 @@ private Optional loadBaseCode() throws ExitException { Submission baseCodeSubmission = processSubmission(baseCodeSubmissionDirectory.getName(), baseCodeSubmissionDirectory, false); logger.info("Basecode directory \"{}\" will be used.", baseCodeSubmission.getName()); - return Optional.ofNullable(baseCodeSubmission); + return Optional.of(baseCodeSubmission); } - /** - * Read entries in the given root directory. - */ - private String[] listSubmissionFiles(File rootDirectory) throws ExitException { + private List listSubmissionFiles(File rootDirectory, boolean isNew) throws RootDirectoryException { if (!rootDirectory.isDirectory()) { throw new AssertionError("Given root is not a directory."); } - String[] fileNames; - try { - fileNames = rootDirectory.list(); + File[] files = rootDirectory.listFiles(); + if (files == null) { + throw new RootDirectoryException("Cannot list files of the root directory!"); + } + + return Arrays.stream(files).sorted(Comparator.comparing(File::getName)).map(it -> new SubmissionFileData(it, rootDirectory, isNew)) + .toList(); } catch (SecurityException exception) { throw new RootDirectoryException("Cannot list files of the root directory! " + exception.getMessage(), exception); } - - if (fileNames == null) { - throw new RootDirectoryException("Cannot list files of the root directory!"); - } - - Arrays.sort(fileNames); - return fileNames; } /** @@ -200,6 +205,7 @@ private String isExcludedEntry(File submissionEntry) { /** * Process the given directory entry as a submission, the path MUST not be excluded. + * @param submissionName The name of the submission * @param submissionFile the file for the submission. * @param isNew states whether submissions found in the root directory must be checked for plagiarism. * @return The entry converted to a submission. @@ -225,27 +231,16 @@ private Submission processSubmission(String submissionName, File submissionFile, return new Submission(submissionName, submissionFile, isNew, parseFilesRecursively(submissionFile), options.language()); } - /** - * Process entries in the root directory to check whether they qualify as submissions. - * @param rootDirectory is the root directory being examined. - * @param foundSubmissions Submissions found so far, is updated in-place. - * @param isNew states whether submissions found in the root directory must be checked for plagiarism. - */ - private void processRootDirectoryEntries(File rootDirectory, boolean multipleRoots, Map foundSubmissions, boolean isNew) - throws ExitException { - for (String fileName : listSubmissionFiles(rootDirectory)) { - File submissionFile = new File(rootDirectory, fileName); - - String errorMessage = isExcludedEntry(submissionFile); - if (errorMessage == null) { - String rootDirectoryPrefix = multipleRoots ? (rootDirectory.getName() + File.separator) : ""; - String submissionName = rootDirectoryPrefix + fileName; - Submission submission = processSubmission(submissionName, submissionFile, isNew); - foundSubmissions.put(submission.getRoot(), submission); - } else { - logger.error(errorMessage); - } + private void processSubmissionFile(SubmissionFileData file, boolean multipleRoots, Map foundSubmissions) throws ExitException { + String errorMessage = isExcludedEntry(file.submissionFile()); + if (errorMessage != null) { + logger.error(errorMessage); } + + String rootDirectoryPrefix = multipleRoots ? (file.root().getName() + File.separator) : ""; + String submissionName = rootDirectoryPrefix + file.submissionFile().getName(); + Submission submission = processSubmission(submissionName, file.submissionFile(), file.isNew()); + foundSubmissions.put(submission.getRoot(), submission); } /** @@ -311,4 +306,5 @@ private File makeCanonical(File file, Function excepti throw exceptionWrapper.apply(exception); } } + } diff --git a/core/src/main/java/de/jplag/SubsequenceHashLookupTable.java b/core/src/main/java/de/jplag/SubsequenceHashLookupTable.java index aa44c783b..e19a41643 100644 --- a/core/src/main/java/de/jplag/SubsequenceHashLookupTable.java +++ b/core/src/main/java/de/jplag/SubsequenceHashLookupTable.java @@ -46,16 +46,6 @@ class SubsequenceHashLookupTable { computeSubsequenceHashes(marked); } - /** Returns the size of the subsequences used for hashing */ - int getWindowSize() { - return windowSize; - } - - /** Returns the list of values for which the hashes were computed */ - int[] getValues() { - return values; - } - /** * Returns the hash over the subsequence from startIndex to startIndex+windowSize. * @param startIndex the start index. diff --git a/core/src/main/java/de/jplag/logging/ProgressBar.java b/core/src/main/java/de/jplag/logging/ProgressBar.java new file mode 100644 index 000000000..04450434a --- /dev/null +++ b/core/src/main/java/de/jplag/logging/ProgressBar.java @@ -0,0 +1,24 @@ +package de.jplag.logging; + +/** + * Exposed interactions for a running progress bar. + */ +public interface ProgressBar { + /** + * Advances the progress bar by a single step + */ + default void step() { + step(1); + } + + /** + * Advances the progress bar by amount steps + * @param number The number of steps + */ + void step(int number); + + /** + * Closes the progress bar. After this method has been called the behaviour of the other methods is undefined. + */ + void dispose(); +} diff --git a/core/src/main/java/de/jplag/logging/ProgressBarLogger.java b/core/src/main/java/de/jplag/logging/ProgressBarLogger.java new file mode 100644 index 000000000..889d391bb --- /dev/null +++ b/core/src/main/java/de/jplag/logging/ProgressBarLogger.java @@ -0,0 +1,68 @@ +package de.jplag.logging; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides static access to the creation of progress bars. + */ +public class ProgressBarLogger { + private static ProgressBarProvider progressBarProvider = new DummyProvider(); + + private ProgressBarLogger() { + // Hides default constructor + } + + /** + * Creates a new {@link ProgressBar} + * @param type The type of the progress bar + * @param totalSteps The total number of steps + * @return The newly created progress bar + */ + public static ProgressBar createProgressBar(ProgressBarType type, int totalSteps) { + return progressBarProvider.initProgressBar(type, totalSteps); + } + + /** + * Sets the {@link ProgressBarProvider}. Should be used by the ui before calling JPlag, if progress bars should be + * shown. + * @param progressBarProvider The provider + */ + public static void setProgressBarProvider(ProgressBarProvider progressBarProvider) { + ProgressBarLogger.progressBarProvider = progressBarProvider; + } + + private static class DummyProvider implements ProgressBarProvider { + @Override + public ProgressBar initProgressBar(ProgressBarType type, int totalSteps) { + return new DummyBar(type, totalSteps); + } + } + + private static class DummyBar implements ProgressBar { + private static final Logger logger = LoggerFactory.getLogger(DummyBar.class); + private int currentStep; + + public DummyBar(ProgressBarType type, int totalSteps) { + this.currentStep = 0; + logger.info("{} ({})", type.getDefaultText(), totalSteps); + } + + @Override + public void step() { + logger.info("Now at step {}", this.currentStep++); + } + + @Override + public void step(int number) { + for (int i = 0; i < number; i++) { + step(); + } + } + + @Override + public void dispose() { + logger.info("Progress bar done."); + } + } +} diff --git a/core/src/main/java/de/jplag/logging/ProgressBarProvider.java b/core/src/main/java/de/jplag/logging/ProgressBarProvider.java new file mode 100644 index 000000000..13268325b --- /dev/null +++ b/core/src/main/java/de/jplag/logging/ProgressBarProvider.java @@ -0,0 +1,14 @@ +package de.jplag.logging; + +/** + * Provides the capability to create new progress bars, to allow JPlag to access the ui. + */ +public interface ProgressBarProvider { + /** + * Creates a new progress bar + * @param type The type of progress bar. Should mostly determine the name + * @param totalSteps The total number of steps the progress bar should have + * @return The newly created bar + */ + ProgressBar initProgressBar(ProgressBarType type, int totalSteps); +} diff --git a/core/src/main/java/de/jplag/logging/ProgressBarType.java b/core/src/main/java/de/jplag/logging/ProgressBarType.java new file mode 100644 index 000000000..88e520fcc --- /dev/null +++ b/core/src/main/java/de/jplag/logging/ProgressBarType.java @@ -0,0 +1,23 @@ +package de.jplag.logging; + +/** + * The available processes. Used as a hint for the ui, which step JPlag is currently performing. + */ +public enum ProgressBarType { + LOADING("Loading Submissions "), + PARSING("Parsing Submissions "), + COMPARING("Comparing Submissions"); + + private final String defaultText; + + ProgressBarType(String defaultText) { + this.defaultText = defaultText; + } + + /** + * @return The default display text for the type + */ + public String getDefaultText() { + return defaultText; + } +} diff --git a/core/src/main/java/de/jplag/merging/MatchMerging.java b/core/src/main/java/de/jplag/merging/MatchMerging.java index cdcb426dd..3067e32a6 100644 --- a/core/src/main/java/de/jplag/merging/MatchMerging.java +++ b/core/src/main/java/de/jplag/merging/MatchMerging.java @@ -1,10 +1,8 @@ package de.jplag.merging; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; +import java.util.Comparator; import java.util.List; -import java.util.Map; import de.jplag.JPlagComparison; import de.jplag.JPlagResult; @@ -24,7 +22,7 @@ * {@link JPlagOptions} as {@link MergingOptions} and default to (2,6). */ public class MatchMerging { - private JPlagOptions options; + private final JPlagOptions options; /** * Instantiates the match merging algorithm for a comparison result and a set of specific options. @@ -68,25 +66,15 @@ public JPlagResult mergeMatchesOf(JPlagResult result) { */ private List computeNeighbors(List globalMatches) { List neighbors = new ArrayList<>(); + List sortedByLeft = new ArrayList<>(globalMatches); + List sortedByRight = new ArrayList<>(globalMatches); - Map> matchesByLeft = new HashMap<>(); - Map> matchesByRight = new HashMap<>(); + sortedByLeft.sort(Comparator.comparingInt(Match::startOfFirst)); + sortedByRight.sort(Comparator.comparingInt(Match::startOfSecond)); - // Group matches by their left and right positions - for (Match match : globalMatches) { - matchesByLeft.computeIfAbsent(match.startOfFirst(), key -> new ArrayList<>()).add(match); - matchesByRight.computeIfAbsent(match.startOfSecond(), key -> new ArrayList<>()).add(match); - } - - // Iterate through the matches and find neighbors - for (List matches : matchesByLeft.values()) { - for (Match match : matches) { - List rightMatches = matchesByRight.getOrDefault(match.startOfSecond(), Collections.emptyList()); - for (Match rightMatch : rightMatches) { - if (rightMatch != match) { - neighbors.add(new Neighbor(match, rightMatch)); - } - } + for (int i = 0; i < sortedByLeft.size() - 1; i++) { + if (sortedByRight.indexOf(sortedByLeft.get(i)) == (sortedByRight.indexOf(sortedByLeft.get(i + 1)) - 1)) { + neighbors.add(new Neighbor(sortedByLeft.get(i), sortedByLeft.get(i + 1))); } } diff --git a/core/src/main/java/de/jplag/merging/MergingOptions.java b/core/src/main/java/de/jplag/merging/MergingOptions.java index 4c2b49e08..7d77f7f31 100644 --- a/core/src/main/java/de/jplag/merging/MergingOptions.java +++ b/core/src/main/java/de/jplag/merging/MergingOptions.java @@ -10,12 +10,16 @@ public record MergingOptions(@JsonProperty("enabled") boolean enabled, @JsonProperty("min_neighbour_length") int minimumNeighborLength, @JsonProperty("max_gap_size") int maximumGapSize) { + public static final boolean DEFAULT_ENABLED = false; + public static final int DEFAULT_NEIGHBOR_LENGTH = 2; + public static final int DEFAULT_GAP_SIZE = 6; + /** * The default values of MergingOptions are false for the enable-switch, which deactivate MatchMerging, while * minimumNeighborLength and maximumGapSize default to (2,6), which in testing yielded the best results. */ public MergingOptions() { - this(false, 2, 6); + this(DEFAULT_ENABLED, DEFAULT_NEIGHBOR_LENGTH, DEFAULT_GAP_SIZE); } /** diff --git a/core/src/main/java/de/jplag/normalization/MultipleEdge.java b/core/src/main/java/de/jplag/normalization/MultipleEdge.java index 6265c5371..b10fda2ea 100644 --- a/core/src/main/java/de/jplag/normalization/MultipleEdge.java +++ b/core/src/main/java/de/jplag/normalization/MultipleEdge.java @@ -9,7 +9,7 @@ * Models a multiple edge in the normalization graph. Contains multiple edges. */ class MultipleEdge { - private Set edges; + private final Set edges; private boolean isVariableFlow; private boolean isVariableReverseFlow; @@ -27,10 +27,12 @@ boolean isVariableReverseFlow() { } void addEdge(EdgeType type, Variable cause) { - if (type == EdgeType.VARIABLE_FLOW) + if (type == EdgeType.VARIABLE_FLOW) { isVariableFlow = true; - if (type == EdgeType.VARIABLE_REVERSE_FLOW) + } + if (type == EdgeType.VARIABLE_REVERSE_FLOW) { isVariableReverseFlow = true; + } edges.add(new Edge(type, cause)); } } diff --git a/core/src/main/java/de/jplag/normalization/NormalizationGraphConstructor.java b/core/src/main/java/de/jplag/normalization/NormalizationGraphConstructor.java index a5f4e496e..ace0d1ba3 100644 --- a/core/src/main/java/de/jplag/normalization/NormalizationGraphConstructor.java +++ b/core/src/main/java/de/jplag/normalization/NormalizationGraphConstructor.java @@ -17,14 +17,14 @@ * Constructs the normalization graph. */ class NormalizationGraphConstructor { - private SimpleDirectedGraph graph; + private final SimpleDirectedGraph graph; private int bidirectionalBlockDepth; - private Collection fullPositionSignificanceIncoming; + private final Collection fullPositionSignificanceIncoming; private Statement lastFullPositionSignificance; private Statement lastPartialPositionSignificance; - private Map> variableReads; - private Map> variableWrites; - private Set inCurrentBidirectionalBlock; + private final Map> variableReads; + private final Map> variableWrites; + private final Set inCurrentBidirectionalBlock; private Statement current; NormalizationGraphConstructor(List tokens) { @@ -57,24 +57,28 @@ private void addStatement(Statement statement) { processPartialPositionSignificance(); processReads(); processWrites(); - for (Variable variable : current.semantics().reads()) + for (Variable variable : current.semantics().reads()) { addVariableToMap(variableReads, variable); - for (Variable variable : current.semantics().writes()) + } + for (Variable variable : current.semantics().writes()) { addVariableToMap(variableWrites, variable); + } } private void processBidirectionalBlock() { bidirectionalBlockDepth += current.semantics().bidirectionalBlockDepthChange(); - if (bidirectionalBlockDepth > 0) + if (bidirectionalBlockDepth > 0) { inCurrentBidirectionalBlock.add(current); - else + } else { inCurrentBidirectionalBlock.clear(); + } } private void processFullPositionSignificance() { if (current.semantics().hasFullPositionSignificance()) { - for (Statement node : fullPositionSignificanceIncoming) + for (Statement node : fullPositionSignificanceIncoming) { addIncomingEdgeToCurrent(node, EdgeType.POSITION_SIGNIFICANCE_FULL, null); + } fullPositionSignificanceIncoming.clear(); lastFullPositionSignificance = current; } else if (lastFullPositionSignificance != null) { @@ -94,15 +98,17 @@ private void processPartialPositionSignificance() { private void processReads() { for (Variable variable : current.semantics().reads()) { - for (Statement node : variableWrites.getOrDefault(variable, Set.of())) + for (Statement node : variableWrites.getOrDefault(variable, Set.of())) { addIncomingEdgeToCurrent(node, EdgeType.VARIABLE_FLOW, variable); + } } } private void processWrites() { for (Variable variable : current.semantics().writes()) { - for (Statement node : variableWrites.getOrDefault(variable, Set.of())) + for (Statement node : variableWrites.getOrDefault(variable, Set.of())) { addIncomingEdgeToCurrent(node, EdgeType.VARIABLE_ORDER, variable); + } for (Statement node : variableReads.getOrDefault(variable, Set.of())) { EdgeType edgeType = inCurrentBidirectionalBlock.contains(node) ? // EdgeType.VARIABLE_REVERSE_FLOW : EdgeType.VARIABLE_ORDER; diff --git a/core/src/main/java/de/jplag/normalization/Statement.java b/core/src/main/java/de/jplag/normalization/Statement.java index c7086bb74..a749a5774 100644 --- a/core/src/main/java/de/jplag/normalization/Statement.java +++ b/core/src/main/java/de/jplag/normalization/Statement.java @@ -41,24 +41,28 @@ private int tokenOrdinal(Token token) { @Override public int compareTo(Statement other) { int sizeComp = Integer.compare(this.tokens.size(), other.tokens.size()); - if (sizeComp != 0) + if (sizeComp != 0) { return -sizeComp; // bigger size should come first + } Iterator myTokens = this.tokens.iterator(); Iterator otherTokens = other.tokens.iterator(); for (int i = 0; i < this.tokens.size(); i++) { int tokenComp = Integer.compare(tokenOrdinal(myTokens.next()), tokenOrdinal(otherTokens.next())); - if (tokenComp != 0) + if (tokenComp != 0) { return tokenComp; + } } return 0; } @Override public boolean equals(Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null || getClass() != obj.getClass()) + } + if (obj == null || getClass() != obj.getClass()) { return false; + } return tokens.equals(((Statement) obj).tokens); } diff --git a/core/src/main/java/de/jplag/normalization/StatementBuilder.java b/core/src/main/java/de/jplag/normalization/StatementBuilder.java index 1afaa3eb3..eef5d0c82 100644 --- a/core/src/main/java/de/jplag/normalization/StatementBuilder.java +++ b/core/src/main/java/de/jplag/normalization/StatementBuilder.java @@ -10,7 +10,7 @@ */ class StatementBuilder { - private List tokens; + private final List tokens; private final int lineNumber; StatementBuilder(int lineNumber) { diff --git a/core/src/main/java/de/jplag/normalization/TokenStringNormalizer.java b/core/src/main/java/de/jplag/normalization/TokenStringNormalizer.java index c58a9188e..5ece0ff1f 100644 --- a/core/src/main/java/de/jplag/normalization/TokenStringNormalizer.java +++ b/core/src/main/java/de/jplag/normalization/TokenStringNormalizer.java @@ -13,7 +13,7 @@ import de.jplag.Token; /** - * Performs token string normalization. + * Performs token sequence normalization. */ public class TokenStringNormalizer { @@ -21,11 +21,11 @@ private TokenStringNormalizer() { } /** - * Performs token string normalization. Tokens representing dead code have been eliminated and tokens representing + * Performs token sequence normalization. Tokens representing dead code have been eliminated and tokens representing * subsequent independent statements have been put in a fixed order. Works by first constructing a Normalization Graph - * and then turning it back into a token string. - * @param tokens The original token string, remains unaltered. - * @return The normalized token string. + * and then turning it back into a token sequence. + * @param tokens The original token sequence, remains unaltered. + * @return The normalized token sequence. */ public static List normalize(List tokens) { SimpleDirectedGraph normalizationGraph = new NormalizationGraphConstructor(tokens).get(); diff --git a/core/src/main/java/de/jplag/options/JPlagOptions.java b/core/src/main/java/de/jplag/options/JPlagOptions.java index f2e876f43..6d7067748 100644 --- a/core/src/main/java/de/jplag/options/JPlagOptions.java +++ b/core/src/main/java/de/jplag/options/JPlagOptions.java @@ -55,7 +55,8 @@ public record JPlagOptions(@JsonSerialize(using = LanguageSerializer.class) Lang @JsonProperty("subdirectory_name") String subdirectoryName, @JsonProperty("file_suffixes") List fileSuffixes, @JsonProperty("exclusion_file_name") String exclusionFileName, @JsonProperty("similarity_metric") SimilarityMetric similarityMetric, @JsonProperty("similarity_threshold") double similarityThreshold, @JsonProperty("max_comparisons") int maximumNumberOfComparisons, - @JsonProperty("cluster") ClusteringOptions clusteringOptions, boolean debugParser, @JsonProperty("merging") MergingOptions mergingOptions) { + @JsonProperty("cluster") ClusteringOptions clusteringOptions, boolean debugParser, @JsonProperty("merging") MergingOptions mergingOptions, + @JsonProperty("normalize") boolean normalize) { public static final double DEFAULT_SIMILARITY_THRESHOLD = 0; public static final int DEFAULT_SHOWN_COMPARISONS = 500; @@ -68,13 +69,13 @@ public record JPlagOptions(@JsonSerialize(using = LanguageSerializer.class) Lang public JPlagOptions(Language language, Set submissionDirectories, Set oldSubmissionDirectories) { this(language, null, submissionDirectories, oldSubmissionDirectories, null, null, null, null, DEFAULT_SIMILARITY_METRIC, - DEFAULT_SIMILARITY_THRESHOLD, DEFAULT_SHOWN_COMPARISONS, new ClusteringOptions(), false, new MergingOptions()); + DEFAULT_SIMILARITY_THRESHOLD, DEFAULT_SHOWN_COMPARISONS, new ClusteringOptions(), false, new MergingOptions(), false); } public JPlagOptions(Language language, Integer minimumTokenMatch, Set submissionDirectories, Set oldSubmissionDirectories, File baseCodeSubmissionDirectory, String subdirectoryName, List fileSuffixes, String exclusionFileName, SimilarityMetric similarityMetric, double similarityThreshold, int maximumNumberOfComparisons, ClusteringOptions clusteringOptions, - boolean debugParser, MergingOptions mergingOptions) { + boolean debugParser, MergingOptions mergingOptions, boolean normalize) { this.language = language; this.debugParser = debugParser; this.fileSuffixes = fileSuffixes == null || fileSuffixes.isEmpty() ? null : Collections.unmodifiableList(fileSuffixes); @@ -89,90 +90,97 @@ public JPlagOptions(Language language, Integer minimumTokenMatch, Set subm this.subdirectoryName = subdirectoryName; this.clusteringOptions = clusteringOptions; this.mergingOptions = mergingOptions; + this.normalize = normalize; } public JPlagOptions withLanguageOption(Language language) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withDebugParser(boolean debugParser) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withFileSuffixes(List fileSuffixes) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withSimilarityThreshold(double similarityThreshold) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withMaximumNumberOfComparisons(int maximumNumberOfComparisons) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withSimilarityMetric(SimilarityMetric similarityMetric) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withMinimumTokenMatch(Integer minimumTokenMatch) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withExclusionFileName(String exclusionFileName) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withSubmissionDirectories(Set submissionDirectories) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withOldSubmissionDirectories(Set oldSubmissionDirectories) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withBaseCodeSubmissionDirectory(File baseCodeSubmissionDirectory) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withSubdirectoryName(String subdirectoryName) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withClusteringOptions(ClusteringOptions clusteringOptions) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); } public JPlagOptions withMergingOptions(MergingOptions mergingOptions) { return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, - clusteringOptions, debugParser, mergingOptions); + clusteringOptions, debugParser, mergingOptions, normalize); + } + + public JPlagOptions withNormalize(boolean normalize) { + return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionDirectory, + subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons, + clusteringOptions, debugParser, mergingOptions, normalize); } public boolean hasBaseCode() { @@ -186,16 +194,18 @@ public Set excludedFiles() { @Override public List fileSuffixes() { var language = language(); - if ((fileSuffixes == null || fileSuffixes.isEmpty()) && language != null) + if ((fileSuffixes == null || fileSuffixes.isEmpty()) && language != null) { return Arrays.stream(language.suffixes()).toList(); + } return fileSuffixes == null ? null : Collections.unmodifiableList(fileSuffixes); } @Override public Integer minimumTokenMatch() { var language = language(); - if (minimumTokenMatch == null && language != null) + if (minimumTokenMatch == null && language != null) { return language.minimumTokenMatch(); + } return minimumTokenMatch; } @@ -216,7 +226,8 @@ private static double normalizeSimilarityThreshold(double similarityThreshold) { if (similarityThreshold > 1) { logger.warn("Maximum threshold of 1 used instead of {}", similarityThreshold); return 1; - } else if (similarityThreshold < 0) { + } + if (similarityThreshold < 0) { logger.warn("Minimum threshold of 0 used instead of {}", similarityThreshold); return 0; } else { @@ -264,7 +275,7 @@ public JPlagOptions(Language language, Integer minimumTokenMatch, File submissio boolean debugParser, MergingOptions mergingOptions) throws BasecodeException { this(language, minimumTokenMatch, Set.of(submissionDirectory), oldSubmissionDirectories, convertLegacyBaseCodeToFile(baseCodeSubmissionName, submissionDirectory), subdirectoryName, fileSuffixes, exclusionFileName, - similarityMetric, similarityThreshold, maximumNumberOfComparisons, clusteringOptions, debugParser, mergingOptions); + similarityMetric, similarityThreshold, maximumNumberOfComparisons, clusteringOptions, debugParser, mergingOptions, false); } /** @@ -305,19 +316,18 @@ private static File convertLegacyBaseCodeToFile(String baseCodeSubmissionName, F File baseCodeAsAbsolutePath = new File(baseCodeSubmissionName); if (baseCodeAsAbsolutePath.exists()) { return baseCodeAsAbsolutePath; - } else { - String normalizedName = baseCodeSubmissionName; - while (normalizedName.startsWith(File.separator)) { - normalizedName = normalizedName.substring(1); - } - while (normalizedName.endsWith(File.separator)) { - normalizedName = normalizedName.substring(0, normalizedName.length() - 1); - } - if (normalizedName.isEmpty() || normalizedName.contains(File.separator) || normalizedName.contains(".")) { - throw new BasecodeException( - "The basecode directory name \"" + normalizedName + "\" cannot contain dots! Please migrate to the path-based API."); - } - return new File(submissionDirectory, baseCodeSubmissionName); } + String normalizedName = baseCodeSubmissionName; + while (normalizedName.startsWith(File.separator)) { + normalizedName = normalizedName.substring(1); + } + while (normalizedName.endsWith(File.separator)) { + normalizedName = normalizedName.substring(0, normalizedName.length() - 1); + } + if (normalizedName.isEmpty() || normalizedName.contains(File.separator) || normalizedName.contains(".")) { + throw new BasecodeException( + "The basecode directory name \"" + normalizedName + "\" cannot contain dots! Please migrate to the path-based API."); + } + return new File(submissionDirectory, baseCodeSubmissionName); } } diff --git a/core/src/main/java/de/jplag/reporting/FilePathUtil.java b/core/src/main/java/de/jplag/reporting/FilePathUtil.java index d2db74ad7..51aefdd36 100644 --- a/core/src/main/java/de/jplag/reporting/FilePathUtil.java +++ b/core/src/main/java/de/jplag/reporting/FilePathUtil.java @@ -7,6 +7,7 @@ import de.jplag.Submission; public final class FilePathUtil { + private static final String ZIP_PATH_SEPARATOR = "/"; // Paths in zip files are always separated by a slash private FilePathUtil() { // private constructor to prevent instantiation @@ -26,4 +27,23 @@ public static String getRelativeSubmissionPath(File file, Submission submission, return Path.of(submissionToIdFunction.apply(submission), submission.getRoot().toPath().relativize(file.toPath()).toString()).toString(); } + /** + * Joins logical paths using a slash. This method ensures, that no duplicate slashes are created in between. + * @param left The left path segment + * @param right The right path segment + * @return The joined paths + */ + public static String joinZipPathSegments(String left, String right) { + String rightStripped = right; + while (rightStripped.startsWith(ZIP_PATH_SEPARATOR)) { + rightStripped = rightStripped.substring(1); + } + + String leftStripped = left; + while (leftStripped.endsWith(ZIP_PATH_SEPARATOR)) { + leftStripped = leftStripped.substring(0, leftStripped.length() - 1); + } + + return leftStripped + ZIP_PATH_SEPARATOR + rightStripped; + } } diff --git a/core/src/main/java/de/jplag/reporting/jsonfactory/ComparisonReportWriter.java b/core/src/main/java/de/jplag/reporting/jsonfactory/ComparisonReportWriter.java index 8ebb4c555..29f93744c 100644 --- a/core/src/main/java/de/jplag/reporting/jsonfactory/ComparisonReportWriter.java +++ b/core/src/main/java/de/jplag/reporting/jsonfactory/ComparisonReportWriter.java @@ -3,7 +3,6 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @@ -16,7 +15,7 @@ import de.jplag.reporting.FilePathUtil; import de.jplag.reporting.reportobject.model.ComparisonReport; import de.jplag.reporting.reportobject.model.Match; -import de.jplag.reporting.reportobject.writer.JsonWriter; +import de.jplag.reporting.reportobject.writer.JPlagResultWriter; /** * Writes {@link ComparisonReport}s of given {@link JPlagResult} to the disk under the specified path. Instantiated with @@ -24,44 +23,43 @@ */ public class ComparisonReportWriter { - private final JsonWriter fileWriter; + private final JPlagResultWriter resultWriter; private final Function submissionToIdFunction; private final Map> submissionIdToComparisonFileName = new ConcurrentHashMap<>(); private final Map fileNameCollisions = new ConcurrentHashMap<>(); - public ComparisonReportWriter(Function submissionToIdFunction, JsonWriter fileWriter) { + public ComparisonReportWriter(Function submissionToIdFunction, JPlagResultWriter resultWriter) { this.submissionToIdFunction = submissionToIdFunction; - this.fileWriter = fileWriter; + this.resultWriter = resultWriter; } /** * Generates detailed ComparisonReport DTO for each comparison in a JPlagResult and writes them to the disk as json * files. * @param jPlagResult The JPlagResult to generate the comparison reports from. contains information about a comparison - * @param path The path to write the comparison files to * @return Nested map that associates each pair of submissions (by their ids) to their comparison file name. The * comparison file name for submission with id id1 and id2 can be fetched by executing get two times: * map.get(id1).get(id2). The nested map is symmetrical therefore, both map.get(id1).get(id2) and map.get(id2).get(id1) * yield the same result. */ - public Map> writeComparisonReports(JPlagResult jPlagResult, String path) { + public Map> writeComparisonReports(JPlagResult jPlagResult) { int numberOfComparisons = jPlagResult.getOptions().maximumNumberOfComparisons(); List comparisons = jPlagResult.getComparisons(numberOfComparisons); - writeComparisons(path, comparisons); + writeComparisons(comparisons); return submissionIdToComparisonFileName; } - private void writeComparisons(String path, List comparisons) { - comparisons.parallelStream().forEach(comparison -> { + private void writeComparisons(List comparisons) { + for (JPlagComparison comparison : comparisons) { String firstSubmissionId = submissionToIdFunction.apply(comparison.firstSubmission()); String secondSubmissionId = submissionToIdFunction.apply(comparison.secondSubmission()); String fileName = generateComparisonName(firstSubmissionId, secondSubmissionId); addToLookUp(firstSubmissionId, secondSubmissionId, fileName); var comparisonReport = new ComparisonReport(firstSubmissionId, secondSubmissionId, Map.of(SimilarityMetric.AVG.name(), comparison.similarity(), SimilarityMetric.MAX.name(), comparison.maximalSimilarity()), - convertMatchesToReportMatches(comparison)); - fileWriter.writeFile(comparisonReport, path, fileName); - }); + convertMatchesToReportMatches(comparison), comparison.similarityOfFirst(), comparison.similarityOfSecond()); + resultWriter.addJsonEntry(comparisonReport, fileName); + } } private void addToLookUp(String firstSubmissionId, String secondSubmissionId, String fileName) { @@ -99,20 +97,16 @@ private Match convertMatchToReportMatch(JPlagComparison comparison, de.jplag.Mat List tokensFirst = comparison.firstSubmission().getTokenList().subList(match.startOfFirst(), match.endOfFirst() + 1); List tokensSecond = comparison.secondSubmission().getTokenList().subList(match.startOfSecond(), match.endOfSecond() + 1); - Comparator lineComparator = (first, second) -> first.getLine() - second.getLine(); + Comparator lineComparator = Comparator.comparingInt(Token::getLine); Token startOfFirst = tokensFirst.stream().min(lineComparator).orElseThrow(); Token endOfFirst = tokensFirst.stream().max(lineComparator).orElseThrow(); Token startOfSecond = tokensSecond.stream().min(lineComparator).orElseThrow(); Token endOfSecond = tokensSecond.stream().max(lineComparator).orElseThrow(); - List firstTotalTokens = tokensFirst.stream().filter(x -> Objects.equals(x.getFile(), startOfFirst.getFile())).toList(); - List secondTotalTokens = tokensSecond.stream().filter(x -> Objects.equals(x.getFile(), startOfSecond.getFile())).toList(); - return new Match(FilePathUtil.getRelativeSubmissionPath(startOfFirst.getFile(), comparison.firstSubmission(), submissionToIdFunction), FilePathUtil.getRelativeSubmissionPath(startOfSecond.getFile(), comparison.secondSubmission(), submissionToIdFunction), - startOfFirst.getLine(), endOfFirst.getLine(), startOfSecond.getLine(), endOfSecond.getLine(), match.length(), firstTotalTokens.size(), - secondTotalTokens.size()); + startOfFirst.getLine(), endOfFirst.getLine(), startOfSecond.getLine(), endOfSecond.getLine(), match.length()); } } diff --git a/core/src/main/java/de/jplag/reporting/jsonfactory/DirectoryManager.java b/core/src/main/java/de/jplag/reporting/jsonfactory/DirectoryManager.java deleted file mode 100644 index 8bbe60593..000000000 --- a/core/src/main/java/de/jplag/reporting/jsonfactory/DirectoryManager.java +++ /dev/null @@ -1,121 +0,0 @@ -package de.jplag.reporting.jsonfactory; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Comparator; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Provides Methods for creating directories. - */ -public class DirectoryManager { - private static final Logger logger = LoggerFactory.getLogger(DirectoryManager.class); - - /** - * Creates a full path directory. - * @param path The path under which the new directory or file ought to be created - * @param name The name of the new directory. According to this name we can get sub-folder's structure after this - * directory. - * @param file The file, which has the path of sub-folders - * @param submissionRoot The file, which has the root path of submission - * @return The created directory which has the whole structure as file - */ - public static File createDirectory(String path, String name, File file, File submissionRoot) throws IOException { - File directory; - String fileFullPath = file.getPath(); - String submissionRootPath = submissionRoot.getPath(); - String filePathWithoutRootName = fileFullPath.substring(submissionRootPath.length()); - String outputRootDirectory = Path.of(path, name).toString(); - if ("".equals(filePathWithoutRootName)) { - directory = new File(Path.of(outputRootDirectory, name).toString()); - } else { - directory = new File(outputRootDirectory + filePathWithoutRootName); - } - if (!directory.exists() && !directory.mkdirs()) { - throw new IOException("Failed to create dir."); - } - return directory; - } - - /** - * Creates a directory. - * @param path The path under which the new directory ought to be created - * @param name The name of the new directory - * @return The created directory - */ - public static File createDirectory(String path, String name) throws IOException { - File directory = new File(path.concat(File.separator).concat(name)); - if (!directory.exists() && !directory.mkdirs()) { - throw new IOException("Failed to create dir."); - } - return directory; - } - - /** - * Create a directory with the given path - * @param path The path of the new directory - */ - public static void createDirectory(String path) throws IOException { - createDirectory(path, ""); - } - - /** - * Delete the directory and all of its contents, identified by the given path - * @param path The path that identifies the directory to delete - */ - public static void deleteDirectory(String path) { - try (var f = Files.walk(Path.of(path))) { - f.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); - } catch (IOException e) { - logger.error("Could not delete folder " + path, e); - } - } - - /** - * Zip the directory identified by the given path - * @param path The path that identifies the directory to zip - * @return True if zip was successful, false otherwise - */ - public static boolean zipDirectory(String path) { - Path p = Path.of(path); - String zipName = path + ".zip"; - - try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipName))) { - - Files.walkFileTree(p, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Path targetFile = p.relativize(file); - zos.putNextEntry(new ZipEntry(targetFile.toString())); - - byte[] bytes = Files.readAllBytes(file); - zos.write(bytes, 0, bytes.length); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { - logger.error("Unable to zip " + file, exc); - throw exc; - } - }); - } catch (IOException e) { - logger.error(e.getMessage(), e); - deleteDirectory(zipName); - return false; - } - logger.info("Successfully zipped report files: {}", zipName); - logger.info("Display the results with the report viewer at https://jplag.github.io/JPlag/"); - return true; - } -} diff --git a/core/src/main/java/de/jplag/reporting/jsonfactory/serializer/LanguageSerializer.java b/core/src/main/java/de/jplag/reporting/jsonfactory/serializer/LanguageSerializer.java index 22e961105..c521507c7 100644 --- a/core/src/main/java/de/jplag/reporting/jsonfactory/serializer/LanguageSerializer.java +++ b/core/src/main/java/de/jplag/reporting/jsonfactory/serializer/LanguageSerializer.java @@ -1,6 +1,7 @@ package de.jplag.reporting.jsonfactory.serializer; import java.io.IOException; +import java.io.Serial; import de.jplag.Language; @@ -10,6 +11,9 @@ public class LanguageSerializer extends StdSerializer { + @Serial + private static final long serialVersionUID = 5944655736767387268L; // generated + /** * Constructor used by the fasterxml.jackson */ diff --git a/core/src/main/java/de/jplag/reporting/reportobject/ReportObjectFactory.java b/core/src/main/java/de/jplag/reporting/reportobject/ReportObjectFactory.java index f8bc7c4dd..d569a5252 100644 --- a/core/src/main/java/de/jplag/reporting/reportobject/ReportObjectFactory.java +++ b/core/src/main/java/de/jplag/reporting/reportobject/ReportObjectFactory.java @@ -1,19 +1,13 @@ package de.jplag.reporting.reportobject; -import static de.jplag.reporting.jsonfactory.DirectoryManager.createDirectory; -import static de.jplag.reporting.jsonfactory.DirectoryManager.deleteDirectory; -import static de.jplag.reporting.jsonfactory.DirectoryManager.zipDirectory; import static de.jplag.reporting.reportobject.mapper.SubmissionNameToIdMapper.buildSubmissionNameToIdMap; import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; +import java.io.FileNotFoundException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -34,21 +28,19 @@ import de.jplag.reporting.reportobject.mapper.ClusteringResultMapper; import de.jplag.reporting.reportobject.mapper.MetricMapper; import de.jplag.reporting.reportobject.model.OverviewReport; +import de.jplag.reporting.reportobject.model.SubmissionFile; import de.jplag.reporting.reportobject.model.SubmissionFileIndex; import de.jplag.reporting.reportobject.model.Version; -import de.jplag.reporting.reportobject.writer.JsonWriter; -import de.jplag.reporting.reportobject.writer.TextWriter; +import de.jplag.reporting.reportobject.writer.JPlagResultWriter; +import de.jplag.reporting.reportobject.writer.ZipWriter; /** * Factory class, responsible for converting a JPlagResult object to Overview and Comparison DTO classes and writing it * to the disk. */ public class ReportObjectFactory { - private static final String DIRECTORY_ERROR = "Could not create directory {} for report viewer generation"; - private static final Logger logger = LoggerFactory.getLogger(ReportObjectFactory.class); - private static final JsonWriter jsonFileWriter = new JsonWriter(); public static final String OVERVIEW_FILE_NAME = "overview.json"; public static final String README_FILE_NAME = "README.txt"; @@ -56,49 +48,51 @@ public class ReportObjectFactory { private static final String[] README_CONTENT = new String[] {"This is a software plagiarism report generated by JPlag.", "To view the report go to https://jplag.github.io/JPlag/ and drag the generated zip file onto the page."}; - public static final String SUBMISSIONS_FOLDER = "files"; public static final String SUBMISSION_FILE_INDEX_FILE_NAME = "submissionFileIndex.json"; public static final Version REPORT_VIEWER_VERSION = JPlag.JPLAG_VERSION; + private static final String SUBMISSIONS_ROOT_PATH = "files/"; + private Map submissionNameToIdMap; private Function submissionToIdFunction; private Map> submissionNameToNameToComparisonFileName; + private final JPlagResultWriter resultWriter; + /** - * Creates all necessary report viewer files, writes them to the disk as zip. - * @param result The JPlagResult to be converted into a report. - * @param path The Path to save the report to + * Creates a new report object factory, that can be used to write a report. + * @param resultWriter The writer to use for writing report content */ - public void createAndSaveReport(JPlagResult result, String path) { - - try { - logger.info("Start writing report files..."); - createDirectory(path); - buildSubmissionToIdMap(result); + public ReportObjectFactory(JPlagResultWriter resultWriter) { + this.resultWriter = resultWriter; + } - copySubmissionFilesToReport(path, result); + /** + * Creates a new report object factory, that can be used to write a zip report. + * @param zipFile The zip file to write the report to + * @throws FileNotFoundException If the file cannot be opened for writing + */ + public ReportObjectFactory(File zipFile) throws FileNotFoundException { + this(new ZipWriter(zipFile)); + } - writeComparisons(result, path); - writeOverview(result, path); - writeSubmissionIndexFile(result, path); - writeReadMeFile(path); - writeOptionsFiles(result.getOptions(), path); + /** + * Creates all necessary report viewer files, writes them to the disk as zip. + * @param result The JPlagResult to be converted into a report. + */ + public void createAndSaveReport(JPlagResult result) { + logger.info("Start writing report..."); + buildSubmissionToIdMap(result); - logger.info("Zipping report files..."); - zipAndDelete(path); - } catch (IOException e) { - logger.error(DIRECTORY_ERROR, e, path); - } + copySubmissionFilesToReport(result); - } + writeComparisons(result); + writeOverview(result); + writeSubmissionIndexFile(result); + writeReadMeFile(); + writeOptionsFiles(result.getOptions()); - private void zipAndDelete(String path) { - boolean zipWasSuccessful = zipDirectory(path); - if (zipWasSuccessful) { - deleteDirectory(path); - } else { - logger.error("Could not zip results. The results are still available uncompressed at " + path); - } + this.resultWriter.close(); } private void buildSubmissionToIdMap(JPlagResult result) { @@ -106,61 +100,23 @@ private void buildSubmissionToIdMap(JPlagResult result) { submissionToIdFunction = (Submission submission) -> submissionNameToIdMap.get(submission.getName()); } - private void copySubmissionFilesToReport(String path, JPlagResult result) { - logger.info("Start copying submission files to the output directory..."); + private void copySubmissionFilesToReport(JPlagResult result) { + logger.info("Start to export results..."); List comparisons = result.getComparisons(result.getOptions().maximumNumberOfComparisons()); Set submissions = getSubmissions(comparisons); - File submissionsPath = createSubmissionsDirectory(path); - if (submissionsPath == null) { - return; - } Language language = result.getOptions().language(); for (Submission submission : submissions) { - File directory = createSubmissionDirectory(path, submissionsPath, submission); - File submissionRoot = submission.getRoot(); - if (directory == null) { - continue; - } + String submissionRootPath = SUBMISSIONS_ROOT_PATH + submissionToIdFunction.apply(submission); for (File file : submission.getFiles()) { - File fullPath = createSubmissionDirectory(path, submissionsPath, submission, file, submissionRoot); - File fileToCopy = getFileToCopy(language, file); - try { - if (fullPath != null) { - Files.copy(fileToCopy.toPath(), fullPath.toPath(), StandardCopyOption.REPLACE_EXISTING); - } else { - throw new NullPointerException("Could not create file with full path"); - } - } catch (IOException e) { - logger.error("Could not save submission file " + fileToCopy, e); + String relativeFilePath = file.getAbsolutePath().substring(submission.getRoot().getAbsolutePath().length()); + if (relativeFilePath.isEmpty()) { + relativeFilePath = file.getName(); } - } - } - } - - private File createSubmissionDirectory(String path, File submissionsPath, Submission submission, File file, File submissionRoot) { - try { - return createDirectory(submissionsPath.getPath(), submissionToIdFunction.apply(submission), file, submissionRoot); - } catch (IOException e) { - logger.error(DIRECTORY_ERROR, e, path); - return null; - } - } - - private File createSubmissionDirectory(String path, File submissionsPath, Submission submission) { - try { - return createDirectory(submissionsPath.getPath(), submissionToIdFunction.apply(submission)); - } catch (IOException e) { - logger.error(DIRECTORY_ERROR, e, path); - return null; - } - } + String zipPath = FilePathUtil.joinZipPathSegments(submissionRootPath, relativeFilePath); - private File createSubmissionsDirectory(String path) { - try { - return createDirectory(path, SUBMISSIONS_FOLDER); - } catch (IOException e) { - logger.error(DIRECTORY_ERROR, e, path); - return null; + File fileToCopy = getFileToCopy(language, file); + this.resultWriter.addFileContentEntry(zipPath, fileToCopy); + } } } @@ -168,13 +124,12 @@ private File getFileToCopy(Language language, File file) { return language.useViewFiles() ? new File(file.getPath() + language.viewFileSuffix()) : file; } - private void writeComparisons(JPlagResult result, String path) { - ComparisonReportWriter comparisonReportWriter = new ComparisonReportWriter(submissionToIdFunction, jsonFileWriter); - submissionNameToNameToComparisonFileName = comparisonReportWriter.writeComparisonReports(result, path); + private void writeComparisons(JPlagResult result) { + ComparisonReportWriter comparisonReportWriter = new ComparisonReportWriter(submissionToIdFunction, this.resultWriter); + submissionNameToNameToComparisonFileName = comparisonReportWriter.writeComparisonReports(result); } - private void writeOverview(JPlagResult result, String path) { - + private void writeOverview(JPlagResult result) { List folders = new ArrayList<>(); folders.addAll(result.getOptions().submissionDirectories()); folders.addAll(result.getOptions().oldSubmissionDirectories()); @@ -204,30 +159,34 @@ private void writeOverview(JPlagResult result, String path) { clusteringResultMapper.map(result), // clusters totalComparisons); // totalComparisons - jsonFileWriter.writeFile(overviewReport, path, OVERVIEW_FILE_NAME); + this.resultWriter.addJsonEntry(overviewReport, OVERVIEW_FILE_NAME); + } - private void writeReadMeFile(String path) { - new TextWriter().writeFile(String.join(System.lineSeparator(), README_CONTENT), path, README_FILE_NAME); + private void writeReadMeFile() { + this.resultWriter.writeStringEntry(String.join(System.lineSeparator(), README_CONTENT), README_FILE_NAME); } - private void writeSubmissionIndexFile(JPlagResult result, String path) { + private void writeSubmissionIndexFile(JPlagResult result) { List comparisons = result.getComparisons(result.getOptions().maximumNumberOfComparisons()); Set submissions = getSubmissions(comparisons); SubmissionFileIndex fileIndex = new SubmissionFileIndex(new HashMap<>()); - for (Submission submission : submissions) { - List filePaths = new LinkedList<>(); - for (File file : submission.getFiles()) { - filePaths.add(FilePathUtil.getRelativeSubmissionPath(file, submission, submissionToIdFunction)); + List>> submissionTokenCountList = submissions.stream().parallel().map(submission -> { + Map tokenCounts = new HashMap<>(); + for (Map.Entry entry : submission.getTokenCountPerFile().entrySet()) { + String key = FilePathUtil.getRelativeSubmissionPath(entry.getKey(), submission, submissionToIdFunction); + tokenCounts.put(key, new SubmissionFile(entry.getValue())); } - fileIndex.fileIndexes().put(submissionNameToIdMap.get(submission.getName()), filePaths); - } - jsonFileWriter.writeFile(fileIndex, path, SUBMISSION_FILE_INDEX_FILE_NAME); + return Map.of(submissionNameToIdMap.get(submission.getName()), tokenCounts); + }).toList(); + + submissionTokenCountList.forEach(submission -> fileIndex.fileIndexes().putAll(submission)); + this.resultWriter.addJsonEntry(fileIndex, SUBMISSION_FILE_INDEX_FILE_NAME); } - private void writeOptionsFiles(JPlagOptions options, String path) { - jsonFileWriter.writeFile(options, path, OPTIONS_FILE_NAME); + private void writeOptionsFiles(JPlagOptions options) { + resultWriter.addJsonEntry(options, OPTIONS_FILE_NAME); } private Set getSubmissions(List comparisons) { diff --git a/core/src/main/java/de/jplag/reporting/reportobject/model/ComparisonReport.java b/core/src/main/java/de/jplag/reporting/reportobject/model/ComparisonReport.java index 5b8d2fc4b..f207279dc 100644 --- a/core/src/main/java/de/jplag/reporting/reportobject/model/ComparisonReport.java +++ b/core/src/main/java/de/jplag/reporting/reportobject/model/ComparisonReport.java @@ -13,6 +13,7 @@ * @param matches the list of matches found in the comparison of the two submissions */ public record ComparisonReport(@JsonProperty("id1") String firstSubmissionId, @JsonProperty("id2") String secondSubmissionId, - @JsonProperty("similarities") Map similarities, @JsonProperty("matches") List matches) { + @JsonProperty("similarities") Map similarities, @JsonProperty("matches") List matches, + @JsonProperty("first_similarity") double firstSimilarity, @JsonProperty("second_similarity") double secondSimilarity) { } diff --git a/core/src/main/java/de/jplag/reporting/reportobject/model/Match.java b/core/src/main/java/de/jplag/reporting/reportobject/model/Match.java index 3f708dc1b..8af13af20 100644 --- a/core/src/main/java/de/jplag/reporting/reportobject/model/Match.java +++ b/core/src/main/java/de/jplag/reporting/reportobject/model/Match.java @@ -4,6 +4,5 @@ public record Match(@JsonProperty("file1") String firstFileName, @JsonProperty("file2") String secondFileName, @JsonProperty("start1") int startInFirst, @JsonProperty("end1") int endInFirst, @JsonProperty("start2") int startInSecond, - @JsonProperty("end2") int endInSecond, @JsonProperty("tokens") int tokens, @JsonProperty("file1Tokens") long file1Tokens, - @JsonProperty("file2Tokens") long file2Tokens) { + @JsonProperty("end2") int endInSecond, @JsonProperty("tokens") int tokens) { } diff --git a/core/src/main/java/de/jplag/reporting/reportobject/model/SubmissionFile.java b/core/src/main/java/de/jplag/reporting/reportobject/model/SubmissionFile.java new file mode 100644 index 000000000..33754b2a4 --- /dev/null +++ b/core/src/main/java/de/jplag/reporting/reportobject/model/SubmissionFile.java @@ -0,0 +1,6 @@ +package de.jplag.reporting.reportobject.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record SubmissionFile(@JsonProperty("token_count") int tokenCount) { +} diff --git a/core/src/main/java/de/jplag/reporting/reportobject/model/SubmissionFileIndex.java b/core/src/main/java/de/jplag/reporting/reportobject/model/SubmissionFileIndex.java index 828508ad9..f33589044 100644 --- a/core/src/main/java/de/jplag/reporting/reportobject/model/SubmissionFileIndex.java +++ b/core/src/main/java/de/jplag/reporting/reportobject/model/SubmissionFileIndex.java @@ -1,9 +1,8 @@ package de.jplag.reporting.reportobject.model; -import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonProperty; -public record SubmissionFileIndex(@JsonProperty("submission_file_indexes") Map> fileIndexes) { +public record SubmissionFileIndex(@JsonProperty("submission_file_indexes") Map> fileIndexes) { } diff --git a/core/src/main/java/de/jplag/reporting/reportobject/writer/DummyResultWriter.java b/core/src/main/java/de/jplag/reporting/reportobject/writer/DummyResultWriter.java new file mode 100644 index 000000000..1da95b72d --- /dev/null +++ b/core/src/main/java/de/jplag/reporting/reportobject/writer/DummyResultWriter.java @@ -0,0 +1,37 @@ +package de.jplag.reporting.reportobject.writer; + +import java.io.File; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Dummy writer, that does nothing + */ +public class DummyResultWriter implements JPlagResultWriter { + private static final Logger logger = LoggerFactory.getLogger(DummyResultWriter.class); + private static final String MESSAGE_JSON = "DummyWriter writes object {} to path {} as JSON."; + private static final String MESSAGE_FILE = "DummyWriter writes file {} to path {}."; + private static final String MESSAGE_STRING = "DummyWriter writes String ({}) to path {}."; + private static final String MESSAGE_CLOSE = "DummyWriter closed."; + + @Override + public void addJsonEntry(Object jsonContent, String path) { + logger.info(MESSAGE_JSON, jsonContent, path); + } + + @Override + public void addFileContentEntry(String path, File original) { + logger.info(MESSAGE_FILE, original.getAbsolutePath(), path); + } + + @Override + public void writeStringEntry(String entry, String path) { + logger.info(MESSAGE_STRING, entry, path); + } + + @Override + public void close() { + logger.info(MESSAGE_CLOSE); + } +} diff --git a/core/src/main/java/de/jplag/reporting/reportobject/writer/DummyWriter.java b/core/src/main/java/de/jplag/reporting/reportobject/writer/DummyWriter.java deleted file mode 100644 index c344802dc..000000000 --- a/core/src/main/java/de/jplag/reporting/reportobject/writer/DummyWriter.java +++ /dev/null @@ -1,17 +0,0 @@ -package de.jplag.reporting.reportobject.writer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This writer is used as a mock for testing purposes only. - */ -public class DummyWriter extends JsonWriter { - private static final Logger logger = LoggerFactory.getLogger(DummyWriter.class); - private static final String MESSAGE = "DummyWriter writes object {} to path {} with name {} as JSON."; - - @Override - public void writeFile(Object fileToSave, String folderPath, String fileName) { - logger.info(MESSAGE, fileToSave, folderPath, fileName); - } -} diff --git a/core/src/main/java/de/jplag/reporting/reportobject/writer/FileWriter.java b/core/src/main/java/de/jplag/reporting/reportobject/writer/FileWriter.java deleted file mode 100644 index ec8888a49..000000000 --- a/core/src/main/java/de/jplag/reporting/reportobject/writer/FileWriter.java +++ /dev/null @@ -1,16 +0,0 @@ -package de.jplag.reporting.reportobject.writer; - -/** - * Responsible for writing a specific file type to the disk. - * @param Object that the FileWriter writes. - */ -public interface FileWriter { - - /** - * Saves the provided object to the provided path under the provided name - * @param fileContent The object to save - * @param folderPath The path to save the object to - * @param fileName The name to save the object under - */ - void writeFile(T fileContent, String folderPath, String fileName); -} diff --git a/core/src/main/java/de/jplag/reporting/reportobject/writer/JPlagResultWriter.java b/core/src/main/java/de/jplag/reporting/reportobject/writer/JPlagResultWriter.java new file mode 100644 index 000000000..57fbed8c4 --- /dev/null +++ b/core/src/main/java/de/jplag/reporting/reportobject/writer/JPlagResultWriter.java @@ -0,0 +1,34 @@ +package de.jplag.reporting.reportobject.writer; + +import java.io.File; + +/** + * Writer for JPlag result data. The way paths are resolved depends on the implementation + */ +public interface JPlagResultWriter { + /** + * Writes data as json + * @param jsonContent The json content + * @param path The path to write to + */ + void addJsonEntry(Object jsonContent, String path); + + /** + * Writes data from a file + * @param path The path to write to + * @param original The original file + */ + void addFileContentEntry(String path, File original); + + /** + * Writes data from a string + * @param entry The string to write + * @param path The path to write to + */ + void writeStringEntry(String entry, String path); + + /** + * Closes the writer + */ + void close(); +} diff --git a/core/src/main/java/de/jplag/reporting/reportobject/writer/JsonWriter.java b/core/src/main/java/de/jplag/reporting/reportobject/writer/JsonWriter.java deleted file mode 100644 index 723718cf8..000000000 --- a/core/src/main/java/de/jplag/reporting/reportobject/writer/JsonWriter.java +++ /dev/null @@ -1,29 +0,0 @@ -package de.jplag.reporting.reportobject.writer; - -import java.io.IOException; -import java.nio.file.Path; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; - -/** - * Writes an object with {@link com.fasterxml.jackson.annotation.JsonProperty}s to the disk. - */ -public class JsonWriter implements FileWriter { - private static final Logger logger = LoggerFactory.getLogger(JsonWriter.class); - private static final ObjectMapper objectMapper = new ObjectMapper(); - private static final String WRITE_ERROR = "Failed to write JSON file {}"; - - @Override - public void writeFile(Object fileToSave, String folderPath, String fileName) { - Path path = Path.of(folderPath, fileName); - try { - objectMapper.writeValue(path.toFile(), fileToSave); - } catch (IOException e) { - logger.error(WRITE_ERROR, e, path); - } - } - -} diff --git a/core/src/main/java/de/jplag/reporting/reportobject/writer/TextWriter.java b/core/src/main/java/de/jplag/reporting/reportobject/writer/TextWriter.java deleted file mode 100644 index d4b3601d4..000000000 --- a/core/src/main/java/de/jplag/reporting/reportobject/writer/TextWriter.java +++ /dev/null @@ -1,27 +0,0 @@ -package de.jplag.reporting.reportobject.writer; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.nio.file.Path; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Writes plain text to a file. - */ -public class TextWriter implements FileWriter { - - private static final Logger logger = LoggerFactory.getLogger(TextWriter.class); - private static final String WRITE_ERROR = "Failed to write text file {}"; - - @Override - public void writeFile(String fileContent, String folderPath, String fileName) { - String path = Path.of(folderPath, fileName).toString(); - try (BufferedWriter writer = new BufferedWriter(new java.io.FileWriter(path))) { - writer.write(fileContent); - } catch (IOException e) { - logger.error(WRITE_ERROR, e, path); - } - } -} diff --git a/core/src/main/java/de/jplag/reporting/reportobject/writer/ZipWriter.java b/core/src/main/java/de/jplag/reporting/reportobject/writer/ZipWriter.java new file mode 100644 index 000000000..b7606c535 --- /dev/null +++ b/core/src/main/java/de/jplag/reporting/reportobject/writer/ZipWriter.java @@ -0,0 +1,80 @@ +package de.jplag.reporting.reportobject.writer; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Writes JPlag result data as a zip + */ +public class ZipWriter implements JPlagResultWriter { + private static final Logger logger = LoggerFactory.getLogger(ZipWriter.class); + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private static final String WRITE_JSON_ERROR = "Failed to write JSON entry %s"; + private static final String COPY_FILE_ERROR = "Failed to copy file (%s) to entry (%s)"; + private static final String WRITE_STRING_ERROR = "Failed to write string entry %s"; + private static final String CLOSE_FILE_ERROR = "Failed to close zip file properly"; + + private final ZipOutputStream file; + + /** + * The zip file to write to + * @param zipFile The file + * @throws FileNotFoundException If the file cannot be opened for writing + */ + public ZipWriter(File zipFile) throws FileNotFoundException { + zipFile.getAbsoluteFile().getParentFile().mkdirs(); + this.file = new ZipOutputStream(new FileOutputStream(zipFile)); + } + + @Override + public void addJsonEntry(Object jsonContent, String path) { + try { + this.file.putNextEntry(new ZipEntry(path)); + this.file.write(objectMapper.writeValueAsBytes(jsonContent)); + this.file.closeEntry(); + } catch (IOException e) { + logger.error(String.format(WRITE_JSON_ERROR, path), e); + } + } + + @Override + public void addFileContentEntry(String path, File original) { + try (FileInputStream inputStream = new FileInputStream(original)) { + this.file.putNextEntry(new ZipEntry(path)); + inputStream.transferTo(this.file); + } catch (IOException e) { + logger.error(String.format(COPY_FILE_ERROR, original.getAbsolutePath(), path), e); + } + } + + @Override + public void writeStringEntry(String entry, String path) { + try { + this.file.putNextEntry(new ZipEntry(path)); + this.file.write(entry.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + logger.error(String.format(WRITE_STRING_ERROR, path), e); + } + } + + @Override + public void close() { + try { + this.file.close(); + } catch (IOException e) { + logger.error(CLOSE_FILE_ERROR, e); + } + } +} diff --git a/core/src/main/java/de/jplag/strategy/AbstractComparisonStrategy.java b/core/src/main/java/de/jplag/strategy/AbstractComparisonStrategy.java index 19822ef41..29809df31 100644 --- a/core/src/main/java/de/jplag/strategy/AbstractComparisonStrategy.java +++ b/core/src/main/java/de/jplag/strategy/AbstractComparisonStrategy.java @@ -3,14 +3,19 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import de.jplag.GreedyStringTiling; import de.jplag.JPlagComparison; +import de.jplag.JPlagResult; import de.jplag.Submission; import de.jplag.SubmissionSet; +import de.jplag.logging.ProgressBar; +import de.jplag.logging.ProgressBarLogger; +import de.jplag.logging.ProgressBarType; import de.jplag.options.JPlagOptions; public abstract class AbstractComparisonStrategy implements ComparisonStrategy { @@ -19,7 +24,7 @@ public abstract class AbstractComparisonStrategy implements ComparisonStrategy { private final GreedyStringTiling greedyStringTiling; - protected final JPlagOptions options; + private final JPlagOptions options; protected AbstractComparisonStrategy(JPlagOptions options, GreedyStringTiling greedyStringTiling) { this.greedyStringTiling = greedyStringTiling; @@ -46,7 +51,7 @@ protected void compareSubmissionsToBaseCode(SubmissionSet submissionSet) { */ protected Optional compareSubmissions(Submission first, Submission second) { JPlagComparison comparison = greedyStringTiling.compare(first, second); - logger.info("Comparing {}-{}: {}", first.getName(), second.getName(), comparison.similarity()); + logger.trace("Comparing {}-{}: {}", first.getName(), second.getName(), comparison.similarity()); if (options.similarityMetric().isAboveThreshold(comparison, options.similarityThreshold())) { return Optional.of(comparison); @@ -57,7 +62,7 @@ protected Optional compareSubmissions(Submission first, Submiss /** * @return a list of all submission tuples to be processed. */ - protected static List buildComparisonTuples(List submissions) { + private List buildComparisonTuples(List submissions) { List tuples = new ArrayList<>(); List validSubmissions = submissions.stream().filter(s -> s.getTokenList() != null).toList(); @@ -72,4 +77,44 @@ protected static List buildComparisonTuples(List su } return tuples; } + + @Override + public JPlagResult compareSubmissions(SubmissionSet submissionSet) { + long timeBeforeStartInMillis = System.currentTimeMillis(); + + handleBaseCode(submissionSet); + + List tuples = buildComparisonTuples(submissionSet.getSubmissions()); + ProgressBar progressBar = ProgressBarLogger.createProgressBar(ProgressBarType.COMPARING, tuples.size()); + List comparisons = prepareStream(tuples).flatMap(tuple -> { + Optional result = compareTuple(tuple); + progressBar.step(); + return result.stream(); + }).toList(); + progressBar.dispose(); + + long durationInMillis = System.currentTimeMillis() - timeBeforeStartInMillis; + + return new JPlagResult(comparisons, submissionSet, durationInMillis, options); + } + + /** + * Handle the parsing of the base code. + * @param submissionSet The submission set to parse + */ + protected abstract void handleBaseCode(SubmissionSet submissionSet); + + /** + * Prepare a stream for parsing the tuples. Here you can modify the tuples or the stream as necessary. + * @param tuples The tuples to stream + * @return The Stream of tuples + */ + protected abstract Stream prepareStream(List tuples); + + /** + * Compares a single tuple. Returns nothing, if the similarity is not high enough. + * @param tuple The Tuple to compare + * @return The comparison + */ + protected abstract Optional compareTuple(SubmissionTuple tuple); } diff --git a/core/src/main/java/de/jplag/strategy/ParallelComparisonStrategy.java b/core/src/main/java/de/jplag/strategy/ParallelComparisonStrategy.java index fd94b9293..43cc66ae6 100644 --- a/core/src/main/java/de/jplag/strategy/ParallelComparisonStrategy.java +++ b/core/src/main/java/de/jplag/strategy/ParallelComparisonStrategy.java @@ -2,10 +2,10 @@ import java.util.List; import java.util.Optional; +import java.util.stream.Stream; import de.jplag.GreedyStringTiling; import de.jplag.JPlagComparison; -import de.jplag.JPlagResult; import de.jplag.SubmissionSet; import de.jplag.options.JPlagOptions; @@ -19,19 +19,20 @@ public ParallelComparisonStrategy(JPlagOptions options, GreedyStringTiling greed } @Override - public JPlagResult compareSubmissions(SubmissionSet submissionSet) { - // Initialize: - long timeBeforeStartInMillis = System.currentTimeMillis(); + protected void handleBaseCode(SubmissionSet submissionSet) { boolean withBaseCode = submissionSet.hasBaseCode(); if (withBaseCode) { compareSubmissionsToBaseCode(submissionSet); } + } - List tuples = buildComparisonTuples(submissionSet.getSubmissions()); - List comparisons = tuples.stream().parallel().map(tuple -> compareSubmissions(tuple.left(), tuple.right())) - .flatMap(Optional::stream).toList(); + @Override + protected Stream prepareStream(List tuples) { + return tuples.stream().parallel(); + } - long durationInMillis = System.currentTimeMillis() - timeBeforeStartInMillis; - return new JPlagResult(comparisons, submissionSet, durationInMillis, options); + @Override + protected Optional compareTuple(SubmissionTuple tuple) { + return compareSubmissions(tuple.left(), tuple.right()); } } diff --git a/core/src/test/java/de/jplag/NormalizationTest.java b/core/src/test/java/de/jplag/NormalizationTest.java index f2e447b1c..c6a9db9ed 100644 --- a/core/src/test/java/de/jplag/NormalizationTest.java +++ b/core/src/test/java/de/jplag/NormalizationTest.java @@ -12,8 +12,8 @@ import de.jplag.options.JPlagOptions; class NormalizationTest extends TestBase { - private Map> tokenStringMap; - private List originalTokenString; + private final Map> tokenStringMap; + private final List originalTokenString; NormalizationTest() throws ExitException { JPlagOptions options = getDefaultOptions("normalization"); diff --git a/core/src/test/java/de/jplag/merging/MergingTest.java b/core/src/test/java/de/jplag/merging/MergingTest.java index 0d6780b9f..e2a606e16 100644 --- a/core/src/test/java/de/jplag/merging/MergingTest.java +++ b/core/src/test/java/de/jplag/merging/MergingTest.java @@ -4,7 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; -import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.function.Function; @@ -33,15 +33,14 @@ * CC BY 4.0 license. */ class MergingTest extends TestBase { - private JPlagOptions options; - private JPlagResult result; + private final JPlagOptions options; private List matches; private List comparisonsBefore; private List comparisonsAfter; - private ComparisonStrategy comparisonStrategy; - private SubmissionSet submissionSet; - private final int MINIMUM_NEIGHBOR_LENGTH = 1; - private final int MAXIMUM_GAP_SIZE = 10; + private final ComparisonStrategy comparisonStrategy; + private final SubmissionSet submissionSet; + private static final int MINIMUM_NEIGHBOR_LENGTH = 1; + private static final int MAXIMUM_GAP_SIZE = 10; MergingTest() throws ExitException { options = getDefaultOptions("merging").withMergingOptions(new MergingOptions(true, MINIMUM_NEIGHBOR_LENGTH, MAXIMUM_GAP_SIZE)); @@ -55,13 +54,16 @@ class MergingTest extends TestBase { @BeforeEach void prepareTestState() { - result = comparisonStrategy.compareSubmissions(submissionSet); - comparisonsBefore = result.getAllComparisons(); + JPlagResult result = comparisonStrategy.compareSubmissions(submissionSet); + comparisonsBefore = new ArrayList<>(result.getAllComparisons()); if (options.mergingOptions().enabled()) { result = new MatchMerging(options).mergeMatchesOf(result); } - comparisonsAfter = result.getAllComparisons(); + comparisonsAfter = new ArrayList<>(result.getAllComparisons()); + + comparisonsBefore.sort(Comparator.comparing(Object::toString)); + comparisonsAfter.sort(Comparator.comparing(Object::toString)); } @Test @@ -83,10 +85,10 @@ void testGSTIgnoredMatches() { } private void checkMatchLength(Function> matchFunction, int threshold, List comparisons) { - for (int i = 0; i < comparisons.size(); i++) { - matches = matchFunction.apply(comparisons.get(i)); - for (int j = 0; j < matches.size(); j++) { - assertTrue(matches.get(j).length() >= threshold); + for (JPlagComparison comparison : comparisons) { + matches = matchFunction.apply(comparison); + for (Match match : matches) { + assertTrue(match.length() >= threshold); } } } @@ -169,11 +171,11 @@ void testCorrectMerges() { matches = comparisonsAfter.get(i).matches(); List sortedByFirst = new ArrayList<>(comparisonsBefore.get(i).matches()); sortedByFirst.addAll(comparisonsBefore.get(i).ignoredMatches()); - Collections.sort(sortedByFirst, (m1, m2) -> m1.startOfFirst() - m2.startOfFirst()); - for (int j = 0; j < matches.size(); j++) { + sortedByFirst.sort(Comparator.comparingInt(Match::startOfFirst)); + for (Match match : matches) { int begin = -1; for (int k = 0; k < sortedByFirst.size(); k++) { - if (sortedByFirst.get(k).startOfFirst() == matches.get(j).startOfFirst()) { + if (sortedByFirst.get(k).startOfFirst() == match.startOfFirst()) { begin = k; break; } @@ -182,10 +184,10 @@ void testCorrectMerges() { correctMerges = false; } else { int foundToken = 0; - while (foundToken < matches.get(j).length()) { + while (foundToken < match.length()) { foundToken += sortedByFirst.get(begin).length(); begin++; - if (foundToken > matches.get(j).length()) { + if (foundToken > match.length()) { correctMerges = false; } } @@ -194,4 +196,36 @@ void testCorrectMerges() { } assertTrue(correctMerges); } + + @Test + @DisplayName("Sanity check for match merging") + void testSanity() { + + List matchesBefore = new ArrayList<>(); + List matchesAfter = new ArrayList<>(); + + for (JPlagComparison comparison : comparisonsBefore) { + if (comparison.toString().equals("sanityA.java <-> sanityB.java")) { + matchesBefore = comparison.ignoredMatches(); + } + } + for (JPlagComparison comparison : comparisonsAfter) { + if (comparison.toString().equals("sanityA.java <-> sanityB.java")) { + matchesAfter = comparison.matches(); + } + } + + List expectedBefore = new ArrayList<>(); + expectedBefore.add(new Match(5, 3, 6)); + expectedBefore.add(new Match(11, 12, 6)); + expectedBefore.add(new Match(0, 0, 3)); + expectedBefore.add(new Match(3, 18, 2)); + expectedBefore.add(new Match(17, 20, 2)); + + List expectedAfter = new ArrayList<>(); + expectedAfter.add(new Match(5, 3, 12)); + + assertEquals(expectedBefore, matchesBefore); + assertEquals(expectedAfter, matchesAfter); + } } \ No newline at end of file diff --git a/core/src/test/java/de/jplag/reporting/FilePathUtilTest.java b/core/src/test/java/de/jplag/reporting/FilePathUtilTest.java new file mode 100644 index 000000000..661d7cc0b --- /dev/null +++ b/core/src/test/java/de/jplag/reporting/FilePathUtilTest.java @@ -0,0 +1,26 @@ +package de.jplag.reporting; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class FilePathUtilTest { + private static final String JOINED = "left/right"; + private static final String LEFT = "left"; + private static final String RIGHT = "right"; + + @Test + void testJoinPath() { + assertEquals(JOINED, FilePathUtil.joinZipPathSegments(LEFT, RIGHT)); + } + + @Test + void testJoinPathWithLeftSlashSuffix() { + assertEquals(JOINED, FilePathUtil.joinZipPathSegments(LEFT + "/", RIGHT)); + } + + @Test + void testJoinPathWithRightSlashSuffix() { + assertEquals(JOINED, FilePathUtil.joinZipPathSegments(LEFT, "/" + RIGHT)); + } +} \ No newline at end of file diff --git a/core/src/test/java/de/jplag/reporting/jsonfactory/DirectoryManagerTest.java b/core/src/test/java/de/jplag/reporting/jsonfactory/DirectoryManagerTest.java deleted file mode 100644 index f95b34932..000000000 --- a/core/src/test/java/de/jplag/reporting/jsonfactory/DirectoryManagerTest.java +++ /dev/null @@ -1,70 +0,0 @@ -package de.jplag.reporting.jsonfactory; - -import static org.junit.jupiter.api.Assertions.fail; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import de.jplag.TestBase; - -/** - * Test for the directory manager that persists the results for the report viewer. - */ -class DirectoryManagerTest extends TestBase { - private static final Path OUTPUT_PATH = Path.of(BASE_PATH, "output", "submissions"); - - private static final String SUBMISSION_1 = "A"; - private static final String FILE_PATH_1 = "TerrainType.java"; - private static final String ROOT_1 = "basecode"; - - private static final String SUBMISSION_2 = "Submission1.java"; - private static final String ROOT_2 = "FilesAsSubmissions"; - - private static final String SUBMISSION_3 = "A"; - private static final Path FILE_PATH_3 = Path.of("B", "A", "TerrainType.java"); - private static final String ROOT_3 = "basecode-sameNameOfSubdirectoryAndRootdirectory"; - - @Test - @DisplayName("test normal submission with file in folder") - void testCreateDirectoryBasecode() throws IOException { - testDirectoryManager(ROOT_1, SUBMISSION_1, FILE_PATH_1); - } - - @Test - @DisplayName("test single file as submission") - void testCreateDirectoryFileAsSubmission() throws IOException { - testDirectoryManager(ROOT_2, SUBMISSION_2, ""); - } - - @Test - @DisplayName("test same name of subdirectory and root directory") - void testCreateDirectorySharedName() throws IOException { - testDirectoryManager(ROOT_3, SUBMISSION_3, FILE_PATH_3.toString()); - } - - /** - * Test the directory manager for a given scenario. - * @param rootName is the name of the root folder. - * @param submissionName is the name of the submission. - * @param filePath is the path to the file relative to the submission. Empty for single file submissions. - */ - private static void testDirectoryManager(String rootName, String submissionName, String filePath) { - File submissionPath = Path.of(BASE_PATH, rootName, submissionName).toFile(); - File fullFilePath = new File(submissionPath, filePath); - File expectation = new File(OUTPUT_PATH.toFile(), Path.of(submissionName, filePath.isEmpty() ? submissionName : filePath).toString()); - try { - File directory = DirectoryManager.createDirectory(OUTPUT_PATH.toString(), submissionName, fullFilePath, submissionPath); - Assertions.assertNotNull(directory); - Assertions.assertEquals(expectation.getPath(), directory.getPath()); - } catch (IOException e) { - fail("Directory manager threw an exception:", e); - } finally { - deleteDirectory(expectation); - } - } -} diff --git a/core/src/test/java/de/jplag/reporting/reportobject/ReportObjectFactoryTest.java b/core/src/test/java/de/jplag/reporting/reportobject/ReportObjectFactoryTest.java index 19e4ee05c..36a7f610d 100644 --- a/core/src/test/java/de/jplag/reporting/reportobject/ReportObjectFactoryTest.java +++ b/core/src/test/java/de/jplag/reporting/reportobject/ReportObjectFactoryTest.java @@ -4,7 +4,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; -import java.nio.file.Path; +import java.io.IOException; +import java.io.RandomAccessFile; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -15,11 +16,8 @@ import de.jplag.reporting.reportobject.model.Version; class ReportObjectFactoryTest extends TestBase { - private static final String FILE_SUFFIX = ".zip"; private static final String BASECODE = "basecode"; private static final String BASECODE_BASE = "basecode-base"; - private static final String OUTPUT = "output"; - private static final String SUBMISSIONS = "submissions"; @Test void testVersionLoading() { @@ -28,15 +26,28 @@ void testVersionLoading() { } @Test - void testCreateAndSaveReportWithBasecode() throws ExitException { + void testCreateAndSaveReportWithBasecode() throws ExitException, IOException { JPlagResult result = runJPlag(BASECODE, it -> it.withBaseCodeSubmissionDirectory(new File(BASE_PATH, BASECODE_BASE))); - Path path = Path.of(BASE_PATH, OUTPUT, SUBMISSIONS); - ReportObjectFactory reportObjectFactory = new ReportObjectFactory(); - reportObjectFactory.createAndSaveReport(result, path.toString()); + File testZip = File.createTempFile("result", ".zip"); + + ReportObjectFactory reportObjectFactory = new ReportObjectFactory(testZip); + reportObjectFactory.createAndSaveReport(result); + assertNotNull(result); - File expectedFile = new File(path.toString() + FILE_SUFFIX); - assertTrue(expectedFile.exists()); - expectedFile.delete(); + assertTrue(isArchive(testZip)); + } + + /** + * Checks if the given file is a valid archive + * @param file The file to check + * @return True, if file is an archive + */ + private static boolean isArchive(File file) throws IOException { + int fileSignature = 0; + try (RandomAccessFile randomAccessFile = new RandomAccessFile(file, "r")) { + fileSignature = randomAccessFile.readInt(); + } + return fileSignature == 0x504B0304 || fileSignature == 0x504B0506 || fileSignature == 0x504B0708; } } \ No newline at end of file diff --git a/core/src/test/java/de/jplag/reporting/reportobject/mapper/ComparisonReportWriterTest.java b/core/src/test/java/de/jplag/reporting/reportobject/mapper/ComparisonReportWriterTest.java index 02af18f52..72ebe4b16 100644 --- a/core/src/test/java/de/jplag/reporting/reportobject/mapper/ComparisonReportWriterTest.java +++ b/core/src/test/java/de/jplag/reporting/reportobject/mapper/ComparisonReportWriterTest.java @@ -10,18 +10,18 @@ import de.jplag.TestBase; import de.jplag.exceptions.ExitException; import de.jplag.reporting.jsonfactory.ComparisonReportWriter; -import de.jplag.reporting.reportobject.writer.DummyWriter; -import de.jplag.reporting.reportobject.writer.JsonWriter; +import de.jplag.reporting.reportobject.writer.DummyResultWriter; +import de.jplag.reporting.reportobject.writer.JPlagResultWriter; public class ComparisonReportWriterTest extends TestBase { - private final JsonWriter fileWriter = new DummyWriter(); + private final JPlagResultWriter fileWriter = new DummyResultWriter(); @Test public void firsLevelOfLookupMapComplete() throws ExitException { JPlagResult result = runJPlagWithDefaultOptions("PartialPlagiarism"); var mapper = new ComparisonReportWriter(Submission::getName, fileWriter); - Map> stringMapMap = mapper.writeComparisonReports(result, ""); + Map> stringMapMap = mapper.writeComparisonReports(result); firstLevelOfMapContains(stringMapMap, "A", "B", "C", "D", "E"); } @@ -31,7 +31,7 @@ public void secondLevelOfLookupMapComplete() throws ExitException { JPlagResult result = runJPlagWithDefaultOptions("PartialPlagiarism"); var mapper = new ComparisonReportWriter(Submission::getName, fileWriter); - Map> stringMapMap = mapper.writeComparisonReports(result, ""); + Map> stringMapMap = mapper.writeComparisonReports(result); secondLevelOfMapContains(stringMapMap, "A", "B", "C", "D", "E"); secondLevelOfMapContains(stringMapMap, "B", "A", "C", "D", "E"); diff --git a/core/src/test/java/de/jplag/special/ReadmeCodeExampleTest.java b/core/src/test/java/de/jplag/special/ReadmeCodeExampleTest.java index 7c4507fda..4127979ea 100644 --- a/core/src/test/java/de/jplag/special/ReadmeCodeExampleTest.java +++ b/core/src/test/java/de/jplag/special/ReadmeCodeExampleTest.java @@ -1,6 +1,7 @@ package de.jplag.special; import java.io.File; +import java.io.FileNotFoundException; import java.util.Set; import org.junit.jupiter.api.Disabled; @@ -35,10 +36,12 @@ void testReadmeCodeExample() { JPlagResult result = JPlag.run(options); // Optional - ReportObjectFactory reportObjectFactory = new ReportObjectFactory(); - reportObjectFactory.createAndSaveReport(result, "/path/to/output"); + ReportObjectFactory reportObjectFactory = new ReportObjectFactory(new File("/path/to/output")); + reportObjectFactory.createAndSaveReport(result); } catch (ExitException e) { // error handling here + } catch (FileNotFoundException e) { + // handle IO exception here } } } diff --git a/core/src/test/resources/de/jplag/samples/merging/sanityA.java b/core/src/test/resources/de/jplag/samples/merging/sanityA.java new file mode 100644 index 000000000..d4a042dc6 --- /dev/null +++ b/core/src/test/resources/de/jplag/samples/merging/sanityA.java @@ -0,0 +1,18 @@ +public class Minimal { + public static void main (String [] Argv) { + System.out.println("Test"); + System.out.println("Test"); + int a = 1; + a = 1; + int b = 1; + a = 1; + int c = 1; + a = 1; + int d = 1; + a = 1; + int e = 1; + a = 1; + int f = 1; + a = 1; + } +} \ No newline at end of file diff --git a/core/src/test/resources/de/jplag/samples/merging/sanityB.java b/core/src/test/resources/de/jplag/samples/merging/sanityB.java new file mode 100644 index 000000000..75c55752a --- /dev/null +++ b/core/src/test/resources/de/jplag/samples/merging/sanityB.java @@ -0,0 +1,21 @@ +public class Minimal { + public static void main (String [] Argv) { + int a = 1; + a = 1; + int b = 1; + a = 1; + int c = 1; + a = 1; + if(a==1){ + a = 2; + } + int d = 1; + a = 1; + int e = 1; + a = 1; + int f = 1; + a = 1; + System.out.println("Test"); + System.out.println("Test"); + } +} \ No newline at end of file diff --git a/coverage-report/pom.xml b/coverage-report/pom.xml index 3d918dd10..71f01179a 100644 --- a/coverage-report/pom.xml +++ b/coverage-report/pom.xml @@ -63,12 +63,12 @@ de.jplag - cpp + c ${revision} de.jplag - cpp2 + cpp ${revision} diff --git a/docs/1.-How-to-Use-JPlag.md b/docs/1.-How-to-Use-JPlag.md index a3cdf37e6..ac231b6f4 100644 --- a/docs/1.-How-to-Use-JPlag.md +++ b/docs/1.-How-to-Use-JPlag.md @@ -11,81 +11,58 @@ A list of language specific options can be obtained by requesting the help page The following arguments can be used to control JPlag: ``` -Usage: jplag [OPTIONS] [root-dirs[,root-dirs...]...] [COMMAND] - +Parameter descriptions: [root-dirs[,root-dirs...]...] - Root-directory with submissions to check for plagiarism - + Root-directory with submissions to check for plagiarism. -bc, --bc, --base-code= - Path of the directory containing the base code - (common framework used in all submissions) - - -h, --help display this help and exit - -l, --language= - Select the language to parse the submissions (default: - java). The language names are the same as the - subcommands. - - -n, --shown-comparisons= - The maximum number of comparisons that will be shown - in the generated report, if set to -1 all comparisons - will be shown (default: 100) - + Path to the base code directory (common framework used in all submissions). + -l, --language= + Select the language of the submissions (default: java). See subcommands below. + -M, --mode=<{RUN, VIEW, RUN_AND_VIEW}> + The mode of JPlag: either only run analysis, only open the viewer, or do both (default: null) + -n, --shown-comparisons= + The maximum number of comparisons that will be shown in the generated report, if set to -1 all comparisons will be shown (default: 500) -new, --new=[,...] - Root-directory with submissions to check for plagiarism - (same as the root directory) - + Root-directories with submissions to check for plagiarism (same as root). + --normalize Activate the normalization of tokens. Supported for languages: Java, C++. -old, --old=[,...] - Root-directory with prior submissions to compare against - - -r, --result-directory= - Name of the directory in which the comparison results - will be stored (default: result) - - -t, --min-tokens= - Tunes the comparison sensitivity by adjusting the - minimum token required to be counted as a matching - section. A smaller increases the sensitivity but - might lead to more false-positives + Root-directories with prior submissions to compare against. + -r, --result-file= + Name of the file in which the comparison results will be stored (default: results). Missing .zip endings will be automatically added. + -t, --min-tokens= + Tunes the comparison sensitivity by adjusting the minimum token required to be counted as a matching section. A smaller value increases the sensitivity but might lead to more + false-positives. Advanced - -d, --debug Debug parser. Non-parsable files will be stored - (default: false) - - -m, --similarity-threshold= - Comparison similarity threshold [0.0-1.0]: All - comparisons above this threshold will be saved - (default: 0.0) - - -p, --suffixes=[,...] - comma-separated list of all filename suffixes that are - included - - -s, --subdirectory= - Look in directories /*/ for programs - - -x, --exclusion-file= - All files named in this file will be ignored in the - comparison (line-separated list) + --csv-export Export pairwise similarity values as a CSV file. + -d, --debug Store on-parsable files in error folder. + -m, --similarity-threshold= + Comparison similarity threshold [0.0-1.0]: All comparisons above this threshold will be saved (default: 0.0). + -p, --suffixes=[,...] + comma-separated list of all filename suffixes that are included. + -P, --port= The port used for the internal report viewer (default: 1996). + -s, --subdirectory= + Look in directories /*/ for programs. + -x, --exclusion-file= + All files named in this file will be ignored in the comparison (line-separated list). Clustering - --cluster-alg, --cluster-algorithm= - Which clustering algorithm to use. Agglomerative merges - similar submissions bottom up. Spectral clustering is - combined with Bayesian Optimization to execute - the k-Means clustering algorithm multiple times, - hopefully finding a "good" clustering - automatically. (default: spectral) - - --cluster-metric= - The metric used for clustering. AVG is intersection - over union, MAX can expose some attempts of - obfuscation. (default: MAX) - - --cluster-skip Skips the clustering (default: false) -Commands: + --cluster-alg, --cluster-algorithm=<{AGGLOMERATIVE, SPECTRAL}> + Specifies the clustering algorithm (default: spectral). + --cluster-metric=<{AVG, MIN, MAX, INTERSECTION}> + The similarity metric used for clustering (default: average similarity). + --cluster-skip Skips the cluster calculation. + +Subsequence Match Merging + --gap-size= + Maximal gap between neighboring matches to be merged (between 1 and minTokenMatch, default: 6). + --match-merging Enables merging of neighboring matches to counteract obfuscation attempts. + --neighbor-length= + Minimal length of neighboring matches to be merged (between 1 and minTokenMatch, default: 2). + +Subcommands (supported languages): + c cpp - cpp2 csharp emf emf-model @@ -123,7 +100,7 @@ try { JPlagResult result = JPlag.run(options); // Optional - ReportObjectFactory reportObjectFactory = new ReportObjectFactory(); + ReportObjectFactory reportObjectFactory = new ReportObjectFactory(new File("/path/to/output")); reportObjectFactory.createAndSaveReport(result, "/path/to/output"); } catch (ExitException e) { // error handling here diff --git a/docs/2.-Supported-Languages.md b/docs/2.-Supported-Languages.md index b3b54345a..c89bb8ee3 100644 --- a/docs/2.-Supported-Languages.md +++ b/docs/2.-Supported-Languages.md @@ -1,4 +1,4 @@ -JPlag currently supports Java, C/C++, C#, Go, Kotlin, Python, R, Rust, Scala, Swift, and Scheme. Additionally, it has primitive support for text and prototypical support for EMF metamodels. A detailed list, including the supported language versions can be found in the [project readme](https://github.com/jplag/JPlag/blob/main/README.md#supported-languages). +JPlag currently supports Java, C, C++, C#, Go, Kotlin, Python, R, Rust, Scala, Swift, and Scheme. Additionally, it has primitive support for text and prototypical support for EMF metamodels. A detailed list, including the supported language versions can be found in the [project readme](https://github.com/jplag/JPlag/blob/main/README.md#supported-languages). The language modules differ in their maturity due to their age and different usage frequencies. Thus, each frontend has a state label: diff --git a/docs/Home.md b/docs/Home.md index e7e2cbd2c..d6388bd54 100644 --- a/docs/Home.md +++ b/docs/Home.md @@ -3,14 +3,18 @@

## What is JPlag -JPlag is a system that finds similarities among multiple sets of source code files. This way it can detect software plagiarism and collusion in software development. JPlag does not merely compare bytes of text but is aware of programming language syntax and program structure and hence is robust against many kinds of attempts to disguise similarities between plagiarized files. JPlag currently supports Java, C#, C/C++, Python 3, Go, Rust, Kotlin, Swift, Scala, Scheme, EMF, and natural language text. +JPlag finds pairwise similarities among a set of multiple programs. It can reliably detect software plagiarism and collusion in software development. All similarities are calculated locally, and no source code or plagiarism results are ever uploaded to the internet. JPlag supports a large number of programming and modeling languages. JPlag does not merely compare bytes of text but is aware of programming language syntax and program structure and hence is robust against many kinds of attempts to disguise similarities (_obfusction_) between plagiarized files. JPlag is typically used to detect and thus discourage the unallowed copying of student exercise programs in programming education. But in principle, it can also be used to detect stolen software parts among large amounts of source text or modules that have been duplicated (and only slightly modified). JPlag has already played a part in several intellectual property cases where it has been successfully used by expert witnesses. -[TODO]: <> (Link or visualize example report) - **Just to make it clear**: JPlag does not compare to the internet! It is designed to find similarities among the student solutions, which is usually sufficient for computer programs. +* 📈 [JPlag Demo](https://jplag.github.io/Demo/) + +* 🏛️ [JPlag on Helmholtz RSD](https://helmholtz.software/software/jplag) + +* 🤩 [Give us Feedback in a **short (<5 min) survey**](https://docs.google.com/forms/d/e/1FAIpQLSckqUlXhIlJ-H2jtu2VmGf_mJt4hcnHXaDlwhpUL3XG1I8UYw/viewform?usp=sf_link) + ## History Originally, JPlag was developed in 1996 by Guido Mahlpohl and others at the chair of Prof. Walter Tichy at Karlsruhe Institute of Technology (KIT). It was first documented in a [Tech Report](https://publikationen.bibliothek.kit.edu/542000) in 2000 and later more formally in the [Journal of Universal Computer Science](http://www.ipd.kit.edu/tichy/uploads/publikationen/16/finding_plagiarisms_among_a_set_of_progr_638847.pdf). Since 2015 JPlag is hosted here on GitHub. After 25 years of its creation, JPlag is still used frequently in many universities in different countries around the world. @@ -26,15 +30,9 @@ JPlag is released on [Maven Central](https://search.maven.org/search?q=de.jplag) de.jplag jplag + ``` ## JPlag legacy version -In case you depend on the legacy version of JPlag we refer to the [legacy release v2.12.1](https://github.com/jplag/jplag/releases/tag/v2.12.1-SNAPSHOT) and the [legacy branch](https://github.com/jplag/jplag/tree/legacy). Note that the legacy CLI usage is slightly different. - -The following features are only available in version v4.0.0 and onwards: -* a modern web-based UI -* a simplified command-line interface -* support for Kotlin, Scala, Go, Rust, and R -* support for Java 17 language features -* a Java API for third-party integration +In case you depend on the legacy version of JPlag, we refer to the [legacy release v2.12.1](https://github.com/jplag/jplag/releases/tag/v2.12.1-SNAPSHOT) and the [legacy branch](https://github.com/jplag/jplag/tree/legacy). Note that the legacy CLI and report UI are different and provide fewer features. diff --git a/endtoend-testing/src/main/java/de/jplag/endtoend/helper/FileHelper.java b/endtoend-testing/src/main/java/de/jplag/endtoend/helper/FileHelper.java index ffcff5b2e..abc9991f3 100644 --- a/endtoend-testing/src/main/java/de/jplag/endtoend/helper/FileHelper.java +++ b/endtoend-testing/src/main/java/de/jplag/endtoend/helper/FileHelper.java @@ -72,6 +72,7 @@ private static String createNewIOExceptionStringForFileOrFOlderCreation(File fil public static void unzip(File zip, File targetDirectory) throws IOException { try (ZipFile zipFile = new ZipFile(zip)) { Enumeration entries = zipFile.entries(); + File canonicalTarget = targetDirectory.getCanonicalFile(); long totalSizeArchive = 0; long totalEntriesArchive = 0; @@ -80,9 +81,9 @@ public static void unzip(File zip, File targetDirectory) throws IOException { totalEntriesArchive++; ZipEntry entry = entries.nextElement(); - File unzippedFile = new File(targetDirectory, entry.getName()).getCanonicalFile(); + File unzippedFile = new File(canonicalTarget, entry.getName()).getCanonicalFile(); - if (unzippedFile.getAbsolutePath().startsWith(targetDirectory.getAbsolutePath())) { + if (unzippedFile.getAbsolutePath().startsWith(canonicalTarget.getAbsolutePath())) { if (entry.isDirectory()) { unzippedFile.mkdirs(); } else { @@ -91,10 +92,7 @@ public static void unzip(File zip, File targetDirectory) throws IOException { } } - if (totalSizeArchive > ZIP_THRESHOLD_SIZE) { - throw new IllegalStateException(String.format(ZIP_BOMB_ERROR_MESSAGE, zip.getAbsolutePath())); - } - if (totalEntriesArchive > ZIP_THRESHOLD_ENTRIES) { + if ((totalSizeArchive > ZIP_THRESHOLD_SIZE) || (totalEntriesArchive > ZIP_THRESHOLD_ENTRIES)) { throw new IllegalStateException(String.format(ZIP_BOMB_ERROR_MESSAGE, zip.getAbsolutePath())); } } diff --git a/endtoend-testing/src/main/java/de/jplag/endtoend/model/DataSet.java b/endtoend-testing/src/main/java/de/jplag/endtoend/model/DataSet.java index 4be001365..229018daa 100644 --- a/endtoend-testing/src/main/java/de/jplag/endtoend/model/DataSet.java +++ b/endtoend-testing/src/main/java/de/jplag/endtoend/model/DataSet.java @@ -57,7 +57,8 @@ File actualSourceDirectory() throws IOException { location = String.format(DEFAULT_SOURCE_DIRECTORY, this.name); } return new File(TestDirectoryConstants.BASE_PATH_TO_RESOURCES.toFile(), location); - } else if (actualStorageFormat == StorageFormat.ZIP) { + } + if (actualStorageFormat == StorageFormat.ZIP) { String location = sourceLocation; if (location == null) { location = String.format(DEFAULT_SOURCE_ZIP, this.name); @@ -75,9 +76,8 @@ File actualSourceDirectory() throws IOException { public File getResultFile() { if (resultFile == null) { return new File(TestDirectoryConstants.BASE_PATH_TO_RESULT_JSON.toFile(), String.format(DEFAULT_RESULT_FILE_NAME, this.name)); - } else { - return new File(TestDirectoryConstants.BASE_PATH_TO_RESULT_JSON.toFile(), resultFile); } + return new File(TestDirectoryConstants.BASE_PATH_TO_RESULT_JSON.toFile(), resultFile); } /** diff --git a/endtoend-testing/src/main/java/de/jplag/endtoend/model/Options.java b/endtoend-testing/src/main/java/de/jplag/endtoend/model/Options.java index 651f7d5bb..c84041cf5 100644 --- a/endtoend-testing/src/main/java/de/jplag/endtoend/model/Options.java +++ b/endtoend-testing/src/main/java/de/jplag/endtoend/model/Options.java @@ -41,10 +41,12 @@ public int[] getMinimumTokenMatches() { @Override public boolean equals(Object o) { - if (this == o) + if (this == o) { return true; - if (o == null || getClass() != o.getClass()) + } + if (o == null || getClass() != o.getClass()) { return false; + } Options options = (Options) o; return Arrays.equals(minimumTokenMatches, options.minimumTokenMatches) && Objects.equals(baseCodeDirectory, options.baseCodeDirectory); } diff --git a/endtoend-testing/src/main/java/de/jplag/endtoend/model/ResultDescription.java b/endtoend-testing/src/main/java/de/jplag/endtoend/model/ResultDescription.java index 81ebc6627..c46cc6f6e 100644 --- a/endtoend-testing/src/main/java/de/jplag/endtoend/model/ResultDescription.java +++ b/endtoend-testing/src/main/java/de/jplag/endtoend/model/ResultDescription.java @@ -6,8 +6,8 @@ /** * Object that maps the results of the end top end tests using the identifierToResultMap. this creates a map of test - * data and its results for each possible option specified. this is important both for serializing the data into json - * format and for deserialization. + * stream and its results for each possible option specified. this is important both for serializing the stream into + * json format and for deserialization. */ public record ResultDescription(@JsonProperty String identifier, @JsonProperty("tests") Map identifierToResultMap, @JsonProperty GoldStandard goldStandard) { diff --git a/language-antlr-utils/pom.xml b/language-antlr-utils/pom.xml index c5542fbfc..6697788a5 100644 --- a/language-antlr-utils/pom.xml +++ b/language-antlr-utils/pom.xml @@ -14,7 +14,6 @@ org.antlr antlr4-runtime - 4.13.1 de.jplag diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrLanguage.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrLanguage.java index 93f6ace30..1d75847bd 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrLanguage.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractAntlrLanguage.java @@ -34,9 +34,9 @@ protected AbstractAntlrLanguage() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { if (this.parser == null) { - this.parser = this.initializeParser(); + this.parser = this.initializeParser(normalize); } return this.parser.parse(files); @@ -46,7 +46,7 @@ public List parse(Set files) throws ParsingException { * Lazily creates the parser. Has to be implemented, if no parser is passed in the constructor. * @return The newly initialized parser */ - protected AbstractAntlrParserAdapter initializeParser() { + protected AbstractAntlrParserAdapter initializeParser(boolean normalize) { throw new UnsupportedOperationException( String.format("The initializeParser method needs to be implemented for %s", this.getClass().getName())); } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java index 7946df1d9..0ec4c19af 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java @@ -2,7 +2,11 @@ import java.util.ArrayList; import java.util.List; -import java.util.function.*; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; import org.antlr.v4.runtime.Token; import org.slf4j.Logger; diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java index 0d3aabe3a..96436248d 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java @@ -2,7 +2,11 @@ import java.util.ArrayList; import java.util.List; -import java.util.function.*; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; @@ -132,6 +136,7 @@ void exit(HandlerData data) { exitHandlers.forEach(handler -> handler.accept(data)); } + @Override Token extractEnterToken(T entity) { return entity.getStart(); } diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/InternalListener.java b/language-antlr-utils/src/main/java/de/jplag/antlr/InternalListener.java index 39178e0fb..5d2a8e51a 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/InternalListener.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/InternalListener.java @@ -14,7 +14,7 @@ class InternalListener implements ParseTreeListener { private final AbstractAntlrListener listener; private final TokenCollector collector; - protected final VariableRegistry variableRegistry; + private final VariableRegistry variableRegistry; InternalListener(AbstractAntlrListener listener, TokenCollector collector) { this.listener = listener; diff --git a/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java b/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java index 170f5d627..80fa407f9 100644 --- a/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java +++ b/language-antlr-utils/src/main/java/de/jplag/antlr/TerminalVisitor.java @@ -13,6 +13,7 @@ public class TerminalVisitor extends AbstractVisitor { super(condition); } + @Override Token extractEnterToken(Token token) { return token; } diff --git a/language-antlr-utils/src/test/java/de/jplag/antlr/LanguageTest.java b/language-antlr-utils/src/test/java/de/jplag/antlr/LanguageTest.java index 81941ac8d..b8fe50f51 100644 --- a/language-antlr-utils/src/test/java/de/jplag/antlr/LanguageTest.java +++ b/language-antlr-utils/src/test/java/de/jplag/antlr/LanguageTest.java @@ -20,19 +20,19 @@ class LanguageTest { void testExceptionForNoDefinedParser() { LanguageWithoutParser lang = new LanguageWithoutParser(); Set emptySet = Set.of(); - assertThrows(UnsupportedOperationException.class, () -> lang.parse(emptySet)); + assertThrows(UnsupportedOperationException.class, () -> lang.parse(emptySet, false)); } @Test void testLanguageWithStaticParser() throws ParsingException { TestLanguage lang = new TestLanguage(); - Assertions.assertEquals(0, lang.parse(Set.of()).size()); + Assertions.assertEquals(0, lang.parse(Set.of(), false).size()); } @Test void testLanguageWithLazyParser() throws ParsingException { LanguageWithLazyParser lang = new LanguageWithLazyParser(); - Assertions.assertEquals(0, lang.parse(Set.of()).size()); + Assertions.assertEquals(0, lang.parse(Set.of(), false).size()); } private static class LanguageWithoutParser extends AbstractAntlrLanguage { @@ -59,7 +59,7 @@ public int minimumTokenMatch() { private static class LanguageWithLazyParser extends LanguageWithoutParser { @Override - protected AbstractAntlrParserAdapter initializeParser() { + protected AbstractAntlrParserAdapter initializeParser(boolean normalize) { return new TestParserAdapter(); } } diff --git a/language-antlr-utils/src/test/java/de/jplag/antlr/ParserTest.java b/language-antlr-utils/src/test/java/de/jplag/antlr/ParserTest.java index e33277201..6538dc40c 100644 --- a/language-antlr-utils/src/test/java/de/jplag/antlr/ParserTest.java +++ b/language-antlr-utils/src/test/java/de/jplag/antlr/ParserTest.java @@ -1,6 +1,10 @@ package de.jplag.antlr; -import static de.jplag.antlr.testLanguage.TestTokenType.*; +import static de.jplag.antlr.testLanguage.TestTokenType.ADDITION; +import static de.jplag.antlr.testLanguage.TestTokenType.NUMBER; +import static de.jplag.antlr.testLanguage.TestTokenType.SUBTRACTION; +import static de.jplag.antlr.testLanguage.TestTokenType.SUB_EXPRESSION_BEGIN; +import static de.jplag.antlr.testLanguage.TestTokenType.SUB_EXPRESSION_END; import de.jplag.antlr.testLanguage.TestLanguage; import de.jplag.antlr.testLanguage.TestTokenType; diff --git a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java index 9073ad954..c61ec4c3c 100644 --- a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java +++ b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestListener.java @@ -1,9 +1,18 @@ package de.jplag.antlr.testLanguage; -import static de.jplag.antlr.testLanguage.TestTokenType.*; +import static de.jplag.antlr.testLanguage.TestTokenType.ADDITION; +import static de.jplag.antlr.testLanguage.TestTokenType.NUMBER; +import static de.jplag.antlr.testLanguage.TestTokenType.SUBTRACTION; +import static de.jplag.antlr.testLanguage.TestTokenType.SUB_EXPRESSION_BEGIN; +import static de.jplag.antlr.testLanguage.TestTokenType.SUB_EXPRESSION_END; +import static de.jplag.antlr.testLanguage.TestTokenType.VARDEF; -import de.jplag.antlr.*; -import de.jplag.antlr.TestParser.*; +import de.jplag.antlr.AbstractAntlrListener; +import de.jplag.antlr.TestParser; +import de.jplag.antlr.TestParser.CalcExpressionContext; +import de.jplag.antlr.TestParser.OperatorContext; +import de.jplag.antlr.TestParser.SubExpressionContext; +import de.jplag.antlr.TestParser.VarDefContext; import de.jplag.semantics.CodeSemantics; import de.jplag.semantics.VariableScope; diff --git a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestParserAdapter.java b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestParserAdapter.java index c2873a167..985477250 100644 --- a/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestParserAdapter.java +++ b/language-antlr-utils/src/test/java/de/jplag/antlr/testLanguage/TestParserAdapter.java @@ -5,7 +5,10 @@ import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.ParserRuleContext; -import de.jplag.antlr.*; +import de.jplag.antlr.AbstractAntlrListener; +import de.jplag.antlr.AbstractAntlrParserAdapter; +import de.jplag.antlr.TestLexer; +import de.jplag.antlr.TestParser; public class TestParserAdapter extends AbstractAntlrParserAdapter { private static final TestListener listener = new TestListener(); diff --git a/language-api/src/main/java/de/jplag/Language.java b/language-api/src/main/java/de/jplag/Language.java index c7e199667..ffe6aa9a6 100644 --- a/language-api/src/main/java/de/jplag/Language.java +++ b/language-api/src/main/java/de/jplag/Language.java @@ -32,12 +32,25 @@ public interface Language { int minimumTokenMatch(); /** - * Parses a set of files. + * Parses a set of files. Override this method, if you don't require normalization. * @param files are the files to parse. * @return the list of parsed JPlag tokens. * @throws ParsingException if an error during parsing the files occurred. + * @deprecated Replaced by {@link #parse(Set, boolean)} */ - List parse(Set files) throws ParsingException; + @Deprecated(forRemoval = true) + default List parse(Set files) throws ParsingException { + return parse(files, false); + } + + /** + * Parses a set of files. Override this method, if you require normalization within the language module. + * @param files are the files to parse. + * @param normalize True, if the tokens should be normalized + * @return the list of parsed JPlag tokens. + * @throws ParsingException if an error during parsing the files occurred. + */ + List parse(Set files, boolean normalize) throws ParsingException; /** * Indicates whether the tokens returned by parse have semantic information added to them, i.e. whether the token @@ -93,4 +106,20 @@ default boolean expectsSubmissionOrder() { default List customizeSubmissionOrder(List submissions) { return submissions; } + + /** + * @return True, if this language supports token sequence normalization. This does not include other normalization + * mechanisms that might be part of the language modules. + */ + default boolean supportsNormalization() { + return false; + } + + /** + * Override this method, if you need normalization within the language module, but not in the core module. + * @return True, If the core normalization should be used. + */ + default boolean requiresCoreNormalization() { + return true; + } } diff --git a/language-api/src/main/java/de/jplag/ParsingException.java b/language-api/src/main/java/de/jplag/ParsingException.java index 4f83721b4..c6e1ad627 100644 --- a/language-api/src/main/java/de/jplag/ParsingException.java +++ b/language-api/src/main/java/de/jplag/ParsingException.java @@ -60,16 +60,14 @@ public ParsingException(File file, String reason, Throwable cause) { * the provided exception if only one was provided. */ public static ParsingException wrappingExceptions(Collection exceptions) { - switch (exceptions.size()) { - case 0: - return null; - case 1: - return exceptions.iterator().next(); - default: { + return switch (exceptions.size()) { + case 0 -> null; + case 1 -> exceptions.iterator().next(); + default -> { String message = exceptions.stream().map(ParsingException::getMessage).collect(Collectors.joining("\n")); - return new ParsingException(message); + yield new ParsingException(message); } - } + }; } private ParsingException(String message) { @@ -78,9 +76,11 @@ private ParsingException(String message) { private static String constructMessage(File file, String reason) { StringBuilder messageBuilder = new StringBuilder(); - messageBuilder.append("failed to parse '%s'".formatted(file)); + if (reason == null || !reason.contains(file.toString())) { + messageBuilder.append("failed to parse '%s'".formatted(file)); + } if (reason != null && !reason.isBlank()) { - messageBuilder.append(" with reason: %s".formatted(reason)); + messageBuilder.append(reason); } return messageBuilder.toString(); } diff --git a/language-api/src/main/java/de/jplag/Token.java b/language-api/src/main/java/de/jplag/Token.java index b279b886d..d4ac238b6 100644 --- a/language-api/src/main/java/de/jplag/Token.java +++ b/language-api/src/main/java/de/jplag/Token.java @@ -13,15 +13,15 @@ * The language parsers decide what is a token and what is not. */ public class Token { - private Logger logger = LoggerFactory.getLogger(this.getClass()); + private final Logger logger = LoggerFactory.getLogger(this.getClass()); /** Indicates that the requested field has no value. */ public static final int NO_VALUE = -1; - private int line; - private int column; - private int length; - private File file; - private TokenType type; + private final int line; + private final int column; + private final int length; + private final File file; + private final TokenType type; private CodeSemantics semantics; // value null if no semantics /** diff --git a/language-api/src/main/java/de/jplag/TokenPrinter.java b/language-api/src/main/java/de/jplag/TokenPrinter.java index fb2198c4b..1581beb02 100644 --- a/language-api/src/main/java/de/jplag/TokenPrinter.java +++ b/language-api/src/main/java/de/jplag/TokenPrinter.java @@ -224,8 +224,9 @@ private void resetLinePosition() { public PrinterOutputBuilder append(String str) { // Avoid too many blank lines trailingLineSeparators = str.equals(LINE_SEPARATOR) ? trailingLineSeparators + 1 : 0; - if (trailingLineSeparators >= 3) + if (trailingLineSeparators >= 3) { return this; + } builder.append(str); columnIndex += str.length(); diff --git a/language-api/src/main/java/de/jplag/semantics/CodeSemantics.java b/language-api/src/main/java/de/jplag/semantics/CodeSemantics.java index 2ef9f0883..2eb99262d 100644 --- a/language-api/src/main/java/de/jplag/semantics/CodeSemantics.java +++ b/language-api/src/main/java/de/jplag/semantics/CodeSemantics.java @@ -14,8 +14,8 @@ public class CodeSemantics { private boolean keep; private PositionSignificance positionSignificance; private final int bidirectionalBlockDepthChange; - private Set reads; - private Set writes; + private final Set reads; + private final Set writes; /** * Creates new semantics. reads and writes, which each contain the variables which were (potentially) read from/written @@ -196,16 +196,21 @@ public static CodeSemantics join(List semanticsList) { @Override public String toString() { List properties = new LinkedList<>(); - if (keep) + if (keep) { properties.add("keep"); - if (positionSignificance != PositionSignificance.NONE) + } + if (positionSignificance != PositionSignificance.NONE) { properties.add(positionSignificance.name().toLowerCase() + " position significance"); - if (bidirectionalBlockDepthChange != 0) + } + if (bidirectionalBlockDepthChange != 0) { properties.add("change bidirectional block depth by " + bidirectionalBlockDepthChange); - if (!reads.isEmpty()) + } + if (!reads.isEmpty()) { properties.add("read " + String.join(" ", reads.stream().map(Variable::toString).toList())); - if (!writes.isEmpty()) + } + if (!writes.isEmpty()) { properties.add("write " + String.join(" ", writes.stream().map(Variable::toString).toList())); + } return String.join(", ", properties); } } \ No newline at end of file diff --git a/language-api/src/main/java/de/jplag/semantics/VariableRegistry.java b/language-api/src/main/java/de/jplag/semantics/VariableRegistry.java index 82b48f325..b187db34c 100644 --- a/language-api/src/main/java/de/jplag/semantics/VariableRegistry.java +++ b/language-api/src/main/java/de/jplag/semantics/VariableRegistry.java @@ -17,10 +17,10 @@ public class VariableRegistry { private static final Logger logger = LoggerFactory.getLogger(VariableRegistry.class); private CodeSemantics semantics; - private Map fileVariables; - private Deque> classVariables; // map class name to map of variable names to variables - private Map> localVariables; // map local variable name to stack of variables - private Deque> localVariablesByScope; // stack of local variable names in scope + private final Map fileVariables; + private final Deque> classVariables; // map class name to map of variable names to variables + private final Map> localVariables; // map local variable name to stack of variables + private final Deque> localVariablesByScope; // stack of local variable names in scope private VariableAccessType nextVariableAccessType; private boolean ignoreNextVariableAccess; private boolean mutableWrite; @@ -98,8 +98,9 @@ public void exitLocalScope() { for (String variableName : localVariablesByScope.pop()) { Deque variableStack = localVariables.get(variableName); variableStack.pop(); - if (variableStack.isEmpty()) + if (variableStack.isEmpty()) { localVariables.remove(variableName); + } } } @@ -146,10 +147,12 @@ public void registerVariableAccess(String variableName, boolean isClassVariable) } Variable variable = isClassVariable ? getClassVariable(variableName) : getVariable(variableName); if (variable != null) { - if (nextVariableAccessType.isRead) + if (nextVariableAccessType.isRead) { semantics.addRead(variable); - if (nextVariableAccessType.isWrite || (mutableWrite && variable.isMutable())) + } + if (nextVariableAccessType.isWrite || (mutableWrite && variable.isMutable())) { semantics.addWrite(variable); + } } // track global variables here through else nextVariableAccessType = VariableAccessType.READ; } @@ -169,8 +172,9 @@ public void addAllNonLocalVariablesAsReads() { private Variable getVariable(String variableName) { Deque variableIdStack = localVariables.get(variableName); - if (variableIdStack != null) + if (variableIdStack != null) { return variableIdStack.getFirst(); // stack is never empty + } Variable variable = getClassVariable(variableName); return variable != null ? variable : fileVariables.get(variableName); } diff --git a/language-api/src/main/java/de/jplag/util/FileUtils.java b/language-api/src/main/java/de/jplag/util/FileUtils.java index 37cb84ec9..5ce3c62b6 100644 --- a/language-api/src/main/java/de/jplag/util/FileUtils.java +++ b/language-api/src/main/java/de/jplag/util/FileUtils.java @@ -1,9 +1,24 @@ package de.jplag.util; -import java.io.*; +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.Writer; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; diff --git a/language-testutils/src/test/java/de/jplag/testutils/datacollector/FileTestData.java b/language-testutils/src/test/java/de/jplag/testutils/datacollector/FileTestData.java index f2facf796..2e917fd92 100644 --- a/language-testutils/src/test/java/de/jplag/testutils/datacollector/FileTestData.java +++ b/language-testutils/src/test/java/de/jplag/testutils/datacollector/FileTestData.java @@ -38,10 +38,12 @@ public String describeTestSource() { @Override public boolean equals(Object o) { - if (this == o) + if (this == o) { return true; - if (o == null || getClass() != o.getClass()) + } + if (o == null || getClass() != o.getClass()) { return false; + } FileTestData that = (FileTestData) o; return Objects.equals(file, that.file); } diff --git a/language-testutils/src/test/java/de/jplag/testutils/datacollector/TestDataCollector.java b/language-testutils/src/test/java/de/jplag/testutils/datacollector/TestDataCollector.java index c99a3741d..196d5cb2a 100644 --- a/language-testutils/src/test/java/de/jplag/testutils/datacollector/TestDataCollector.java +++ b/language-testutils/src/test/java/de/jplag/testutils/datacollector/TestDataCollector.java @@ -1,7 +1,11 @@ package de.jplag.testutils.datacollector; import java.io.File; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import de.jplag.TokenType; diff --git a/languages/c/README.md b/languages/c/README.md new file mode 100644 index 000000000..735466e44 --- /dev/null +++ b/languages/c/README.md @@ -0,0 +1,12 @@ +# JPlag C language module + +This module allows the use of JPlag with submissions in c. + +## Usage + +To parse C submissions run JPlag with: ` c` or use the `-l c` options. +To use the module from the API configure your `JPlagOption` object with `new CLanguage()` as 'Language' as described in the usage information in the [readme](https://github.com/jplag/JPlag#usage) and [in the wiki](https://github.com/jplag/JPlag/wiki/1.-How-to-Use-JPlag). + +## C++ + +This module might work with C++ submissions. However you should use the [cpp module](https://github.com/jplag/JPlag/tree/main/languages/cpp) for that. \ No newline at end of file diff --git a/languages/c/pom.xml b/languages/c/pom.xml new file mode 100644 index 000000000..71d9cdb8e --- /dev/null +++ b/languages/c/pom.xml @@ -0,0 +1,35 @@ + + + 4.0.0 + + de.jplag + languages + ${revision} + + c + + + + + com.helger.maven + ph-javacc-maven-plugin + + + javacc-gen + + javacc + + generate-sources + + 21 + true + de.jplag.c + src/main/javacc + ${project.build.directory}/generated-sources/javacc + + + + + + + diff --git a/languages/cpp/src/main/java/de/jplag/cpp/Language.java b/languages/c/src/main/java/de/jplag/c/CLanguage.java similarity index 67% rename from languages/cpp/src/main/java/de/jplag/cpp/Language.java rename to languages/c/src/main/java/de/jplag/c/CLanguage.java index dcc69edd7..ba99dbf49 100644 --- a/languages/cpp/src/main/java/de/jplag/cpp/Language.java +++ b/languages/c/src/main/java/de/jplag/c/CLanguage.java @@ -1,4 +1,4 @@ -package de.jplag.cpp; +package de.jplag.c; import java.io.File; import java.util.List; @@ -6,16 +6,17 @@ import org.kohsuke.MetaInfServices; +import de.jplag.Language; import de.jplag.ParsingException; import de.jplag.Token; @MetaInfServices(de.jplag.Language.class) -public class Language implements de.jplag.Language { - private static final String IDENTIFIER = "cpp"; +public class CLanguage implements Language { + private static final String IDENTIFIER = "c"; - private final Scanner scanner; // cpp code is scanned not parsed + private final Scanner scanner; // c code is scanned not parsed - public Language() { + public CLanguage() { scanner = new Scanner(); } @@ -26,7 +27,7 @@ public String[] suffixes() { @Override public String getName() { - return "C/C++ Scanner [basic markup]"; + return "C Scanner"; } @Override @@ -40,7 +41,7 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { return this.scanner.scan(files); } } diff --git a/languages/c/src/main/java/de/jplag/c/CTokenType.java b/languages/c/src/main/java/de/jplag/c/CTokenType.java new file mode 100644 index 000000000..185daa3e1 --- /dev/null +++ b/languages/c/src/main/java/de/jplag/c/CTokenType.java @@ -0,0 +1,77 @@ +package de.jplag.c; + +import de.jplag.TokenType; + +public enum CTokenType implements TokenType { + C_BLOCK_BEGIN("BLOCK{"), + C_BLOCK_END("}BLOCK"), + C_QUESTIONMARK("COND"), + C_ELLIPSIS("..."), + C_ASSIGN("ASSIGN"), + C_DOT("DOT"), + C_ARROW("ARROW"), + C_ARROWSTAR("ARROWSTAR"), + C_AUTO("AUTO"), + C_BREAK("BREAK"), + C_CASE("CASE"), + C_CATCH("CATCH"), + C_CHAR("CHAR"), + C_CONST("CONST"), + C_CONTINUE("CONTINUE"), + C_DEFAULT("DEFAULT"), + C_DELETE("DELETE"), + C_DO("DO"), + C_DOUBLE("DOUBLE"), + C_ELSE("ELSE"), + C_ENUM("ENUM"), + C_EXTERN("EXTERN"), + C_FLOAT("FLOAT"), + C_FOR("FOR"), + C_FRIEND("FRIEND"), + C_GOTO("GOTO"), + C_IF("IF"), + C_INLINE("INLINE"), + C_INT("INT"), + C_LONG("LONG"), + C_NEW("NEW"), + C_PRIVATE("PRIVATE"), + C_PROTECTED("PROTECTED"), + C_PUBLIC("PUBLIC"), + C_REDECLARED("REDECLARED"), + C_REGISTER("REGISTER"), + C_RETURN("RETURN"), + C_SHORT("SHORT"), + C_SIGNED("SIGNED"), + C_SIZEOF("SIZEOF"), + C_STATIC("STATIC"), + C_STRUCT("STRUCT"), + C_CLASS("CLASS"), + C_SWITCH("SWITCH"), + C_TEMPLATE("TEMPLATE"), + C_THIS("THIS"), + C_TRY("TRY"), + C_TYPEDEF("TYPEDEF"), + C_UNION("UNION"), + C_UNSIGNED("UNSIGNED"), + C_VIRTUAL("VIRTUAL"), + C_VOID("VOID"), + C_VOLATILE("VOLATILE"), + C_WHILE("WHILE"), + C_OPERATOR("OPERATOR"), + C_THROW("THROW"), + C_ID("ID"), + C_FUN("FUN"), + C_DOTSTAR("DOTSTAR"), + C_NULL("NULL"); + + private final String description; + + @Override + public String getDescription() { + return this.description; + } + + CTokenType(String description) { + this.description = description; + } +} diff --git a/languages/cpp/src/main/java/de/jplag/cpp/NewlineStream.java b/languages/c/src/main/java/de/jplag/c/NewlineStream.java similarity index 97% rename from languages/cpp/src/main/java/de/jplag/cpp/NewlineStream.java rename to languages/c/src/main/java/de/jplag/c/NewlineStream.java index 9cc21a3d8..f45c9af61 100644 --- a/languages/cpp/src/main/java/de/jplag/cpp/NewlineStream.java +++ b/languages/c/src/main/java/de/jplag/c/NewlineStream.java @@ -1,4 +1,4 @@ -package de.jplag.cpp; +package de.jplag.c; import java.io.IOException; import java.io.InputStream; diff --git a/languages/cpp/src/main/java/de/jplag/cpp/Scanner.java b/languages/c/src/main/java/de/jplag/c/Scanner.java similarity index 91% rename from languages/cpp/src/main/java/de/jplag/cpp/Scanner.java rename to languages/c/src/main/java/de/jplag/c/Scanner.java index e7f25476d..c3292814e 100644 --- a/languages/cpp/src/main/java/de/jplag/cpp/Scanner.java +++ b/languages/c/src/main/java/de/jplag/c/Scanner.java @@ -1,4 +1,4 @@ -package de.jplag.cpp; +package de.jplag.c; import java.io.File; import java.util.ArrayList; @@ -32,7 +32,7 @@ public List scan(Set files) throws ParsingException { return tokens; } - public void add(CPPTokenType type, de.jplag.cpp.Token token) { + public void add(CTokenType type, de.jplag.c.Token token) { int length = token.endColumn - token.beginColumn + 1; tokens.add(new Token(type, currentFile, token.beginLine, token.beginColumn, length)); } diff --git a/languages/cpp/src/main/java/de/jplag/cpp/experimental/GCCSourceAnalysis.java b/languages/c/src/main/java/de/jplag/c/experimental/GCCSourceAnalysis.java similarity index 91% rename from languages/cpp/src/main/java/de/jplag/cpp/experimental/GCCSourceAnalysis.java rename to languages/c/src/main/java/de/jplag/c/experimental/GCCSourceAnalysis.java index 52b7bdfcd..d78aceb7a 100644 --- a/languages/cpp/src/main/java/de/jplag/cpp/experimental/GCCSourceAnalysis.java +++ b/languages/c/src/main/java/de/jplag/c/experimental/GCCSourceAnalysis.java @@ -1,10 +1,14 @@ -package de.jplag.cpp.experimental; +package de.jplag.c.experimental; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +29,7 @@ public GCCSourceAnalysis() { } @Override - public boolean isTokenIgnored(de.jplag.cpp.Token token, File file) { + public boolean isTokenIgnored(de.jplag.c.Token token, File file) { String fileName = file.getName(); if (linesToDelete.containsKey(fileName)) { var ignoredLineNumbers = linesToDelete.get(fileName); diff --git a/languages/cpp/src/main/java/de/jplag/cpp/experimental/SourceAnalysis.java b/languages/c/src/main/java/de/jplag/c/experimental/SourceAnalysis.java similarity index 90% rename from languages/cpp/src/main/java/de/jplag/cpp/experimental/SourceAnalysis.java rename to languages/c/src/main/java/de/jplag/c/experimental/SourceAnalysis.java index 0d846370e..f92986d1d 100644 --- a/languages/cpp/src/main/java/de/jplag/cpp/experimental/SourceAnalysis.java +++ b/languages/c/src/main/java/de/jplag/c/experimental/SourceAnalysis.java @@ -1,4 +1,4 @@ -package de.jplag.cpp.experimental; +package de.jplag.c.experimental; import java.io.File; import java.util.Set; @@ -16,7 +16,7 @@ public interface SourceAnalysis { * @param file The file the token was scanned in * @return True, if the token should not be added to a TokenList, false if it should */ - boolean isTokenIgnored(de.jplag.cpp.Token token, File file); + boolean isTokenIgnored(de.jplag.c.Token token, File file); /** * Executes the source analysis on the files of a submission. diff --git a/languages/cpp/src/main/java/de/jplag/cpp/experimental/UnreachableCodeFilter.java b/languages/c/src/main/java/de/jplag/c/experimental/UnreachableCodeFilter.java similarity index 89% rename from languages/cpp/src/main/java/de/jplag/cpp/experimental/UnreachableCodeFilter.java rename to languages/c/src/main/java/de/jplag/c/experimental/UnreachableCodeFilter.java index e795c6976..e325c914b 100644 --- a/languages/cpp/src/main/java/de/jplag/cpp/experimental/UnreachableCodeFilter.java +++ b/languages/c/src/main/java/de/jplag/c/experimental/UnreachableCodeFilter.java @@ -1,7 +1,17 @@ -package de.jplag.cpp.experimental; +package de.jplag.c.experimental; import static de.jplag.SharedTokenType.FILE_END; -import static de.jplag.cpp.CPPTokenType.*; +import static de.jplag.c.CTokenType.C_BLOCK_BEGIN; +import static de.jplag.c.CTokenType.C_BLOCK_END; +import static de.jplag.c.CTokenType.C_BREAK; +import static de.jplag.c.CTokenType.C_CASE; +import static de.jplag.c.CTokenType.C_CONTINUE; +import static de.jplag.c.CTokenType.C_FOR; +import static de.jplag.c.CTokenType.C_GOTO; +import static de.jplag.c.CTokenType.C_IF; +import static de.jplag.c.CTokenType.C_RETURN; +import static de.jplag.c.CTokenType.C_THROW; +import static de.jplag.c.CTokenType.C_WHILE; import java.util.List; import java.util.ListIterator; diff --git a/languages/cpp/src/main/javacc/CPP.jj b/languages/c/src/main/javacc/CPP.jj similarity index 99% rename from languages/cpp/src/main/javacc/CPP.jj rename to languages/c/src/main/javacc/CPP.jj index cc1689fbe..ee4ca34d8 100644 --- a/languages/cpp/src/main/javacc/CPP.jj +++ b/languages/c/src/main/javacc/CPP.jj @@ -14,7 +14,7 @@ options } PARSER_BEGIN(CPPScanner) -package de.jplag.cpp; +package de.jplag.c; import java.io.File; import java.io.FileInputStream; @@ -23,7 +23,7 @@ import java.io.InputStream; import de.jplag.ParsingException; -import static de.jplag.cpp.CPPTokenType.*; +import static de.jplag.c.CTokenType.*; public class CPPScanner { private Scanner delegatingScanner; diff --git a/languages/cpp2/README.md b/languages/cpp/README.md similarity index 59% rename from languages/cpp2/README.md rename to languages/cpp/README.md index a8046b20f..9eb9b09aa 100644 --- a/languages/cpp2/README.md +++ b/languages/cpp/README.md @@ -1,11 +1,8 @@ # JPlag C++ language module -**Note**: This language module is meant to replace the existing C++ language module in the future. -While the old language module is based on lexer tokens, this language module uses a parse tree for token extraction. -The base package name of this language module and its identifier are `cpp2` currently, but this might change if the old -language module gets replaced. +**Note**: This replaces the old cpp module, which is now only meant for c, as it works better for c than this one. -The JPlag C++ frontend allows the use of JPlag with submissions in C/C++.
+The JPlag C++ frontend allows the use of JPlag with submissions in C++.
It is based on the [C++ ANTLR4 grammar](https://github.com/antlr/grammars-v4/tree/master/cpp), licensed under MIT. ### C++ specification compatibility @@ -21,11 +18,11 @@ While the Java language module is based on an AST, this language module uses a p There are differences, including: - `import` is extracted in Java, while `using` is not extracted due to the fact that it can be placed freely in the code. -More syntactic elements of C/C++ may turn out to be helpful to include in the future, especially those that are newly introduced. +More syntactic elements of C++ may turn out to be helpful to include in the future, especially those that are newly introduced. ### Usage -To use the C++ frontend, add the `-l cpp2` flag in the CLI, or use a `JPlagOption` object with `new de.jplag.cpp2.CPPLanguage()` as `Language` in the Java API as described in the usage information in the [readme of the main project](https://github.com/jplag/JPlag#usage) and [in the wiki](https://github.com/jplag/JPlag/wiki/1.-How-to-Use-JPlag). +To use the C++ frontend, add the `-l cpp` flag in the CLI, or use a `JPlagOption` object with `new de.jplag.cpp.CPPLanguage()` as `Language` in the Java API as described in the usage information in the [readme of the main project](https://github.com/jplag/JPlag#usage) and [in the wiki](https://github.com/jplag/JPlag/wiki/1.-How-to-Use-JPlag). ### Changes to the Grammar diff --git a/languages/cpp/pom.xml b/languages/cpp/pom.xml index 734f52d92..d3cfb4881 100644 --- a/languages/cpp/pom.xml +++ b/languages/cpp/pom.xml @@ -8,25 +8,28 @@ cpp + + + org.antlr + antlr4-runtime + + + de.jplag + language-antlr-utils + ${revision} + + + - com.helger.maven - ph-javacc-maven-plugin + org.antlr + antlr4-maven-plugin - javacc-gen - javacc + antlr4 - generate-sources - - 21 - true - de.jplag.cpp - src/main/javacc - ${project.build.directory}/generated-sources/javacc - diff --git a/languages/cpp2/src/main/antlr4/de/jplag/cpp2/grammar/CPP14Lexer.g4 b/languages/cpp/src/main/antlr4/de/jplag/cpp/grammar/CPP14Lexer.g4 similarity index 100% rename from languages/cpp2/src/main/antlr4/de/jplag/cpp2/grammar/CPP14Lexer.g4 rename to languages/cpp/src/main/antlr4/de/jplag/cpp/grammar/CPP14Lexer.g4 diff --git a/languages/cpp2/src/main/antlr4/de/jplag/cpp2/grammar/CPP14Parser.g4 b/languages/cpp/src/main/antlr4/de/jplag/cpp/grammar/CPP14Parser.g4 similarity index 100% rename from languages/cpp2/src/main/antlr4/de/jplag/cpp2/grammar/CPP14Parser.g4 rename to languages/cpp/src/main/antlr4/de/jplag/cpp/grammar/CPP14Parser.g4 diff --git a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPLanguage.java b/languages/cpp/src/main/java/de/jplag/cpp/CPPLanguage.java similarity index 81% rename from languages/cpp2/src/main/java/de/jplag/cpp2/CPPLanguage.java rename to languages/cpp/src/main/java/de/jplag/cpp/CPPLanguage.java index 990ab8c9f..c08e53dce 100644 --- a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPLanguage.java +++ b/languages/cpp/src/main/java/de/jplag/cpp/CPPLanguage.java @@ -1,4 +1,4 @@ -package de.jplag.cpp2; +package de.jplag.cpp; import org.kohsuke.MetaInfServices; @@ -10,7 +10,7 @@ */ @MetaInfServices(Language.class) public class CPPLanguage extends AbstractAntlrLanguage { - private static final String IDENTIFIER = "cpp2"; + private static final String IDENTIFIER = "cpp"; public CPPLanguage() { super(new CPPParserAdapter()); @@ -23,7 +23,7 @@ public String[] suffixes() { @Override public String getName() { - return "C/C++ Parser"; + return "C++ Parser"; } @Override @@ -40,4 +40,9 @@ public int minimumTokenMatch() { public boolean tokensHaveSemantics() { return true; } + + @Override + public boolean supportsNormalization() { + return true; + } } diff --git a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java b/languages/cpp/src/main/java/de/jplag/cpp/CPPListener.java similarity index 69% rename from languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java rename to languages/cpp/src/main/java/de/jplag/cpp/CPPListener.java index 115ec54de..28a6a88b9 100644 --- a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPListener.java +++ b/languages/cpp/src/main/java/de/jplag/cpp/CPPListener.java @@ -1,6 +1,47 @@ -package de.jplag.cpp2; +package de.jplag.cpp; -import static de.jplag.cpp2.CPPTokenType.*; +import static de.jplag.cpp.CPPTokenType.APPLY; +import static de.jplag.cpp.CPPTokenType.ASSIGN; +import static de.jplag.cpp.CPPTokenType.BRACED_INIT_BEGIN; +import static de.jplag.cpp.CPPTokenType.BRACED_INIT_END; +import static de.jplag.cpp.CPPTokenType.BREAK; +import static de.jplag.cpp.CPPTokenType.CASE; +import static de.jplag.cpp.CPPTokenType.CATCH_BEGIN; +import static de.jplag.cpp.CPPTokenType.CATCH_END; +import static de.jplag.cpp.CPPTokenType.CLASS_BEGIN; +import static de.jplag.cpp.CPPTokenType.CLASS_END; +import static de.jplag.cpp.CPPTokenType.CONTINUE; +import static de.jplag.cpp.CPPTokenType.DEFAULT; +import static de.jplag.cpp.CPPTokenType.DO_BEGIN; +import static de.jplag.cpp.CPPTokenType.DO_END; +import static de.jplag.cpp.CPPTokenType.ELSE; +import static de.jplag.cpp.CPPTokenType.ENUM_BEGIN; +import static de.jplag.cpp.CPPTokenType.ENUM_END; +import static de.jplag.cpp.CPPTokenType.FOR_BEGIN; +import static de.jplag.cpp.CPPTokenType.FOR_END; +import static de.jplag.cpp.CPPTokenType.FUNCTION_BEGIN; +import static de.jplag.cpp.CPPTokenType.FUNCTION_END; +import static de.jplag.cpp.CPPTokenType.GENERIC; +import static de.jplag.cpp.CPPTokenType.GOTO; +import static de.jplag.cpp.CPPTokenType.IF_BEGIN; +import static de.jplag.cpp.CPPTokenType.IF_END; +import static de.jplag.cpp.CPPTokenType.NEWARRAY; +import static de.jplag.cpp.CPPTokenType.NEWCLASS; +import static de.jplag.cpp.CPPTokenType.QUESTIONMARK; +import static de.jplag.cpp.CPPTokenType.RETURN; +import static de.jplag.cpp.CPPTokenType.STATIC_ASSERT; +import static de.jplag.cpp.CPPTokenType.STRUCT_BEGIN; +import static de.jplag.cpp.CPPTokenType.STRUCT_END; +import static de.jplag.cpp.CPPTokenType.SWITCH_BEGIN; +import static de.jplag.cpp.CPPTokenType.SWITCH_END; +import static de.jplag.cpp.CPPTokenType.THROW; +import static de.jplag.cpp.CPPTokenType.TRY_BEGIN; +import static de.jplag.cpp.CPPTokenType.TRY_END; +import static de.jplag.cpp.CPPTokenType.UNION_BEGIN; +import static de.jplag.cpp.CPPTokenType.UNION_END; +import static de.jplag.cpp.CPPTokenType.VARDEF; +import static de.jplag.cpp.CPPTokenType.WHILE_BEGIN; +import static de.jplag.cpp.CPPTokenType.WHILE_END; import java.util.function.Function; @@ -10,8 +51,41 @@ import de.jplag.TokenType; import de.jplag.antlr.AbstractAntlrListener; import de.jplag.antlr.ContextVisitor; -import de.jplag.cpp2.grammar.CPP14Parser; -import de.jplag.cpp2.grammar.CPP14Parser.*; +import de.jplag.cpp.grammar.CPP14Parser; +import de.jplag.cpp.grammar.CPP14Parser.AssignmentOperatorContext; +import de.jplag.cpp.grammar.CPP14Parser.BraceOrEqualInitializerContext; +import de.jplag.cpp.grammar.CPP14Parser.BracedInitListContext; +import de.jplag.cpp.grammar.CPP14Parser.ClassKeyContext; +import de.jplag.cpp.grammar.CPP14Parser.ClassSpecifierContext; +import de.jplag.cpp.grammar.CPP14Parser.ConditionalExpressionContext; +import de.jplag.cpp.grammar.CPP14Parser.DeclaratorContext; +import de.jplag.cpp.grammar.CPP14Parser.EnumSpecifierContext; +import de.jplag.cpp.grammar.CPP14Parser.EnumeratorDefinitionContext; +import de.jplag.cpp.grammar.CPP14Parser.FunctionBodyContext; +import de.jplag.cpp.grammar.CPP14Parser.FunctionDefinitionContext; +import de.jplag.cpp.grammar.CPP14Parser.HandlerContext; +import de.jplag.cpp.grammar.CPP14Parser.InitDeclaratorContext; +import de.jplag.cpp.grammar.CPP14Parser.IterationStatementContext; +import de.jplag.cpp.grammar.CPP14Parser.JumpStatementContext; +import de.jplag.cpp.grammar.CPP14Parser.LabeledStatementContext; +import de.jplag.cpp.grammar.CPP14Parser.MemberDeclaratorContext; +import de.jplag.cpp.grammar.CPP14Parser.MemberSpecificationContext; +import de.jplag.cpp.grammar.CPP14Parser.MemberdeclarationContext; +import de.jplag.cpp.grammar.CPP14Parser.NewExpressionContext; +import de.jplag.cpp.grammar.CPP14Parser.NewTypeIdContext; +import de.jplag.cpp.grammar.CPP14Parser.NoPointerDeclaratorContext; +import de.jplag.cpp.grammar.CPP14Parser.ParameterDeclarationContext; +import de.jplag.cpp.grammar.CPP14Parser.PostfixExpressionContext; +import de.jplag.cpp.grammar.CPP14Parser.SelectionStatementContext; +import de.jplag.cpp.grammar.CPP14Parser.SimpleDeclarationContext; +import de.jplag.cpp.grammar.CPP14Parser.SimpleTypeSpecifierContext; +import de.jplag.cpp.grammar.CPP14Parser.StaticAssertDeclarationContext; +import de.jplag.cpp.grammar.CPP14Parser.TemplateArgumentContext; +import de.jplag.cpp.grammar.CPP14Parser.TemplateDeclarationContext; +import de.jplag.cpp.grammar.CPP14Parser.ThrowExpressionContext; +import de.jplag.cpp.grammar.CPP14Parser.TryBlockContext; +import de.jplag.cpp.grammar.CPP14Parser.UnaryExpressionContext; +import de.jplag.cpp.grammar.CPP14Parser.UnqualifiedIdContext; import de.jplag.semantics.CodeSemantics; import de.jplag.semantics.VariableAccessType; import de.jplag.semantics.VariableRegistry; @@ -33,29 +107,11 @@ class CPPListener extends AbstractAntlrListener { visit(FunctionDefinitionContext.class).map(FUNCTION_BEGIN, FUNCTION_END).addLocalScope().withSemantics(CodeSemantics::createControl); - visit(IterationStatementContext.class, rule -> rule.Do() != null).map(DO_BEGIN, DO_END).addLocalScope().withLoopSemantics(); - visit(IterationStatementContext.class, rule -> rule.For() != null).map(FOR_BEGIN, FOR_END).addLocalScope().withLoopSemantics(); - visit(IterationStatementContext.class, rule -> rule.While() != null && rule.Do() == null).map(WHILE_BEGIN, WHILE_END).addLocalScope() - .withLoopSemantics(); - - visit(SelectionStatementContext.class, rule -> rule.Switch() != null).map(SWITCH_BEGIN, SWITCH_END).addLocalScope() - .withSemantics(CodeSemantics::createControl); - visit(SelectionStatementContext.class, rule -> rule.If() != null).map(IF_BEGIN, IF_END).addLocalScope() - .withSemantics(CodeSemantics::createControl); - // possible problem: variable from if visible in else, but in reality is not -- doesn't really matter - visit(CPP14Parser.Else).map(ELSE).withSemantics(CodeSemantics::createControl); - - visit(LabeledStatementContext.class, rule -> rule.Case() != null).map(CASE).withSemantics(CodeSemantics::createControl); - visit(LabeledStatementContext.class, rule -> rule.Default() != null).map(DEFAULT).withSemantics(CodeSemantics::createControl); + statementRules(); visit(TryBlockContext.class).map(TRY_BEGIN, TRY_END).addLocalScope().withSemantics(CodeSemantics::createControl); visit(HandlerContext.class).map(CATCH_BEGIN, CATCH_END).addLocalScope().withSemantics(CodeSemantics::createControl); - visit(JumpStatementContext.class, rule -> rule.Break() != null).map(BREAK).withSemantics(CodeSemantics::createControl); - visit(JumpStatementContext.class, rule -> rule.Continue() != null).map(CONTINUE).withSemantics(CodeSemantics::createControl); - visit(JumpStatementContext.class, rule -> rule.Goto() != null).map(GOTO).withSemantics(CodeSemantics::createControl); - visit(JumpStatementContext.class, rule -> rule.Return() != null).map(RETURN).withSemantics(CodeSemantics::createControl); - visit(ThrowExpressionContext.class).map(THROW).withSemantics(CodeSemantics::createControl); visit(NewExpressionContext.class, rule -> rule.newInitializer() != null).map(NEWCLASS).withSemantics(CodeSemantics::new); @@ -73,20 +129,51 @@ class CPPListener extends AbstractAntlrListener { .onEnter((rule, varReg) -> varReg.setNextVariableAccessType(VariableAccessType.WRITE)); visit(BracedInitListContext.class).map(BRACED_INIT_BEGIN, BRACED_INIT_END).withSemantics(CodeSemantics::new); + typeSpecifierRule(); + declarationRules(); + expressionRules(); + idRules(); + } + + private void statementRules() { + visit(IterationStatementContext.class, rule -> rule.Do() != null).map(DO_BEGIN, DO_END).addLocalScope().withLoopSemantics(); + visit(IterationStatementContext.class, rule -> rule.For() != null).map(FOR_BEGIN, FOR_END).addLocalScope().withLoopSemantics(); + visit(IterationStatementContext.class, rule -> rule.While() != null && rule.Do() == null).map(WHILE_BEGIN, WHILE_END).addLocalScope() + .withLoopSemantics(); + + visit(SelectionStatementContext.class, rule -> rule.Switch() != null).map(SWITCH_BEGIN, SWITCH_END).addLocalScope() + .withSemantics(CodeSemantics::createControl); + visit(SelectionStatementContext.class, rule -> rule.If() != null).map(IF_BEGIN, IF_END).addLocalScope() + .withSemantics(CodeSemantics::createControl); + // possible problem: variable from if visible in else, but in reality is not -- doesn't really matter + visit(CPP14Parser.Else).map(ELSE).withSemantics(CodeSemantics::createControl); + + visit(LabeledStatementContext.class, rule -> rule.Case() != null).map(CASE).withSemantics(CodeSemantics::createControl); + visit(LabeledStatementContext.class, rule -> rule.Default() != null).map(DEFAULT).withSemantics(CodeSemantics::createControl); + + visit(JumpStatementContext.class, rule -> rule.Break() != null).map(BREAK).withSemantics(CodeSemantics::createControl); + visit(JumpStatementContext.class, rule -> rule.Continue() != null).map(CONTINUE).withSemantics(CodeSemantics::createControl); + visit(JumpStatementContext.class, rule -> rule.Goto() != null).map(GOTO).withSemantics(CodeSemantics::createControl); + visit(JumpStatementContext.class, rule -> rule.Return() != null).map(RETURN).withSemantics(CodeSemantics::createControl); + } + + private void typeSpecifierRule() { visit(SimpleTypeSpecifierContext.class, rule -> { if (hasAncestor(rule, MemberdeclarationContext.class, FunctionDefinitionContext.class)) { return true; } SimpleDeclarationContext parent = getAncestor(rule, SimpleDeclarationContext.class, TemplateArgumentContext.class, FunctionDefinitionContext.class); - if (parent == null) + if (parent == null) { return false; + } NoPointerDeclaratorContext noPointerDecl = getDescendant(parent, NoPointerDeclaratorContext.class); return !noPointerInFunctionCallContext(noPointerDecl) && !hasAncestor(rule, NewTypeIdContext.class); }).map(VARDEF).withSemantics(CodeSemantics::new).onEnter((context, variableRegistry) -> { SimpleDeclarationContext parent = getAncestor(context, SimpleDeclarationContext.class); - if (parent == null) // at this point we know parent exists + if (parent == null) { // at this point we know parent exists throw new IllegalStateException(); + } // boolean typeMutable = context.theTypeName() != null; // block is duplicate to member variable register // possible issue: what if multiple variables are declared in the same line? variableRegistry.setNextVariableAccessType(VariableAccessType.WRITE); @@ -99,7 +186,9 @@ class CPPListener extends AbstractAntlrListener { variableRegistry.registerVariable(name, scope, true); } }); + } + private void declarationRules() { mapApply(visit(SimpleDeclarationContext.class, rule -> { if (!hasAncestor(rule, FunctionBodyContext.class)) { return false; @@ -125,12 +214,17 @@ class CPPListener extends AbstractAntlrListener { varReg.setNextVariableAccessType(VariableAccessType.WRITE); } }); + } + + private void expressionRules() { visit(ConditionalExpressionContext.class, rule -> rule.Question() != null).map(QUESTIONMARK).withSemantics(CodeSemantics::new); mapApply(visit(PostfixExpressionContext.class, rule -> rule.LeftParen() != null)); visit(PostfixExpressionContext.class, rule -> rule.PlusPlus() != null || rule.MinusMinus() != null).map(ASSIGN) .withSemantics(CodeSemantics::new).onEnter((rule, varReg) -> varReg.setNextVariableAccessType(VariableAccessType.READ_WRITE)); + } + private void idRules() { visit(UnqualifiedIdContext.class).onEnter((ctx, varReg) -> { ParserRuleContext parentCtx = ctx.getParent().getParent(); if (!parentCtx.getParent().getParent().getText().contains("(")) { diff --git a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPParserAdapter.java b/languages/cpp/src/main/java/de/jplag/cpp/CPPParserAdapter.java similarity index 91% rename from languages/cpp2/src/main/java/de/jplag/cpp2/CPPParserAdapter.java rename to languages/cpp/src/main/java/de/jplag/cpp/CPPParserAdapter.java index 925406360..d1bc12a6b 100644 --- a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPParserAdapter.java +++ b/languages/cpp/src/main/java/de/jplag/cpp/CPPParserAdapter.java @@ -1,4 +1,4 @@ -package de.jplag.cpp2; +package de.jplag.cpp; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CommonTokenStream; @@ -8,8 +8,8 @@ import de.jplag.AbstractParser; import de.jplag.antlr.AbstractAntlrListener; import de.jplag.antlr.AbstractAntlrParserAdapter; -import de.jplag.cpp2.grammar.CPP14Lexer; -import de.jplag.cpp2.grammar.CPP14Parser; +import de.jplag.cpp.grammar.CPP14Lexer; +import de.jplag.cpp.grammar.CPP14Parser; /** * The adapter between {@link AbstractParser} and the ANTLR based parser of this language module. diff --git a/languages/cpp/src/main/java/de/jplag/cpp/CPPTokenType.java b/languages/cpp/src/main/java/de/jplag/cpp/CPPTokenType.java index e8143b98f..0bf830cd6 100644 --- a/languages/cpp/src/main/java/de/jplag/cpp/CPPTokenType.java +++ b/languages/cpp/src/main/java/de/jplag/cpp/CPPTokenType.java @@ -2,67 +2,52 @@ import de.jplag.TokenType; +/** + * C++ token types extracted by this language module. + */ public enum CPPTokenType implements TokenType { - C_BLOCK_BEGIN("BLOCK{"), - C_BLOCK_END("}BLOCK"), - C_QUESTIONMARK("COND"), - C_ELLIPSIS("..."), - C_ASSIGN("ASSIGN"), - C_DOT("DOT"), - C_ARROW("ARROW"), - C_ARROWSTAR("ARROWSTAR"), - C_AUTO("AUTO"), - C_BREAK("BREAK"), - C_CASE("CASE"), - C_CATCH("CATCH"), - C_CHAR("CHAR"), - C_CONST("CONST"), - C_CONTINUE("CONTINUE"), - C_DEFAULT("DEFAULT"), - C_DELETE("DELETE"), - C_DO("DO"), - C_DOUBLE("DOUBLE"), - C_ELSE("ELSE"), - C_ENUM("ENUM"), - C_EXTERN("EXTERN"), - C_FLOAT("FLOAT"), - C_FOR("FOR"), - C_FRIEND("FRIEND"), - C_GOTO("GOTO"), - C_IF("IF"), - C_INLINE("INLINE"), - C_INT("INT"), - C_LONG("LONG"), - C_NEW("NEW"), - C_PRIVATE("PRIVATE"), - C_PROTECTED("PROTECTED"), - C_PUBLIC("PUBLIC"), - C_REDECLARED("REDECLARED"), - C_REGISTER("REGISTER"), - C_RETURN("RETURN"), - C_SHORT("SHORT"), - C_SIGNED("SIGNED"), - C_SIZEOF("SIZEOF"), - C_STATIC("STATIC"), - C_STRUCT("STRUCT"), - C_CLASS("CLASS"), - C_SWITCH("SWITCH"), - C_TEMPLATE("TEMPLATE"), - C_THIS("THIS"), - C_TRY("TRY"), - C_TYPEDEF("TYPEDEF"), - C_UNION("UNION"), - C_UNSIGNED("UNSIGNED"), - C_VIRTUAL("VIRTUAL"), - C_VOID("VOID"), - C_VOLATILE("VOLATILE"), - C_WHILE("WHILE"), - C_OPERATOR("OPERATOR"), - C_THROW("THROW"), - C_ID("ID"), - C_FUN("FUN"), - C_DOTSTAR("DOTSTAR"), - C_NULL("NULL"); + CLASS_BEGIN("CLASS{"), + CLASS_END("}CLASS"), + STRUCT_BEGIN("STRUCT{"), + STRUCT_END("}STRUCT"), + ENUM_BEGIN("ENUM{"), + ENUM_END("}ENUM"), + UNION_BEGIN("UNION{"), + UNION_END("}UNION"), + FUNCTION_BEGIN("FUNCTION{"), + FUNCTION_END("}FUNCTION"), + DO_BEGIN("DO{"), + DO_END("}DO"), + WHILE_BEGIN("WHILE{"), + WHILE_END("}WHILE"), + FOR_BEGIN("FOR{"), + FOR_END("}FOR"), + SWITCH_BEGIN("SWITCH{"), + SWITCH_END("}SWITCH"), + CASE("CASE"), + TRY_BEGIN("TRY{"), + TRY_END("}TRY"), + CATCH_BEGIN("CATCH{"), + CATCH_END("}CATCH"), + IF_BEGIN("IF{"), + IF_END("}IF"), + ELSE("ELSE"), + BREAK("BREAK"), + CONTINUE("CONTINUE"), + GOTO("GOTO"), + RETURN("RETURN"), + THROW("THROW"), + NEWCLASS("NEWCLASS"), + GENERIC("GENERIC"), + NEWARRAY("NEWARRAY"), + BRACED_INIT_BEGIN("BRACED_INIT{"), + BRACED_INIT_END("}BRACED_INIT"), + ASSIGN("ASSIGN"), + STATIC_ASSERT("STATIC_ASSERT"), + VARDEF("VARDEF"), + QUESTIONMARK("COND"), + DEFAULT("DEFAULT"), + APPLY("APPLY"); private final String description; diff --git a/languages/cpp2/src/test/java/de/jplag/cpp2/CppLanguageTest.java b/languages/cpp/src/test/java/de/jplag/cpp/CppLanguageTest.java similarity index 99% rename from languages/cpp2/src/test/java/de/jplag/cpp2/CppLanguageTest.java rename to languages/cpp/src/test/java/de/jplag/cpp/CppLanguageTest.java index 52d4342d9..c885c4562 100644 --- a/languages/cpp2/src/test/java/de/jplag/cpp2/CppLanguageTest.java +++ b/languages/cpp/src/test/java/de/jplag/cpp/CppLanguageTest.java @@ -1,4 +1,4 @@ -package de.jplag.cpp2; +package de.jplag.cpp; import java.util.Arrays; diff --git a/languages/cpp2/src/test/resources/de/jplag/cpp2/CallOutsideMethodInClass.cpp b/languages/cpp/src/test/resources/de/jplag/cpp/CallOutsideMethodInClass.cpp similarity index 100% rename from languages/cpp2/src/test/resources/de/jplag/cpp2/CallOutsideMethodInClass.cpp rename to languages/cpp/src/test/resources/de/jplag/cpp/CallOutsideMethodInClass.cpp diff --git a/languages/cpp2/src/test/resources/de/jplag/cpp2/FunctionCall.cpp b/languages/cpp/src/test/resources/de/jplag/cpp/FunctionCall.cpp similarity index 100% rename from languages/cpp2/src/test/resources/de/jplag/cpp2/FunctionCall.cpp rename to languages/cpp/src/test/resources/de/jplag/cpp/FunctionCall.cpp diff --git a/languages/cpp2/src/test/resources/de/jplag/cpp2/IfElse.cpp b/languages/cpp/src/test/resources/de/jplag/cpp/IfElse.cpp similarity index 100% rename from languages/cpp2/src/test/resources/de/jplag/cpp2/IfElse.cpp rename to languages/cpp/src/test/resources/de/jplag/cpp/IfElse.cpp diff --git a/languages/cpp2/src/test/resources/de/jplag/cpp2/IntArray.cpp b/languages/cpp/src/test/resources/de/jplag/cpp/IntArray.cpp similarity index 100% rename from languages/cpp2/src/test/resources/de/jplag/cpp2/IntArray.cpp rename to languages/cpp/src/test/resources/de/jplag/cpp/IntArray.cpp diff --git a/languages/cpp2/src/test/resources/de/jplag/cpp2/Loop.cpp b/languages/cpp/src/test/resources/de/jplag/cpp/Loop.cpp similarity index 100% rename from languages/cpp2/src/test/resources/de/jplag/cpp2/Loop.cpp rename to languages/cpp/src/test/resources/de/jplag/cpp/Loop.cpp diff --git a/languages/cpp2/src/test/resources/de/jplag/cpp2/Union.cpp b/languages/cpp/src/test/resources/de/jplag/cpp/Union.cpp similarity index 100% rename from languages/cpp2/src/test/resources/de/jplag/cpp2/Union.cpp rename to languages/cpp/src/test/resources/de/jplag/cpp/Union.cpp diff --git a/languages/cpp2/src/test/resources/de/jplag/cpp2/bc6h_enc.h b/languages/cpp/src/test/resources/de/jplag/cpp/bc6h_enc.h similarity index 100% rename from languages/cpp2/src/test/resources/de/jplag/cpp2/bc6h_enc.h rename to languages/cpp/src/test/resources/de/jplag/cpp/bc6h_enc.h diff --git a/languages/cpp2/pom.xml b/languages/cpp2/pom.xml deleted file mode 100644 index cfd8366cc..000000000 --- a/languages/cpp2/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - 4.0.0 - - de.jplag - languages - ${revision} - - cpp2 - - - - org.antlr - antlr4-runtime - - - de.jplag - language-antlr-utils - ${revision} - - - - - - - org.antlr - antlr4-maven-plugin - - - - antlr4 - - - - - - - diff --git a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPTokenType.java b/languages/cpp2/src/main/java/de/jplag/cpp2/CPPTokenType.java deleted file mode 100644 index a87a43630..000000000 --- a/languages/cpp2/src/main/java/de/jplag/cpp2/CPPTokenType.java +++ /dev/null @@ -1,62 +0,0 @@ -package de.jplag.cpp2; - -import de.jplag.TokenType; - -/** - * C++ token types extracted by this language module. - */ -public enum CPPTokenType implements TokenType { - CLASS_BEGIN("CLASS{"), - CLASS_END("}CLASS"), - STRUCT_BEGIN("STRUCT{"), - STRUCT_END("}STRUCT"), - ENUM_BEGIN("ENUM{"), - ENUM_END("}ENUM"), - UNION_BEGIN("UNION{"), - UNION_END("}UNION"), - FUNCTION_BEGIN("FUNCTION{"), - FUNCTION_END("}FUNCTION"), - DO_BEGIN("DO{"), - DO_END("}DO"), - WHILE_BEGIN("WHILE{"), - WHILE_END("}WHILE"), - FOR_BEGIN("FOR{"), - FOR_END("}FOR"), - SWITCH_BEGIN("SWITCH{"), - SWITCH_END("}SWITCH"), - CASE("CASE"), - TRY_BEGIN("TRY{"), - TRY_END("}TRY"), - CATCH_BEGIN("CATCH{"), - CATCH_END("}CATCH"), - IF_BEGIN("IF{"), - IF_END("}IF"), - ELSE("ELSE"), - BREAK("BREAK"), - CONTINUE("CONTINUE"), - GOTO("GOTO"), - RETURN("RETURN"), - THROW("THROW"), - NEWCLASS("NEWCLASS"), - GENERIC("GENERIC"), - NEWARRAY("NEWARRAY"), - BRACED_INIT_BEGIN("BRACED_INIT{"), - BRACED_INIT_END("}BRACED_INIT"), - ASSIGN("ASSIGN"), - STATIC_ASSERT("STATIC_ASSERT"), - VARDEF("VARDEF"), - QUESTIONMARK("COND"), - DEFAULT("DEFAULT"), - APPLY("APPLY"); - - private final String description; - - @Override - public String getDescription() { - return this.description; - } - - CPPTokenType(String description) { - this.description = description; - } -} diff --git a/languages/emf-metamodel-dynamic/README.md b/languages/emf-metamodel-dynamic/README.md index a0ff03849..0502cfac5 100644 --- a/languages/emf-metamodel-dynamic/README.md +++ b/languages/emf-metamodel-dynamic/README.md @@ -1,5 +1,5 @@ # Dynamic EMF metamodel language module -The dynamic EMF metamodel language module allows the use of JPlag with metamodel submissions. +The dynamic EMF metamodel language module allows the use of JPlag with EMF metamodel submissions. It is based on the EMF API. ### EMF specification compatibility @@ -9,8 +9,14 @@ This module is based on the EMF dependencies available on maven central. These m For the token extraction, we visit the containment tree of the metamodel and extract tokens for all metamodel elements based on their concrete metaclass. In this module, we thus extract tokens based on a dynamic token set. ### Usage -To use this module, add the `-l emf-metamodel-dynamic` flag in the CLI, or use a `JPlagOption` object with `new DynamicEmfLanguage()` as `language` in the Java API as described in the usage information in the [readme of the main project](https://github.com/jplag/JPlag#usage) and [in the wiki](https://github.com/jplag/JPlag/wiki/1.-How-to-Use-JPlag). +Note that this language module is currently not offered via the CLI. +Use the non-dymamic version instead (`-l emf`). -### More Info -More information can be found in the paper [*"Token-based Plagiarism Detection for Metamodels" (MODELS-C'22)*](https://dl.acm.org/doi/10.1145/3550356.3556508). -A short summary can be found on [Kudos](https://www.growkudos.com/publications/10.1145%25252F3550356.3556508/reader). +### Report Viewer +In the report viewer, Emfatic is used as a textual model view. + +### Literature +* [*"Token-based Plagiarism Detection for Metamodels" (MODELS-C'22)*](https://dl.acm.org/doi/10.1145/3550356.3556508). +* Its [Kudos Summary](https://www.growkudos.com/publications/10.1145%25252F3550356.3556508/reader). +* [*"Token-based Plagiarism Detection for Metamodels" (MODELS-C'22)*] +* *"Automated Detection of AI-Obfuscated Plagiarism in Modeling Assignments" (ICSE-SEET'24)* diff --git a/languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicElementTokenizer.java b/languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicElementTokenizer.java index 346771411..e3ee69097 100644 --- a/languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicElementTokenizer.java +++ b/languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicElementTokenizer.java @@ -1,7 +1,7 @@ package de.jplag.emf.dynamic.parser; -import java.util.HashSet; -import java.util.Set; +import java.util.LinkedHashSet; +import java.util.SequencedSet; import org.eclipse.emf.ecore.EClass; import org.eclipse.emf.ecore.EObject; @@ -15,14 +15,7 @@ */ public class DynamicElementTokenizer implements ModelingElementTokenizer { - private final Set knownTokenTypes; - - /** - * Creates the tokenizer, initially with an empty token set. - */ - public DynamicElementTokenizer() { - knownTokenTypes = new HashSet<>(); - } + private static final SequencedSet knownTokenTypes = new LinkedHashSet<>(); @Override public TokenType element2Token(EObject modelElement) { @@ -32,7 +25,7 @@ public TokenType element2Token(EObject modelElement) { } @Override - public Set allTokenTypes() { - return Set.copyOf(knownTokenTypes); + public SequencedSet allTokenTypes() { + return new LinkedHashSet<>(knownTokenTypes); } } diff --git a/languages/emf-metamodel-dynamic/src/test/java/de/jplag/emf/dynamic/MinimalDynamicMetamodelTest.java b/languages/emf-metamodel-dynamic/src/test/java/de/jplag/emf/dynamic/MinimalDynamicMetamodelTest.java index 0911fd455..a437be6e2 100644 --- a/languages/emf-metamodel-dynamic/src/test/java/de/jplag/emf/dynamic/MinimalDynamicMetamodelTest.java +++ b/languages/emf-metamodel-dynamic/src/test/java/de/jplag/emf/dynamic/MinimalDynamicMetamodelTest.java @@ -47,7 +47,7 @@ public void setUp() { @DisplayName("Test tokens generated from example metamodels") void testBookstoreMetamodels() throws ParsingException { List testFiles = Arrays.stream(TEST_SUBJECTS).map(path -> new File(BASE_PATH.toFile(), path)).toList(); - List result = language.parse(new HashSet<>(testFiles)); + List result = language.parse(new HashSet<>(testFiles), true); List tokenTypes = result.stream().map(Token::getType).toList(); logger.debug(TokenPrinter.printTokens(result, baseDirectory, Optional.of(EmfLanguage.VIEW_FILE_SUFFIX))); logger.info("parsed token types: " + tokenTypes.stream().map(TokenType::getDescription).toList()); diff --git a/languages/emf-metamodel/README.md b/languages/emf-metamodel/README.md index fc203805d..071bfbcaf 100644 --- a/languages/emf-metamodel/README.md +++ b/languages/emf-metamodel/README.md @@ -1,5 +1,5 @@ # EMF metamodel language module -The EMF metamodel language module allows the use of JPlag with metamodel submissions. +The EMF metamodel language module allows the use of JPlag with EMF metamodel submissions. It is based on the EMF API. ### EMF specification compatibility @@ -9,8 +9,14 @@ This module is based on the EMF dependencies available on maven central. These m For the token extraction, we visit the containment tree of the metamodel and extract tokens for certain metamodel elements based on their metaclass. In this module, we extract tokens based on a [handcrafted token set](https://github.com/jplag/JPlag/blob/master/languages/emf-metamodel/src/main/java/de/jplag/emf/MetamodelTokenType.java). Note that not for all concrete metaclasses tokens are extracted. `EFactory`, `EGenericType`, and `EObject` are ignored. Moreover, for some metaclasses, multiple token types are extracted. Finally, some references are also used for token extraction. ### Usage -To use this module, add the `-l emf-metamodel` flag in the CLI, or use a `JPlagOption` object with `new EmfLanguage()` as `language` in the Java API as described in the usage information in the [readme of the main project](https://github.com/jplag/JPlag#usage) and [in the wiki](https://github.com/jplag/JPlag/wiki/1.-How-to-Use-JPlag). +The input for this module is a set of EMF metamodels (`.ecore` files). +To use this module, add the `-l emf` flag in the CLI, or use a `JPlagOption` object with `new EmfLanguage()` as `language` in the Java API as described in the usage information in the [readme of the main project](https://github.com/jplag/JPlag#usage) and [in the wiki](https://github.com/jplag/JPlag/wiki/1.-How-to-Use-JPlag). -### More Info -More information can be found in the paper [*"Token-based Plagiarism Detection for Metamodels" (MODELS-C'22)*](https://dl.acm.org/doi/10.1145/3550356.3556508). -A short summary can be found on [Kudos](https://www.growkudos.com/publications/10.1145%25252F3550356.3556508/reader). +### Report Viewer +In the report viewer, Emfatic is used as a textual model view. + +### Literature +* [*"Token-based Plagiarism Detection for Metamodels" (MODELS-C'22)*](https://dl.acm.org/doi/10.1145/3550356.3556508). +* Its [Kudos Summary](https://www.growkudos.com/publications/10.1145%25252F3550356.3556508/reader). +* [*"Token-based Plagiarism Detection for Metamodels" (MODELS-C'22)*] +* *"Automated Detection of AI-Obfuscated Plagiarism in Modeling Assignments" (ICSE-SEET'24)* diff --git a/languages/emf-metamodel/src/main/java/de/jplag/emf/EmfLanguage.java b/languages/emf-metamodel/src/main/java/de/jplag/emf/EmfLanguage.java index ba3c967ac..fac48399f 100644 --- a/languages/emf-metamodel/src/main/java/de/jplag/emf/EmfLanguage.java +++ b/languages/emf-metamodel/src/main/java/de/jplag/emf/EmfLanguage.java @@ -55,8 +55,8 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { - return parser.parse(files); + public List parse(Set files, boolean normalize) throws ParsingException { + return parser.parse(files, normalize); } @Override diff --git a/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/ContainmentOrderNormalizer.java b/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/ContainmentOrderNormalizer.java index f8dc57934..0e40f976d 100644 --- a/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/ContainmentOrderNormalizer.java +++ b/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/ContainmentOrderNormalizer.java @@ -45,9 +45,11 @@ public int compare(EObject first, EObject second) { // 0. comparison if token types are absent for one or more elements. if (firstType == null && secondType == null) { return 0; - } else if (firstType == null) { + } + if (firstType == null) { return -1; - } else if (secondType == null) { + } + if (secondType == null) { return 1; } @@ -91,7 +93,7 @@ private List calculatePath(TokenType type) { List elements = modelElementsToSort.stream().filter(it -> type.equals(tokenizer.element2Token(it))).toList(); // Generate token type distributions for the subtrees of the elements to sort: - Map> subtreeVectors = new HashMap<>(); + Map subtreeVectors = new HashMap<>(); elements.forEach(it -> subtreeVectors.put(it, tokenVectorGenerator.generateOccurenceVector(it.eAllContents()))); // Calculate distance matrix: @@ -118,10 +120,11 @@ private int countSubtreeTokens(EObject modelElement) { return count; } - private static double euclideanDistance(List first, List second) { - if (first.size() != second.size()) { - throw new IllegalArgumentException("Lists must have the same size"); - } + /** + * Calculates the euclidean distance for two token occurrence vectors. As they are zero-padded, they are virtually of + * the same length. + */ + private static double euclideanDistance(TokenOccurenceVector first, TokenOccurenceVector second) { double sum = 0; for (int i = 0; i < first.size(); i++) { double diff = first.get(i) - second.get(i); @@ -129,4 +132,5 @@ private static double euclideanDistance(List first, List second) } return Math.sqrt(sum); } + } diff --git a/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/TokenOccurenceVector.java b/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/TokenOccurenceVector.java new file mode 100644 index 000000000..47c2c0dbb --- /dev/null +++ b/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/TokenOccurenceVector.java @@ -0,0 +1,39 @@ +package de.jplag.emf.normalization; + +import java.util.List; + +/** + * A vector for the occurrence frequency of different token types. The vector is padded with zeroes beyond its original + * size. The vector content cannot be changed after its creation. + */ +public class TokenOccurenceVector { + private final List originalVector; + + /** + * Creates a zero-padded token occurrence vector. + * @param originalVector specifies the occurrence frequency values for the vector. + */ + public TokenOccurenceVector(List originalVector) { + this.originalVector = originalVector; + } + + /** + * Return a occurrence frequency value of the vector at the specified. + * @param index is the specified index. + * @return the occurrence frequency value or zero if the index is beyond the size of the vector. + */ + public double get(int index) { + if (index >= originalVector.size()) { + return 0.0; + } + return originalVector.get(index); + } + + /** + * The original size of the vector, without padding. + * @return the size. + */ + public int size() { + return originalVector.size(); + } +} \ No newline at end of file diff --git a/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/TokenVectorGenerator.java b/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/TokenVectorGenerator.java index 3eb4ce5ea..734b1d081 100644 --- a/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/TokenVectorGenerator.java +++ b/languages/emf-metamodel/src/main/java/de/jplag/emf/normalization/TokenVectorGenerator.java @@ -10,7 +10,6 @@ import org.eclipse.emf.ecore.EObject; import de.jplag.TokenType; -import de.jplag.emf.MetamodelTokenType; import de.jplag.emf.parser.ModelingElementTokenizer; /** @@ -27,10 +26,10 @@ public TokenVectorGenerator(ModelingElementTokenizer tokenizer) { /** * Generate a token occurrence vector for a subtree of a model. * @param modelElements is a visitor for the subtree. - * @return a list, where each entry represents the number of tokens in the subtree. The order is determined by - * {@link MetamodelTokenType}. + * @return a zero padded token occurrence vector, where each entry represents the number of tokens in the subtree. The + * order is determined by {@link ModelingElementTokenizer#allTokenTypes()}. */ - public List generateOccurenceVector(Iterator modelElements) { + public TokenOccurenceVector generateOccurenceVector(Iterator modelElements) { Map tokenTypeHistogram = new HashMap<>(); while (modelElements.hasNext()) { @@ -40,7 +39,7 @@ public List generateOccurenceVector(Iterator modelElements) { for (TokenType type : tokenizer.allTokenTypes()) { occurenceVector.add(tokenTypeHistogram.getOrDefault(type, 0)); } - return normalize(occurenceVector); + return new TokenOccurenceVector(normalize(occurenceVector)); } public static List normalize(List vector) { diff --git a/languages/emf-metamodel/src/main/java/de/jplag/emf/parser/EcoreParser.java b/languages/emf-metamodel/src/main/java/de/jplag/emf/parser/EcoreParser.java index 924d3d544..5ea3adf11 100644 --- a/languages/emf-metamodel/src/main/java/de/jplag/emf/parser/EcoreParser.java +++ b/languages/emf-metamodel/src/main/java/de/jplag/emf/parser/EcoreParser.java @@ -41,10 +41,10 @@ public EcoreParser() { * @param files is the set of files. * @return the list of parsed tokens. */ - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { tokens = new ArrayList<>(); for (File file : files) { - parseModelFile(file); + parseModelFile(file, normalize); } return tokens; } @@ -53,21 +53,22 @@ public List parse(Set files) throws ParsingException { * Loads a metamodel from a file and parses it. * @param file is the metamodel file. */ - protected void parseModelFile(File file) throws ParsingException { + protected void parseModelFile(File file, boolean normalize) throws ParsingException { currentFile = file; Resource model = EMFUtil.loadModelResource(file); if (model == null) { throw new ParsingException(file, "failed to load model"); - } else { + } + if (normalize) { normalizeOrder(model); - treeView = createView(file, model); - visitor = createMetamodelVisitor(); - for (EObject root : model.getContents()) { - visitor.visit(root); - } - tokens.add(Token.fileEnd(currentFile)); - treeView.writeToFile(getCorrespondingViewFileSuffix()); } + treeView = createView(file, model); + visitor = createMetamodelVisitor(); + for (EObject root : model.getContents()) { + visitor.visit(root); + } + tokens.add(Token.fileEnd(currentFile)); + treeView.writeToFile(getCorrespondingViewFileSuffix()); } /** diff --git a/languages/emf-metamodel/src/main/java/de/jplag/emf/parser/MetamodelElementTokenizer.java b/languages/emf-metamodel/src/main/java/de/jplag/emf/parser/MetamodelElementTokenizer.java index f5f7c66c1..1d2b83c09 100644 --- a/languages/emf-metamodel/src/main/java/de/jplag/emf/parser/MetamodelElementTokenizer.java +++ b/languages/emf-metamodel/src/main/java/de/jplag/emf/parser/MetamodelElementTokenizer.java @@ -38,20 +38,19 @@ public MetamodelTokenType caseEAnnotation(EAnnotation eAnnotation) { public MetamodelTokenType caseEAttribute(EAttribute eAttribute) { if (eAttribute.isID()) { return MetamodelTokenType.ID_ATTRIBUTE; - } else { - return MetamodelTokenType.ATTRIBUTE; } + return MetamodelTokenType.ATTRIBUTE; } @Override public MetamodelTokenType caseEClass(EClass eClass) { if (eClass.isInterface()) { return MetamodelTokenType.INTERFACE; - } else if (eClass.isAbstract()) { + } + if (eClass.isAbstract()) { return MetamodelTokenType.ABSTRACT_CLASS; - } else { - return MetamodelTokenType.CLASS; } + return MetamodelTokenType.CLASS; } @Override @@ -94,16 +93,13 @@ public MetamodelTokenType caseEReference(EReference eReference) { if (eReference.isContainment()) { if (eReference.getUpperBound() == 1) { return MetamodelTokenType.CONTAINMENT; - } else { - return MetamodelTokenType.CONTAINMENT_MULT; - } - } else { - if (eReference.getUpperBound() == 1) { - return MetamodelTokenType.REFERENCE; - } else { - return MetamodelTokenType.REFERENCE_MULT; } + return MetamodelTokenType.CONTAINMENT_MULT; + } + if (eReference.getUpperBound() == 1) { + return MetamodelTokenType.REFERENCE; } + return MetamodelTokenType.REFERENCE_MULT; } @Override diff --git a/languages/emf-metamodel/src/main/java/de/jplag/emf/util/GenericEmfTreeView.java b/languages/emf-metamodel/src/main/java/de/jplag/emf/util/GenericEmfTreeView.java new file mode 100644 index 000000000..f68e08620 --- /dev/null +++ b/languages/emf-metamodel/src/main/java/de/jplag/emf/util/GenericEmfTreeView.java @@ -0,0 +1,153 @@ +package de.jplag.emf.util; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.StringJoiner; + +import org.eclipse.emf.ecore.ENamedElement; +import org.eclipse.emf.ecore.EObject; +import org.eclipse.emf.ecore.EStructuralFeature; +import org.eclipse.emf.ecore.resource.Resource; + +import de.jplag.TokenTrace; +import de.jplag.emf.MetamodelToken; + +/** + * Very basic tree view representation of an EMF metamodel or model. + */ +public class GenericEmfTreeView extends AbstractModelView { + private final List lines; + private final Map objectToLine; + private final ModelingElementIdentifierManager identifierManager; + + /** + * Creates a tree view for a metamodel. + * @param file is the path to the metamodel. + */ + public GenericEmfTreeView(File file, Resource modelResource) { + super(file); + lines = new ArrayList<>(); + objectToLine = new HashMap<>(); + identifierManager = new ModelingElementIdentifierManager(); + TreeViewBuilder visitor = new TreeViewBuilder(); + modelResource.getContents().forEach(visitor::visit); + } + + /** + * Adds a token to the view, thus adding the index information to the token. Returns a new token enriched with the index + * metadata. + * @param token is the token to add. + */ + @Override + public MetamodelToken convertToMetadataEnrichedToken(MetamodelToken token) { + Optional optionalEObject = token.getEObject(); + if (optionalEObject.isPresent()) { + EObject object = optionalEObject.get(); + TokenTrace trace = objectToLine.get(object); + return new MetamodelToken(token.getType(), token.getFile(), trace, optionalEObject); + } + return new MetamodelToken(token.getType(), token.getFile()); + } + + private final class TreeViewBuilder extends AbstractMetamodelVisitor { + private static final String IDENTIFIER_PREFIX = " #"; + private static final String VALUE_ASSIGNMENT = "="; + private static final String COLLECTION_PREFIX = "["; + private static final String COLLECTION_SUFFIX = "]"; + private static final String COLLECTION_DELIMITER = ", "; + private static final int ABBREVIATION_LIMIT = 20; + private static final String ABBREVIATION_SUFFIX = "..."; + private static final String TEXT_AFFIX = "\""; + private static final String IDENTIFIER_REGEX = "name|identifier"; + private static final String INDENTATION = " "; + + @Override + protected void visitEObject(EObject eObject) { + String prefix = INDENTATION.repeat(getCurrentTreeDepth()); + StringBuilder line = new StringBuilder(prefix); + + line.append(eObject.eClass().getName()); // Build element type + line.append(IDENTIFIER_PREFIX); + line.append(identifierManager.getIdentifier(eObject)); + visitStructuralFeatures(eObject, line); // Build element features + + lines.add(line.toString()); + viewBuilder.append(line + System.lineSeparator()); + // line and column values are one-indexed + TokenTrace trace = new TokenTrace(lines.size(), prefix.length() + 1, line.toString().trim().length()); + objectToLine.put(eObject, trace); + } + + private void visitStructuralFeatures(EObject eObject, StringBuilder line) { + List structuralFeatures = eObject.eClass().getEAllStructuralFeatures(); + if (!structuralFeatures.isEmpty()) { + line.append(": "); + StringJoiner joiner = new StringJoiner(COLLECTION_DELIMITER); + for (EStructuralFeature feature : structuralFeatures) { + Object value = eObject.eGet(feature); + String name = featureValueToString(value); + if (name != null) { + joiner.add(feature.getName() + VALUE_ASSIGNMENT + name); + } + } + line.append(joiner.toString()); + + } + } + + private String featureValueToString(Object value) { + String name = null; + if (value != null) { + if (value instanceof EObject featureValue) { + List valueIdentifiers = deriveNameOrIdentifers(featureValue); + + if (!valueIdentifiers.isEmpty()) { + name = TEXT_AFFIX + valueIdentifiers.get(0) + TEXT_AFFIX; + } else { + name = featureValue.eClass().getName() + IDENTIFIER_PREFIX + identifierManager.getIdentifier(featureValue); + } + } else if (value instanceof Collection multipleValues) { + name = valueListToString(multipleValues); + } else { + name = value.toString(); + name = (name.length() > ABBREVIATION_LIMIT) ? name.substring(0, ABBREVIATION_LIMIT) + ABBREVIATION_SUFFIX : name; + name = TEXT_AFFIX + name + TEXT_AFFIX; + } + } + return name; + } + + private String valueListToString(Collection multipleValues) { + String name = null; + if (!multipleValues.isEmpty()) { + name = COLLECTION_PREFIX; + StringJoiner joiner = new StringJoiner(COLLECTION_DELIMITER); + for (Object innerValue : multipleValues) { + joiner.add(featureValueToString(innerValue)); + } + name += joiner.toString() + COLLECTION_SUFFIX; + } + return name; + } + + private static List deriveNameOrIdentifers(EObject eObject) { + List names = new ArrayList<>(); + if (eObject instanceof ENamedElement element) { + names.add(element.getName()); + } else { + for (EStructuralFeature feature : eObject.eClass().getEAllStructuralFeatures()) { + if (feature.getName().toLowerCase().matches(IDENTIFIER_REGEX) && eObject.eGet(feature) != null) { + names.add(eObject.eGet(feature).toString()); + } + } + } + return names; + } + } + +} diff --git a/languages/emf-metamodel/src/main/java/de/jplag/emf/util/MetamodelTreeView.java b/languages/emf-metamodel/src/main/java/de/jplag/emf/util/MetamodelTreeView.java deleted file mode 100644 index 55f3647bc..000000000 --- a/languages/emf-metamodel/src/main/java/de/jplag/emf/util/MetamodelTreeView.java +++ /dev/null @@ -1,73 +0,0 @@ -package de.jplag.emf.util; - -import java.io.File; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import org.eclipse.emf.ecore.ENamedElement; -import org.eclipse.emf.ecore.EObject; -import org.eclipse.emf.ecore.resource.Resource; - -import de.jplag.TokenTrace; -import de.jplag.emf.MetamodelToken; - -/** - * Very basic tree view representation of an EMF metamodel or model. - */ -public class MetamodelTreeView extends AbstractModelView { - private final List lines; - private final Map objectToLine; - - /** - * Creates a tree view for a metamodel. - * @param file is the path to the metamodel. - */ - public MetamodelTreeView(File file, Resource modelResource) { - super(file); - lines = new ArrayList<>(); - objectToLine = new HashMap<>(); - TreeViewBuilder visitor = new TreeViewBuilder(); - modelResource.getContents().forEach(visitor::visit); - } - - /** - * Adds a token to the view, thus adding the index information to the token. Returns a new token enriched with the index - * metadata. - * @param token is the token to add. - */ - @Override - public MetamodelToken convertToMetadataEnrichedToken(MetamodelToken token) { - Optional optionalEObject = token.getEObject(); - if (optionalEObject.isPresent()) { - EObject object = optionalEObject.get(); - TokenTrace trace = objectToLine.get(object); - return new MetamodelToken(token.getType(), token.getFile(), trace, optionalEObject); - } - return new MetamodelToken(token.getType(), token.getFile()); - } - - private final class TreeViewBuilder extends AbstractMetamodelVisitor { - private static final String INDENTATION = " "; - private static final String NAME_SEPARATOR = " : "; - - @Override - protected void visitEObject(EObject eObject) { - String prefix = INDENTATION.repeat(getCurrentTreeDepth()); - String line = prefix; - if (eObject instanceof ENamedElement element) { - line += element.getName() + NAME_SEPARATOR; - } - line += eObject.eClass().getName(); - - lines.add(line); - viewBuilder.append(line + System.lineSeparator()); - // line and column values are one-indexed - TokenTrace trace = new TokenTrace(lines.size(), prefix.length() + 1, line.trim().length()); - objectToLine.put(eObject, trace); - } - } - -} diff --git a/languages/emf-metamodel/src/main/java/de/jplag/emf/util/ModelingElementIdentifierManager.java b/languages/emf-metamodel/src/main/java/de/jplag/emf/util/ModelingElementIdentifierManager.java new file mode 100644 index 000000000..895087dd6 --- /dev/null +++ b/languages/emf-metamodel/src/main/java/de/jplag/emf/util/ModelingElementIdentifierManager.java @@ -0,0 +1,42 @@ +package de.jplag.emf.util; + +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import org.eclipse.emf.ecore.EClass; +import org.eclipse.emf.ecore.EObject; + +/** + * This class provides type-unique identifiers for EObjects. + */ +public class ModelingElementIdentifierManager { + + private final Map> elementToIdentifer; + + /** + * Creates the identifier manager. Identifiers are only unique if managed by the same instance. + */ + public ModelingElementIdentifierManager() { + elementToIdentifer = new HashMap<>(); + } + + /** + * Returns the type-unique identifier for any EMF modeling element. + * @param element is the modeling element for which the identifier is requested. + * @return the identifier, that is unique for all elements of the same EClass. + */ + public int getIdentifier(EObject element) { + Set elements = elementToIdentifer.computeIfAbsent(element.eClass(), key -> new LinkedHashSet<>()); + int index = 0; + for (EObject containedElement : elements) { + if (containedElement.equals(element)) { + return index; + } + ++index; + } + elements.add(element); + return index; + } +} diff --git a/languages/emf-metamodel/src/test/java/de/jplag/emf/MinimalMetamodelTest.java b/languages/emf-metamodel/src/test/java/de/jplag/emf/MinimalMetamodelTest.java index 28f8f5347..c2559f7c8 100644 --- a/languages/emf-metamodel/src/test/java/de/jplag/emf/MinimalMetamodelTest.java +++ b/languages/emf-metamodel/src/test/java/de/jplag/emf/MinimalMetamodelTest.java @@ -28,7 +28,7 @@ class MinimalMetamodelTest extends AbstractEmfTest { @DisplayName("Test tokens generated from example metamodels") void testBookstoreMetamodels() throws ParsingException { List testFiles = Arrays.stream(TEST_SUBJECTS).map(path -> new File(BASE_PATH.toFile(), path)).toList(); - List result = language.parse(new HashSet<>(testFiles)); + List result = language.parse(new HashSet<>(testFiles), true); logger.debug(TokenPrinter.printTokens(result, baseDirectory, Optional.of(EmfLanguage.VIEW_FILE_SUFFIX))); List tokenTypes = result.stream().map(Token::getType).toList(); diff --git a/languages/emf-metamodel/src/test/java/de/jplag/emf/util/MetamodelTreeViewTest.java b/languages/emf-metamodel/src/test/java/de/jplag/emf/util/GenericEmfTreeViewTest.java similarity index 85% rename from languages/emf-metamodel/src/test/java/de/jplag/emf/util/MetamodelTreeViewTest.java rename to languages/emf-metamodel/src/test/java/de/jplag/emf/util/GenericEmfTreeViewTest.java index 9a2976974..bef41d57b 100644 --- a/languages/emf-metamodel/src/test/java/de/jplag/emf/util/MetamodelTreeViewTest.java +++ b/languages/emf-metamodel/src/test/java/de/jplag/emf/util/GenericEmfTreeViewTest.java @@ -13,7 +13,7 @@ import de.jplag.emf.AbstractEmfTest; import de.jplag.testutils.FileUtil; -class MetamodelTreeViewTest extends AbstractEmfTest { +class GenericEmfTreeViewTest extends AbstractEmfTest { private static final String VIEW_FILE_SUFFIX = ".treeview"; private static final String EXPECTED_VIEW_FOLDER = "treeview"; @@ -23,7 +23,7 @@ private static List provideModelNames() { } @ParameterizedTest - @DisplayName("Test content of emfatic view files of example metamodels") + @DisplayName("Test content of generic EMF view files of example metamodels") @MethodSource("provideModelNames") void testEmfaticViewFiles(String modelName) { // Load model: @@ -31,7 +31,7 @@ void testEmfaticViewFiles(String modelName) { Resource modelResource = loadAndVerifyModel(modelFile); // Generate emfatic view: - MetamodelTreeView view = new MetamodelTreeView(modelFile, modelResource); + GenericEmfTreeView view = new GenericEmfTreeView(modelFile, modelResource); view.writeToFile(VIEW_FILE_SUFFIX); // Compare expected vs. actual view file: diff --git a/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStore.ecore.treeview b/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStore.ecore.treeview index ce6ad5331..05c24f48e 100644 --- a/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStore.ecore.treeview +++ b/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStore.ecore.treeview @@ -1,13 +1,13 @@ -BookStorePackage : EPackage - BookStore : EClass - owner : EAttribute - EGenericType - location : EAttribute - EGenericType - books : EReference - EGenericType - Book : EClass - name : EAttribute - EGenericType - isbn : EAttribute - EGenericType +EPackage #0: name="BookStorePackage", nsURI="http:///com.ibm.dyna...", nsPrefix="bookStore", eFactoryInstance=EFactory #0, eClassifiers=["BookStore", "Book"] + EClass #0: name="BookStore", ePackage="BookStorePackage", abstract="false", interface="false", eAllAttributes=["owner", "location"], eAllReferences=["books"], eReferences=["books"], eAttributes=["owner", "location"], eAllContainments=["books"], eAllStructuralFeatures=["owner", "location", "books"], eStructuralFeatures=["owner", "location", "books"] + EAttribute #0: name="owner", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #0, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="BookStore", iD="false", eAttributeType="EString" + EGenericType #0: eRawType="EString", eClassifier="EString" + EAttribute #1: name="location", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #1, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="BookStore", iD="false", eAttributeType="EString" + EGenericType #1: eRawType="EString", eClassifier="EString" + EReference #0: name="books", ordered="true", unique="true", lowerBound="0", upperBound="-1", many="true", required="false", eType="Book", eGenericType=EGenericType #2, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="BookStore", containment="true", container="false", resolveProxies="true", eReferenceType="Book" + EGenericType #2: eRawType="Book", eClassifier="Book" + EClass #1: name="Book", ePackage="BookStorePackage", abstract="false", interface="false", eAllAttributes=["name", "isbn"], eAttributes=["name", "isbn"], eAllStructuralFeatures=["name", "isbn"], eIDAttribute="isbn", eStructuralFeatures=["name", "isbn"] + EAttribute #2: name="name", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #3, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Book", iD="false", eAttributeType="EString" + EGenericType #3: eRawType="EString", eClassifier="EString" + EAttribute #3: name="isbn", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EInt", eGenericType=EGenericType #4, changeable="true", volatile="false", transient="false", defaultValue="0", unsettable="false", derived="false", eContainingClass="Book", iD="true", eAttributeType="EInt" + EGenericType #4: eRawType="EInt", eClassifier="EInt" diff --git a/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreExtended.ecore.treeview b/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreExtended.ecore.treeview index 4fdd2c505..d70740d4f 100644 --- a/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreExtended.ecore.treeview +++ b/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreExtended.ecore.treeview @@ -1,35 +1,35 @@ -BookStorePackage : EPackage - store : EPackage - BookStore : EClass - owner : EReference - EGenericType - name : EAttribute - EGenericType - location : EAttribute - EGenericType - books : EReference - EGenericType - Book : EClass - name : EAttribute - EGenericType - isbn : EAttribute - EGenericType - author : EReference - EGenericType - genre : EAttribute - EGenericType - Genre : EEnum - NOVEL : EEnumLiteral - COOKBOOK : EEnumLiteral - BIOGRAPHY : EEnumLiteral - TEXTBOOK : EEnumLiteral - person : EPackage - Author : EClass - isStageName : EAttribute - EGenericType - EGenericType - Person : EClass - firstName : EAttribute - EGenericType - lastName : EAttribute - EGenericType +EPackage #0: name="BookStorePackage", nsURI="http:///com.ibm.dyna...", nsPrefix="bookStore", eFactoryInstance=EFactory #0, eSubpackages=["store", "person"] + EPackage #1: name="store", nsURI="http:///com.ibm.dyna...", nsPrefix="store", eFactoryInstance=EFactory #1, eClassifiers=["BookStore", "Book", "Genre"], eSuperPackage="BookStorePackage" + EClass #0: name="BookStore", ePackage="store", abstract="false", interface="false", eAllAttributes=["name", "location"], eAllReferences=["owner", "books"], eReferences=["owner", "books"], eAttributes=["name", "location"], eAllContainments=["books"], eAllStructuralFeatures=["owner", "name", "location", "books"], eStructuralFeatures=["owner", "name", "location", "books"] + EReference #0: name="owner", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="Person", eGenericType=EGenericType #0, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="BookStore", containment="false", container="false", resolveProxies="true", eReferenceType="Person" + EGenericType #0: eRawType="Person", eClassifier="Person" + EAttribute #0: name="name", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #1, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="BookStore", iD="false", eAttributeType="EString" + EGenericType #1: eRawType="EString", eClassifier="EString" + EAttribute #1: name="location", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #2, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="BookStore", iD="false", eAttributeType="EString" + EGenericType #2: eRawType="EString", eClassifier="EString" + EReference #1: name="books", ordered="true", unique="true", lowerBound="0", upperBound="-1", many="true", required="false", eType="Book", eGenericType=EGenericType #3, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="BookStore", containment="true", container="false", resolveProxies="true", eReferenceType="Book" + EGenericType #3: eRawType="Book", eClassifier="Book" + EClass #1: name="Book", ePackage="store", abstract="false", interface="false", eAllAttributes=["name", "isbn", "genre"], eAllReferences=["author"], eReferences=["author"], eAttributes=["name", "isbn", "genre"], eAllStructuralFeatures=["name", "isbn", "author", "genre"], eIDAttribute="isbn", eStructuralFeatures=["name", "isbn", "author", "genre"] + EAttribute #2: name="name", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #4, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Book", iD="false", eAttributeType="EString" + EGenericType #4: eRawType="EString", eClassifier="EString" + EAttribute #3: name="isbn", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EInt", eGenericType=EGenericType #5, changeable="true", volatile="false", transient="false", defaultValue="0", unsettable="false", derived="false", eContainingClass="Book", iD="true", eAttributeType="EInt" + EGenericType #5: eRawType="EInt", eClassifier="EInt" + EReference #2: name="author", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="Author", eGenericType=EGenericType #6, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Book", containment="false", container="false", resolveProxies="true", eReferenceType="Author" + EGenericType #6: eRawType="Author", eClassifier="Author" + EAttribute #4: name="genre", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="Genre", eGenericType=EGenericType #7, changeable="true", volatile="false", transient="false", defaultValue="NOVEL", unsettable="false", derived="false", eContainingClass="Book", iD="false", eAttributeType="Genre" + EGenericType #7: eRawType="Genre", eClassifier="Genre" + EEnum #0: name="Genre", defaultValue="NOVEL", ePackage="store", serializable="true", eLiterals=["NOVEL", "COOKBOOK", "BIOGRAPHY", "TEXTBOOK"] + EEnumLiteral #0: name="NOVEL", value="0", instance="NOVEL", literal="NOVEL", eEnum="Genre" + EEnumLiteral #1: name="COOKBOOK", value="1", instance="COOKBOOK", literal="COOKBOOK", eEnum="Genre" + EEnumLiteral #2: name="BIOGRAPHY", value="3", instance="BIOGRAPHY", literal="BIOGRAPHY", eEnum="Genre" + EEnumLiteral #3: name="TEXTBOOK", value="4", instance="TEXTBOOK", literal="TEXTBOOK", eEnum="Genre" + EPackage #2: name="person", nsURI="http:///com.ibm.dyna...", nsPrefix="person", eFactoryInstance=EFactory #2, eClassifiers=["Author", "Person"], eSuperPackage="BookStorePackage" + EClass #2: name="Author", ePackage="person", abstract="false", interface="false", eSuperTypes=["Person"], eAllAttributes=["firstName", "lastName", "isStageName"], eAttributes=["isStageName"], eAllStructuralFeatures=["firstName", "lastName", "isStageName"], eAllSuperTypes=["Person"], eStructuralFeatures=["isStageName"], eGenericSuperTypes=[EGenericType #8], eAllGenericSuperTypes=[EGenericType #8] + EAttribute #5: name="isStageName", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EBoolean", eGenericType=EGenericType #9, changeable="true", volatile="false", transient="false", defaultValue="false", unsettable="false", derived="false", eContainingClass="Author", iD="false", eAttributeType="EBoolean" + EGenericType #9: eRawType="EBoolean", eClassifier="EBoolean" + EGenericType #8: eRawType="Person", eClassifier="Person" + EClass #3: name="Person", ePackage="person", abstract="false", interface="false", eAllAttributes=["firstName", "lastName"], eAttributes=["firstName", "lastName"], eAllStructuralFeatures=["firstName", "lastName"], eStructuralFeatures=["firstName", "lastName"] + EAttribute #6: name="firstName", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #10, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Person", iD="false", eAttributeType="EString" + EGenericType #10: eRawType="EString", eClassifier="EString" + EAttribute #7: name="lastName", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #11, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Person", iD="false", eAttributeType="EString" + EGenericType #11: eRawType="EString", eClassifier="EString" diff --git a/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreExtendedRefactor.ecore.treeview b/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreExtendedRefactor.ecore.treeview index ca127fe5a..ee121f2ea 100644 --- a/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreExtendedRefactor.ecore.treeview +++ b/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreExtendedRefactor.ecore.treeview @@ -1,29 +1,29 @@ -BookStorePackage : EPackage - store : EPackage - Store : EClass - owner : EReference - EGenericType - name : EAttribute - EGenericType - location : EAttribute - EGenericType - BookStore : EClass - books : EReference - EGenericType - EGenericType - Book : EClass - title : EAttribute - EGenericType - isbn : EAttribute - EGenericType - author : EReference - EGenericType - category : EAttribute - EGenericType - Person : EClass - firstName : EAttribute - EGenericType - lastName : EAttribute - EGenericType - isStageName : EAttribute - EGenericType +EPackage #0: name="BookStorePackage", nsURI="http:///com.ibm.dyna...", nsPrefix="bookStore", eFactoryInstance=EFactory #0, eSubpackages=["store"] + EPackage #1: name="store", nsURI="http:///com.ibm.dyna...", nsPrefix="store", eFactoryInstance=EFactory #1, eClassifiers=["Store", "BookStore", "Book", "Person"], eSuperPackage="BookStorePackage" + EClass #0: name="Store", ePackage="store", abstract="false", interface="false", eAllAttributes=["name", "location"], eAllReferences=["owner"], eReferences=["owner"], eAttributes=["name", "location"], eAllStructuralFeatures=["owner", "name", "location"], eStructuralFeatures=["owner", "name", "location"] + EReference #0: name="owner", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="Person", eGenericType=EGenericType #0, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Store", containment="false", container="false", resolveProxies="true", eReferenceType="Person" + EGenericType #0: eRawType="Person", eClassifier="Person" + EAttribute #0: name="name", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #1, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Store", iD="false", eAttributeType="EString" + EGenericType #1: eRawType="EString", eClassifier="EString" + EAttribute #1: name="location", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #2, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Store", iD="false", eAttributeType="EString" + EGenericType #2: eRawType="EString", eClassifier="EString" + EClass #1: name="BookStore", ePackage="store", abstract="false", interface="false", eSuperTypes=["Store"], eAllAttributes=["name", "location"], eAllReferences=["owner", "books"], eReferences=["books"], eAllContainments=["books"], eAllStructuralFeatures=["owner", "name", "location", "books"], eAllSuperTypes=["Store"], eStructuralFeatures=["books"], eGenericSuperTypes=[EGenericType #3], eAllGenericSuperTypes=[EGenericType #3] + EReference #1: name="books", ordered="true", unique="true", lowerBound="0", upperBound="-1", many="true", required="false", eType="Book", eGenericType=EGenericType #4, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="BookStore", containment="true", container="false", resolveProxies="true", eReferenceType="Book" + EGenericType #4: eRawType="Book", eClassifier="Book" + EGenericType #3: eRawType="Store", eClassifier="Store" + EClass #2: name="Book", ePackage="store", abstract="false", interface="false", eAllAttributes=["title", "isbn", "category"], eAllReferences=["author"], eReferences=["author"], eAttributes=["title", "isbn", "category"], eAllStructuralFeatures=["title", "isbn", "author", "category"], eIDAttribute="isbn", eStructuralFeatures=["title", "isbn", "author", "category"] + EAttribute #2: name="title", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #5, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Book", iD="false", eAttributeType="EString" + EGenericType #5: eRawType="EString", eClassifier="EString" + EAttribute #3: name="isbn", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EInt", eGenericType=EGenericType #6, changeable="true", volatile="false", transient="false", defaultValue="0", unsettable="false", derived="false", eContainingClass="Book", iD="true", eAttributeType="EInt" + EGenericType #6: eRawType="EInt", eClassifier="EInt" + EReference #2: name="author", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="Person", eGenericType=EGenericType #7, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Book", containment="false", container="false", resolveProxies="true", eReferenceType="Person" + EGenericType #7: eRawType="Person", eClassifier="Person" + EAttribute #4: name="category", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #8, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Book", iD="false", eAttributeType="EString" + EGenericType #8: eRawType="EString", eClassifier="EString" + EClass #3: name="Person", ePackage="store", abstract="false", interface="false", eAllAttributes=["firstName", "lastName", "isStageName"], eAttributes=["firstName", "lastName", "isStageName"], eAllStructuralFeatures=["firstName", "lastName", "isStageName"], eStructuralFeatures=["firstName", "lastName", "isStageName"] + EAttribute #5: name="firstName", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #9, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Person", iD="false", eAttributeType="EString" + EGenericType #9: eRawType="EString", eClassifier="EString" + EAttribute #6: name="lastName", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #10, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Person", iD="false", eAttributeType="EString" + EGenericType #10: eRawType="EString", eClassifier="EString" + EAttribute #7: name="isStageName", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EBoolean", eGenericType=EGenericType #11, changeable="true", volatile="false", transient="false", defaultValue="false", unsettable="false", derived="false", eContainingClass="Person", iD="false", eAttributeType="EBoolean" + EGenericType #11: eRawType="EBoolean", eClassifier="EBoolean" diff --git a/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreRenamed.ecore.treeview b/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreRenamed.ecore.treeview index 4ee5941e3..43ab1ee1a 100644 --- a/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreRenamed.ecore.treeview +++ b/languages/emf-metamodel/src/test/resources/de/jplag/treeview/bookStoreRenamed.ecore.treeview @@ -1,13 +1,13 @@ -BookStorePackage : EPackage - Store : EClass - nameOfOwner : EAttribute - EGenericType - city : EAttribute - EGenericType - soldItems : EReference - EGenericType - Item : EClass - title : EAttribute - EGenericType - identifier : EAttribute - EGenericType +EPackage #0: name="BookStorePackage", nsURI="http:///com.ibm.dyna...", nsPrefix="bookStore", eFactoryInstance=EFactory #0, eClassifiers=["Store", "Item"] + EClass #0: name="Store", ePackage="BookStorePackage", abstract="false", interface="false", eAllAttributes=["nameOfOwner", "city"], eAllReferences=["soldItems"], eReferences=["soldItems"], eAttributes=["nameOfOwner", "city"], eAllContainments=["soldItems"], eAllStructuralFeatures=["nameOfOwner", "city", "soldItems"], eStructuralFeatures=["nameOfOwner", "city", "soldItems"] + EAttribute #0: name="nameOfOwner", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #0, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Store", iD="false", eAttributeType="EString" + EGenericType #0: eRawType="EString", eClassifier="EString" + EAttribute #1: name="city", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #1, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Store", iD="false", eAttributeType="EString" + EGenericType #1: eRawType="EString", eClassifier="EString" + EReference #0: name="soldItems", ordered="true", unique="true", lowerBound="0", upperBound="-1", many="true", required="false", eType="Item", eGenericType=EGenericType #2, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Store", containment="true", container="false", resolveProxies="true", eReferenceType="Item" + EGenericType #2: eRawType="Item", eClassifier="Item" + EClass #1: name="Item", ePackage="BookStorePackage", abstract="false", interface="false", eAllAttributes=["title", "identifier"], eAttributes=["title", "identifier"], eAllStructuralFeatures=["title", "identifier"], eIDAttribute="identifier", eStructuralFeatures=["title", "identifier"] + EAttribute #2: name="title", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EString", eGenericType=EGenericType #3, changeable="true", volatile="false", transient="false", unsettable="false", derived="false", eContainingClass="Item", iD="false", eAttributeType="EString" + EGenericType #3: eRawType="EString", eClassifier="EString" + EAttribute #3: name="identifier", ordered="true", unique="true", lowerBound="0", upperBound="1", many="false", required="false", eType="EInt", eGenericType=EGenericType #4, changeable="true", volatile="false", transient="false", defaultValue="0", unsettable="false", derived="false", eContainingClass="Item", iD="true", eAttributeType="EInt" + EGenericType #4: eRawType="EInt", eClassifier="EInt" diff --git a/languages/emf-model/README.md b/languages/emf-model/README.md new file mode 100644 index 000000000..0f63945ad --- /dev/null +++ b/languages/emf-model/README.md @@ -0,0 +1,24 @@ +# Dynamic EMF model language module +The dynamic EMF model language module allows the use of JPlag with model submissions. +It is based on the EMF API. + +### EMF specification compatibility +This module is based on the EMF dependencies available on maven central. These might not be the newest versions of EMF. For details, the [JPlag aggregator pom](https://github.com/jplag/JPlag/blob/263e85e544152cc8b0caa3399127debb7a458746/pom.xml#L84-L86). + +### Token Extraction +For the token extraction, we visit the containment tree of the model and extract tokens for all model elements based on their concrete metaclass. In this module, we thus extract tokens based on a dynamic token set. This works well for structural models with tree-like structures. It is less effective for models where the containment structure is not semantically relevant (e.g. state charts). These kinds of models require a dedicated language module. + +### Usage +The input for this is an EMF metamodel and a set of corresponding instances. +To ensure only the intended files are parsed, you can use `-p` to specify allowed file types: `-p ecore,xmi,mysuffix`. +To use this module, add the `-l emf-model` flag in the CLI, or use a `JPlagOption` object with `new DynamicEmfLanguage()` as `language` in the Java API as described in the usage information in the [readme of the main project](https://github.com/jplag/JPlag#usage) and [in the wiki](https://github.com/jplag/JPlag/wiki/1.-How-to-Use-JPlag). + +### Report Viewer +In the report viewer, a simple textual syntax is used to generate a tree-based model view. +To provide a custom visualization of a specific metamodel, a custom language module is required. + +### Literature +* [*"Token-based Plagiarism Detection for Metamodels" (MODELS-C'22)*](https://dl.acm.org/doi/10.1145/3550356.3556508). +* Its [Kudos Summary](https://www.growkudos.com/publications/10.1145%25252F3550356.3556508/reader). +* [*"Token-based Plagiarism Detection for Metamodels" (MODELS-C'22)*] +* *"Automated Detection of AI-Obfuscated Plagiarism in Modeling Assignments" (ICSE-SEET'24)* \ No newline at end of file diff --git a/languages/emf-model/src/main/java/de/jplag/emf/model/EmfModelLanguage.java b/languages/emf-model/src/main/java/de/jplag/emf/model/EmfModelLanguage.java index 36f328e67..e64036bfa 100644 --- a/languages/emf-model/src/main/java/de/jplag/emf/model/EmfModelLanguage.java +++ b/languages/emf-model/src/main/java/de/jplag/emf/model/EmfModelLanguage.java @@ -52,8 +52,6 @@ public boolean expectsSubmissionOrder() { @Override public List customizeSubmissionOrder(List sub) { - Comparator fileEndingComparator = (first, second) -> Boolean.compare(second.getName().endsWith(FILE_ENDING), - first.getName().endsWith(FILE_ENDING)); - return sub.stream().sorted(fileEndingComparator).toList(); + return sub.stream().sorted(Comparator.comparing(file -> file.getName().endsWith(FILE_ENDING) ? 0 : 1)).toList(); } } diff --git a/languages/emf-model/src/main/java/de/jplag/emf/model/parser/DynamicModelParser.java b/languages/emf-model/src/main/java/de/jplag/emf/model/parser/DynamicModelParser.java index 70d77e55e..e2d4ac3ed 100644 --- a/languages/emf-model/src/main/java/de/jplag/emf/model/parser/DynamicModelParser.java +++ b/languages/emf-model/src/main/java/de/jplag/emf/model/parser/DynamicModelParser.java @@ -14,7 +14,7 @@ import de.jplag.emf.model.EmfModelLanguage; import de.jplag.emf.util.AbstractModelView; import de.jplag.emf.util.EMFUtil; -import de.jplag.emf.util.MetamodelTreeView; +import de.jplag.emf.util.GenericEmfTreeView; /** * Parser for EMF metamodels based on dynamically created tokens. @@ -36,7 +36,7 @@ public DynamicModelParser() { } @Override - protected void parseModelFile(File file) throws ParsingException { + protected void parseModelFile(File file, boolean normalize) throws ParsingException { // implicit assumption: Metamodel gets parsed first! if (file.getName().endsWith(EmfLanguage.FILE_ENDING)) { parseMetamodelFile(file); @@ -46,7 +46,7 @@ protected void parseModelFile(File file) throws ParsingException { if (metapackages.isEmpty()) { logger.warn(METAPACKAGE_WARNING, file.getName()); } - super.parseModelFile(file); + super.parseModelFile(file, normalize); } } @@ -57,7 +57,7 @@ protected String getCorrespondingViewFileSuffix() { @Override protected AbstractModelView createView(File file, Resource modelResource) { - return new MetamodelTreeView(file, modelResource); + return new GenericEmfTreeView(file, modelResource); } private void parseMetamodelFile(File file) throws ParsingException { @@ -65,15 +65,14 @@ private void parseMetamodelFile(File file) throws ParsingException { Resource modelResource = EMFUtil.loadModelResource(file); if (modelResource == null) { throw new ParsingException(file, METAMODEL_LOADING_ERROR); - } else { - for (EObject object : modelResource.getContents()) { - if (object instanceof EPackage ePackage) { - metapackages.add(ePackage); - } else { - logger.error(METAPACKAGE_ERROR, object); - } + } + for (EObject object : modelResource.getContents()) { + if (object instanceof EPackage ePackage) { + metapackages.add(ePackage); + } else { + logger.error(METAPACKAGE_ERROR, object); } - EMFUtil.registerEPackageURIs(metapackages); } + EMFUtil.registerEPackageURIs(metapackages); } } diff --git a/languages/emf-model/src/test/java/de/jplag/emf/model/MinimalModelInstanceTest.java b/languages/emf-model/src/test/java/de/jplag/emf/model/MinimalModelInstanceTest.java index 2c13e44a5..2ff7001ad 100644 --- a/languages/emf-model/src/test/java/de/jplag/emf/model/MinimalModelInstanceTest.java +++ b/languages/emf-model/src/test/java/de/jplag/emf/model/MinimalModelInstanceTest.java @@ -8,9 +8,9 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; -import java.util.TreeSet; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -22,7 +22,6 @@ import de.jplag.ParsingException; import de.jplag.Token; import de.jplag.TokenPrinter; -import de.jplag.emf.EmfLanguage; import de.jplag.testutils.FileUtil; class MinimalModelInstanceTest { @@ -46,11 +45,11 @@ public void setUp() { void testBookStoreInstances() { File baseFile = new File(BASE_PATH.toString()); List baseFiles = new ArrayList<>(Arrays.asList(baseFile.listFiles())); - var sortedFiles = new TreeSet<>(language.customizeSubmissionOrder(baseFiles)); + var sortedFiles = new LinkedHashSet<>(language.customizeSubmissionOrder(baseFiles)); try { - List tokens = language.parse(sortedFiles); + List tokens = language.parse(sortedFiles, true); assertNotEquals(0, tokens.size()); - logger.debug(TokenPrinter.printTokens(tokens, baseDirectory, Optional.of(EmfLanguage.VIEW_FILE_SUFFIX))); + logger.debug(TokenPrinter.printTokens(tokens, baseDirectory, Optional.of(EmfModelLanguage.VIEW_FILE_SUFFIX))); logger.info("Parsed tokens: " + tokens); assertEquals(7, tokens.size()); } catch (ParsingException e) { diff --git a/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java b/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java index 581a57a19..3dbd09ec4 100644 --- a/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java +++ b/languages/golang/src/main/java/de/jplag/golang/GoLanguage.java @@ -43,7 +43,7 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { return parserAdapter.parse(files); } } diff --git a/languages/golang/src/main/java/de/jplag/golang/JPlagGoListener.java b/languages/golang/src/main/java/de/jplag/golang/JPlagGoListener.java index 504e2337e..8774fb81d 100644 --- a/languages/golang/src/main/java/de/jplag/golang/JPlagGoListener.java +++ b/languages/golang/src/main/java/de/jplag/golang/JPlagGoListener.java @@ -1,8 +1,79 @@ package de.jplag.golang; -import static de.jplag.golang.GoTokenType.*; - -import java.util.*; +import static de.jplag.golang.GoTokenType.ARGUMENT; +import static de.jplag.golang.GoTokenType.ARRAY_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.ARRAY_BODY_END; +import static de.jplag.golang.GoTokenType.ARRAY_CONSTRUCTOR; +import static de.jplag.golang.GoTokenType.ARRAY_ELEMENT; +import static de.jplag.golang.GoTokenType.ASSIGNMENT; +import static de.jplag.golang.GoTokenType.BREAK; +import static de.jplag.golang.GoTokenType.CASE_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.CASE_BLOCK_END; +import static de.jplag.golang.GoTokenType.CONTINUE; +import static de.jplag.golang.GoTokenType.DEFER; +import static de.jplag.golang.GoTokenType.ELSE_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.ELSE_BLOCK_END; +import static de.jplag.golang.GoTokenType.FALLTHROUGH; +import static de.jplag.golang.GoTokenType.FOR_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.FOR_BLOCK_END; +import static de.jplag.golang.GoTokenType.FOR_STATEMENT; +import static de.jplag.golang.GoTokenType.FUNCTION_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.FUNCTION_BODY_END; +import static de.jplag.golang.GoTokenType.FUNCTION_DECLARATION; +import static de.jplag.golang.GoTokenType.FUNCTION_LITERAL; +import static de.jplag.golang.GoTokenType.FUNCTION_PARAMETER; +import static de.jplag.golang.GoTokenType.GO; +import static de.jplag.golang.GoTokenType.GOTO; +import static de.jplag.golang.GoTokenType.IF_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.IF_BLOCK_END; +import static de.jplag.golang.GoTokenType.IF_STATEMENT; +import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE; +import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE_BEGIN; +import static de.jplag.golang.GoTokenType.IMPORT_CLAUSE_END; +import static de.jplag.golang.GoTokenType.IMPORT_DECLARATION; +import static de.jplag.golang.GoTokenType.INTERFACE_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.INTERFACE_BLOCK_END; +import static de.jplag.golang.GoTokenType.INTERFACE_DECLARATION; +import static de.jplag.golang.GoTokenType.INTERFACE_METHOD; +import static de.jplag.golang.GoTokenType.INVOCATION; +import static de.jplag.golang.GoTokenType.MAP_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.MAP_BODY_END; +import static de.jplag.golang.GoTokenType.MAP_CONSTRUCTOR; +import static de.jplag.golang.GoTokenType.MAP_ELEMENT; +import static de.jplag.golang.GoTokenType.MEMBER_DECLARATION; +import static de.jplag.golang.GoTokenType.NAMED_TYPE_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.NAMED_TYPE_BODY_END; +import static de.jplag.golang.GoTokenType.NAMED_TYPE_CONSTRUCTOR; +import static de.jplag.golang.GoTokenType.NAMED_TYPE_ELEMENT; +import static de.jplag.golang.GoTokenType.PACKAGE; +import static de.jplag.golang.GoTokenType.RECEIVER; +import static de.jplag.golang.GoTokenType.RECEIVE_STATEMENT; +import static de.jplag.golang.GoTokenType.RETURN; +import static de.jplag.golang.GoTokenType.SELECT_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.SELECT_BLOCK_END; +import static de.jplag.golang.GoTokenType.SELECT_STATEMENT; +import static de.jplag.golang.GoTokenType.SEND_STATEMENT; +import static de.jplag.golang.GoTokenType.SLICE_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.SLICE_BODY_END; +import static de.jplag.golang.GoTokenType.SLICE_CONSTRUCTOR; +import static de.jplag.golang.GoTokenType.SLICE_ELEMENT; +import static de.jplag.golang.GoTokenType.STATEMENT_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.STATEMENT_BLOCK_END; +import static de.jplag.golang.GoTokenType.STRUCT_BODY_BEGIN; +import static de.jplag.golang.GoTokenType.STRUCT_BODY_END; +import static de.jplag.golang.GoTokenType.STRUCT_DECLARATION; +import static de.jplag.golang.GoTokenType.SWITCH_BLOCK_BEGIN; +import static de.jplag.golang.GoTokenType.SWITCH_BLOCK_END; +import static de.jplag.golang.GoTokenType.SWITCH_CASE; +import static de.jplag.golang.GoTokenType.SWITCH_STATEMENT; +import static de.jplag.golang.GoTokenType.TYPE_ASSERTION; +import static de.jplag.golang.GoTokenType.TYPE_CONSTRAINT; +import static de.jplag.golang.GoTokenType.VARIABLE_DECLARATION; + +import java.util.Arrays; +import java.util.Deque; +import java.util.LinkedList; +import java.util.Optional; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.TerminalNode; diff --git a/languages/java/src/main/java/de/jplag/java/JavaLanguage.java b/languages/java/src/main/java/de/jplag/java/JavaLanguage.java index 14172b8c6..4db88ef01 100644 --- a/languages/java/src/main/java/de/jplag/java/JavaLanguage.java +++ b/languages/java/src/main/java/de/jplag/java/JavaLanguage.java @@ -44,7 +44,7 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { return this.parser.parse(files); } @@ -53,6 +53,11 @@ public boolean tokensHaveSemantics() { return true; } + @Override + public boolean supportsNormalization() { + return true; + } + @Override public String toString() { return this.getIdentifier(); diff --git a/languages/java/src/main/java/de/jplag/java/JavacAdapter.java b/languages/java/src/main/java/de/jplag/java/JavacAdapter.java index c9127e39e..f1c5c2dc9 100644 --- a/languages/java/src/main/java/de/jplag/java/JavacAdapter.java +++ b/languages/java/src/main/java/de/jplag/java/JavacAdapter.java @@ -6,7 +6,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.Set; import javax.tools.DiagnosticCollector; @@ -51,13 +50,12 @@ public void parseFiles(Set files, final Parser parser) throws ParsingExcep final LineMap map = ast.getLineMap(); var scanner = new TokenGeneratingTreeScanner(file, parser, map, positions, ast); ast.accept(scanner, null); - parsingExceptions.addAll(scanner.getParsingExceptions()); parser.add(Token.semanticFileEnd(file)); } } catch (IOException exception) { throw new ParsingException(null, exception.getMessage(), exception); } - parsingExceptions.addAll(processErrors(parser.logger, listener)); + parsingExceptions.addAll(processErrors(listener)); if (!parsingExceptions.isEmpty()) { throw ParsingException.wrappingExceptions(parsingExceptions); } @@ -73,14 +71,13 @@ private Iterable executeCompilationTask(final Com return abstractSyntaxTrees; } - private List processErrors(Logger logger, DiagnosticCollector listener) { + private List processErrors(DiagnosticCollector listener) { return listener.getDiagnostics().stream().filter(it -> it.getKind() == javax.tools.Diagnostic.Kind.ERROR).map(diagnosticItem -> { File file = null; if (diagnosticItem.getSource() instanceof JavaFileObject fileObject) { file = new File(fileObject.toUri()); } - logger.error("{}", diagnosticItem); - return new ParsingException(file, diagnosticItem.getMessage(Locale.getDefault())); + return new ParsingException(file, diagnosticItem.toString()); }).toList(); } diff --git a/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java b/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java index c8ab37c82..28bd5838a 100644 --- a/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java +++ b/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java @@ -1,11 +1,8 @@ package de.jplag.java; import java.io.File; -import java.util.ArrayList; -import java.util.List; import java.util.Set; -import de.jplag.ParsingException; import de.jplag.Token; import de.jplag.TokenType; import de.jplag.semantics.CodeSemantics; @@ -28,7 +25,6 @@ import com.sun.source.tree.DefaultCaseLabelTree; import com.sun.source.tree.DoWhileLoopTree; import com.sun.source.tree.EnhancedForLoopTree; -import com.sun.source.tree.ErroneousTree; import com.sun.source.tree.ExportsTree; import com.sun.source.tree.ForLoopTree; import com.sun.source.tree.IdentifierTree; @@ -68,8 +64,6 @@ final class TokenGeneratingTreeScanner extends TreeScanner { private final SourcePositions positions; private final CompilationUnitTree ast; - private final List parsingExceptions = new ArrayList<>(); - private final VariableRegistry variableRegistry; private static final Set IMMUTABLES = Set.of( @@ -88,10 +82,6 @@ public TokenGeneratingTreeScanner(File file, Parser parser, LineMap map, SourceP this.variableRegistry = new VariableRegistry(); } - public List getParsingExceptions() { - return parsingExceptions; - } - public void addToken(TokenType type, File file, long line, long column, long length, CodeSemantics semantics) { parser.add(new Token(type, file, (int) line, (int) column, (int) length, semantics)); variableRegistry.updateSemantics(semantics); @@ -547,12 +537,6 @@ public Void visitExports(ExportsTree node, Void unused) { return super.visitExports(node, null); } - @Override - public Void visitErroneous(ErroneousTree node, Void unused) { - parsingExceptions.add(new ParsingException(file, "error while visiting %s".formatted(node))); - return super.visitErroneous(node, null); - } - @Override public Void visitYield(YieldTree node, Void unused) { long start = positions.getStartPosition(ast, node); diff --git a/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java b/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java index cf559d293..e54955781 100644 --- a/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java +++ b/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java @@ -1,6 +1,33 @@ package de.jplag.java; -import static de.jplag.java.JavaTokenType.*; +import static de.jplag.java.JavaTokenType.J_APPLY; +import static de.jplag.java.JavaTokenType.J_ARRAY_INIT_BEGIN; +import static de.jplag.java.JavaTokenType.J_ARRAY_INIT_END; +import static de.jplag.java.JavaTokenType.J_ASSIGN; +import static de.jplag.java.JavaTokenType.J_CATCH_BEGIN; +import static de.jplag.java.JavaTokenType.J_CATCH_END; +import static de.jplag.java.JavaTokenType.J_CLASS_BEGIN; +import static de.jplag.java.JavaTokenType.J_CLASS_END; +import static de.jplag.java.JavaTokenType.J_COND; +import static de.jplag.java.JavaTokenType.J_FINALLY_BEGIN; +import static de.jplag.java.JavaTokenType.J_FINALLY_END; +import static de.jplag.java.JavaTokenType.J_IF_BEGIN; +import static de.jplag.java.JavaTokenType.J_IF_END; +import static de.jplag.java.JavaTokenType.J_IMPORT; +import static de.jplag.java.JavaTokenType.J_LOOP_BEGIN; +import static de.jplag.java.JavaTokenType.J_LOOP_END; +import static de.jplag.java.JavaTokenType.J_METHOD_BEGIN; +import static de.jplag.java.JavaTokenType.J_METHOD_END; +import static de.jplag.java.JavaTokenType.J_NEWARRAY; +import static de.jplag.java.JavaTokenType.J_NEWCLASS; +import static de.jplag.java.JavaTokenType.J_PACKAGE; +import static de.jplag.java.JavaTokenType.J_RECORD_BEGIN; +import static de.jplag.java.JavaTokenType.J_RECORD_END; +import static de.jplag.java.JavaTokenType.J_RETURN; +import static de.jplag.java.JavaTokenType.J_THROW; +import static de.jplag.java.JavaTokenType.J_TRY_BEGIN; +import static de.jplag.java.JavaTokenType.J_TRY_END; +import static de.jplag.java.JavaTokenType.J_VARDEF; import de.jplag.testutils.LanguageModuleTest; import de.jplag.testutils.datacollector.TestDataCollector; diff --git a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java index 19a475ffa..d0dae5d67 100644 --- a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java +++ b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinListener.java @@ -54,47 +54,47 @@ import static de.jplag.kotlin.KotlinTokenType.WHEN_EXPRESSION_START; import static de.jplag.kotlin.KotlinTokenType.WHILE_EXPRESSION_END; import static de.jplag.kotlin.KotlinTokenType.WHILE_EXPRESSION_START; -import static de.jplag.kotlin.grammar.KotlinParser.AnonymousInitializerContext; -import static de.jplag.kotlin.grammar.KotlinParser.AssignmentOperatorContext; -import static de.jplag.kotlin.grammar.KotlinParser.CallSuffixContext; -import static de.jplag.kotlin.grammar.KotlinParser.CatchBodyContext; -import static de.jplag.kotlin.grammar.KotlinParser.CatchStatementContext; -import static de.jplag.kotlin.grammar.KotlinParser.ClassBodyContext; -import static de.jplag.kotlin.grammar.KotlinParser.ClassDeclarationContext; -import static de.jplag.kotlin.grammar.KotlinParser.ClassParameterContext; -import static de.jplag.kotlin.grammar.KotlinParser.CompanionObjectContext; -import static de.jplag.kotlin.grammar.KotlinParser.ConstructorInvocationContext; -import static de.jplag.kotlin.grammar.KotlinParser.ControlStructureBodyContext; -import static de.jplag.kotlin.grammar.KotlinParser.DoWhileExpressionContext; -import static de.jplag.kotlin.grammar.KotlinParser.EnumClassBodyContext; -import static de.jplag.kotlin.grammar.KotlinParser.EnumEntryContext; -import static de.jplag.kotlin.grammar.KotlinParser.FinallyBodyContext; -import static de.jplag.kotlin.grammar.KotlinParser.FinallyStatementContext; -import static de.jplag.kotlin.grammar.KotlinParser.ForExpressionContext; -import static de.jplag.kotlin.grammar.KotlinParser.FunctionBodyContext; -import static de.jplag.kotlin.grammar.KotlinParser.FunctionDeclarationContext; -import static de.jplag.kotlin.grammar.KotlinParser.FunctionLiteralContext; -import static de.jplag.kotlin.grammar.KotlinParser.FunctionValueParameterContext; -import static de.jplag.kotlin.grammar.KotlinParser.GetterContext; -import static de.jplag.kotlin.grammar.KotlinParser.IfExpressionContext; -import static de.jplag.kotlin.grammar.KotlinParser.ImportHeaderContext; -import static de.jplag.kotlin.grammar.KotlinParser.InitBlockContext; -import static de.jplag.kotlin.grammar.KotlinParser.ObjectDeclarationContext; -import static de.jplag.kotlin.grammar.KotlinParser.PackageHeaderContext; -import static de.jplag.kotlin.grammar.KotlinParser.PrimaryConstructorContext; -import static de.jplag.kotlin.grammar.KotlinParser.PropertyDeclarationContext; -import static de.jplag.kotlin.grammar.KotlinParser.SecondaryConstructorContext; -import static de.jplag.kotlin.grammar.KotlinParser.SetterContext; -import static de.jplag.kotlin.grammar.KotlinParser.TryBodyContext; -import static de.jplag.kotlin.grammar.KotlinParser.TryExpressionContext; -import static de.jplag.kotlin.grammar.KotlinParser.TypeParameterContext; -import static de.jplag.kotlin.grammar.KotlinParser.VariableDeclarationContext; -import static de.jplag.kotlin.grammar.KotlinParser.WhenConditionContext; -import static de.jplag.kotlin.grammar.KotlinParser.WhenExpressionContext; -import static de.jplag.kotlin.grammar.KotlinParser.WhileExpressionContext; import de.jplag.antlr.AbstractAntlrListener; import de.jplag.kotlin.grammar.KotlinParser; +import de.jplag.kotlin.grammar.KotlinParser.AnonymousInitializerContext; +import de.jplag.kotlin.grammar.KotlinParser.AssignmentOperatorContext; +import de.jplag.kotlin.grammar.KotlinParser.CallSuffixContext; +import de.jplag.kotlin.grammar.KotlinParser.CatchBodyContext; +import de.jplag.kotlin.grammar.KotlinParser.CatchStatementContext; +import de.jplag.kotlin.grammar.KotlinParser.ClassBodyContext; +import de.jplag.kotlin.grammar.KotlinParser.ClassDeclarationContext; +import de.jplag.kotlin.grammar.KotlinParser.ClassParameterContext; +import de.jplag.kotlin.grammar.KotlinParser.CompanionObjectContext; +import de.jplag.kotlin.grammar.KotlinParser.ConstructorInvocationContext; +import de.jplag.kotlin.grammar.KotlinParser.ControlStructureBodyContext; +import de.jplag.kotlin.grammar.KotlinParser.DoWhileExpressionContext; +import de.jplag.kotlin.grammar.KotlinParser.EnumClassBodyContext; +import de.jplag.kotlin.grammar.KotlinParser.EnumEntryContext; +import de.jplag.kotlin.grammar.KotlinParser.FinallyBodyContext; +import de.jplag.kotlin.grammar.KotlinParser.FinallyStatementContext; +import de.jplag.kotlin.grammar.KotlinParser.ForExpressionContext; +import de.jplag.kotlin.grammar.KotlinParser.FunctionBodyContext; +import de.jplag.kotlin.grammar.KotlinParser.FunctionDeclarationContext; +import de.jplag.kotlin.grammar.KotlinParser.FunctionLiteralContext; +import de.jplag.kotlin.grammar.KotlinParser.FunctionValueParameterContext; +import de.jplag.kotlin.grammar.KotlinParser.GetterContext; +import de.jplag.kotlin.grammar.KotlinParser.IfExpressionContext; +import de.jplag.kotlin.grammar.KotlinParser.ImportHeaderContext; +import de.jplag.kotlin.grammar.KotlinParser.InitBlockContext; +import de.jplag.kotlin.grammar.KotlinParser.ObjectDeclarationContext; +import de.jplag.kotlin.grammar.KotlinParser.PackageHeaderContext; +import de.jplag.kotlin.grammar.KotlinParser.PrimaryConstructorContext; +import de.jplag.kotlin.grammar.KotlinParser.PropertyDeclarationContext; +import de.jplag.kotlin.grammar.KotlinParser.SecondaryConstructorContext; +import de.jplag.kotlin.grammar.KotlinParser.SetterContext; +import de.jplag.kotlin.grammar.KotlinParser.TryBodyContext; +import de.jplag.kotlin.grammar.KotlinParser.TryExpressionContext; +import de.jplag.kotlin.grammar.KotlinParser.TypeParameterContext; +import de.jplag.kotlin.grammar.KotlinParser.VariableDeclarationContext; +import de.jplag.kotlin.grammar.KotlinParser.WhenConditionContext; +import de.jplag.kotlin.grammar.KotlinParser.WhenExpressionContext; +import de.jplag.kotlin.grammar.KotlinParser.WhileExpressionContext; class KotlinListener extends AbstractAntlrListener { diff --git a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java index 0fed85032..ce8762742 100644 --- a/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java +++ b/languages/kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java @@ -1,6 +1,9 @@ package de.jplag.kotlin; -import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.ParserRuleContext; import de.jplag.antlr.AbstractAntlrListener; import de.jplag.antlr.AbstractAntlrParserAdapter; diff --git a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java index 24e70a9ca..9ce933926 100644 --- a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java +++ b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRListener.java @@ -1,116 +1,65 @@ package de.jplag.llvmir; -import static de.jplag.llvmir.LLVMIRTokenType.*; -import static de.jplag.llvmir.grammar.LLVMIRParser.AShrExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.AShrInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.AddExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.AddInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.AddrSpaceCastExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.AddrSpaceCastInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.AllocaInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.AndExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.AndInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ArrayConstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.AtomicOrderingContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.AtomicRMWInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.BasicBlockContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.BitCastExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.BitCastInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.BrTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.CallBrTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.CallInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.Case_Context; -import static de.jplag.llvmir.grammar.LLVMIRParser.CatchPadInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.CatchRetTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.CatchSwitchTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ClauseContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.CleanupPadInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.CleanupRetTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.CmpXchgInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.CondBrTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ExtractElementExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ExtractElementInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ExtractValueInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FAddInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FCmpExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FCmpInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FDivInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FMulInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FRemInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FSubInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FenceInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FpExtExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FpExtInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FpToSiExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FpToSiInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FpToUiExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FpToUiInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FpTruncExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FpTruncInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FuncBodyContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FuncDeclContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.FuncDefContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.GetElementPtrExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.GetElementPtrInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.GlobalDeclContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.GlobalDefContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ICmpExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ICmpInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.IndirectBrTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.InlineAsmContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.InsertElementExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.InsertElementInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.InsertValueInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.IntToPtrExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.IntToPtrInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.InvokeTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.LShrExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.LShrInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.LandingPadInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.LoadInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ModuleAsmContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.MulExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.MulInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.OrExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.OrInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.PhiInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.PtrToIntExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.PtrToIntInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ResumeTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.RetTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SDivInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SExtExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SExtInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SRemInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SelectExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SelectInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ShlExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ShlInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ShuffleVectorExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ShuffleVectorInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SiToFpExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SiToFpInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SourceFilenameContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.StoreInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.StructConstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SubExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SubInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.SwitchTermContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.TruncExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.TruncInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.TypeDefContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.UDivInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.URemInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.UiToFpExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.UiToFpInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.VaargInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.VectorConstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.XorExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.XorInstContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ZExtExprContext; -import static de.jplag.llvmir.grammar.LLVMIRParser.ZExtInstContext; +import static de.jplag.llvmir.LLVMIRTokenType.ADDITION; +import static de.jplag.llvmir.LLVMIRTokenType.ALLOCATION; +import static de.jplag.llvmir.LLVMIRTokenType.AND; +import static de.jplag.llvmir.LLVMIRTokenType.ARRAY; +import static de.jplag.llvmir.LLVMIRTokenType.ASSEMBLY; +import static de.jplag.llvmir.LLVMIRTokenType.ATOMIC_ORDERING; +import static de.jplag.llvmir.LLVMIRTokenType.ATOMIC_READ_MODIFY_WRITE; +import static de.jplag.llvmir.LLVMIRTokenType.BASIC_BLOCK_BEGIN; +import static de.jplag.llvmir.LLVMIRTokenType.BASIC_BLOCK_END; +import static de.jplag.llvmir.LLVMIRTokenType.BITCAST; +import static de.jplag.llvmir.LLVMIRTokenType.BRANCH; +import static de.jplag.llvmir.LLVMIRTokenType.CALL; +import static de.jplag.llvmir.LLVMIRTokenType.CALL_BRANCH; +import static de.jplag.llvmir.LLVMIRTokenType.CASE; +import static de.jplag.llvmir.LLVMIRTokenType.CATCH_PAD; +import static de.jplag.llvmir.LLVMIRTokenType.CATCH_RETURN; +import static de.jplag.llvmir.LLVMIRTokenType.CATCH_SWITCH; +import static de.jplag.llvmir.LLVMIRTokenType.CLAUSE; +import static de.jplag.llvmir.LLVMIRTokenType.CLEAN_UP_PAD; +import static de.jplag.llvmir.LLVMIRTokenType.CLEAN_UP_RETURN; +import static de.jplag.llvmir.LLVMIRTokenType.COMPARE_EXCHANGE; +import static de.jplag.llvmir.LLVMIRTokenType.COMPARISON; +import static de.jplag.llvmir.LLVMIRTokenType.CONDITIONAL_BRANCH; +import static de.jplag.llvmir.LLVMIRTokenType.CONVERSION; +import static de.jplag.llvmir.LLVMIRTokenType.DIVISION; +import static de.jplag.llvmir.LLVMIRTokenType.EXTRACT_ELEMENT; +import static de.jplag.llvmir.LLVMIRTokenType.EXTRACT_VALUE; +import static de.jplag.llvmir.LLVMIRTokenType.FENCE; +import static de.jplag.llvmir.LLVMIRTokenType.FILENAME; +import static de.jplag.llvmir.LLVMIRTokenType.FUNCTION_BODY_BEGIN; +import static de.jplag.llvmir.LLVMIRTokenType.FUNCTION_BODY_END; +import static de.jplag.llvmir.LLVMIRTokenType.FUNCTION_DECLARATION; +import static de.jplag.llvmir.LLVMIRTokenType.FUNCTION_DEFINITION; +import static de.jplag.llvmir.LLVMIRTokenType.GET_ELEMENT_POINTER; +import static de.jplag.llvmir.LLVMIRTokenType.GLOBAL_VARIABLE; +import static de.jplag.llvmir.LLVMIRTokenType.INSERT_ELEMENT; +import static de.jplag.llvmir.LLVMIRTokenType.INSERT_VALUE; +import static de.jplag.llvmir.LLVMIRTokenType.INVOKE; +import static de.jplag.llvmir.LLVMIRTokenType.LANDING_PAD; +import static de.jplag.llvmir.LLVMIRTokenType.LOAD; +import static de.jplag.llvmir.LLVMIRTokenType.MULTIPLICATION; +import static de.jplag.llvmir.LLVMIRTokenType.OR; +import static de.jplag.llvmir.LLVMIRTokenType.PHI; +import static de.jplag.llvmir.LLVMIRTokenType.REMAINDER; +import static de.jplag.llvmir.LLVMIRTokenType.RESUME; +import static de.jplag.llvmir.LLVMIRTokenType.RETURN; +import static de.jplag.llvmir.LLVMIRTokenType.SELECT; +import static de.jplag.llvmir.LLVMIRTokenType.SHIFT; +import static de.jplag.llvmir.LLVMIRTokenType.SHUFFLE_VECTOR; +import static de.jplag.llvmir.LLVMIRTokenType.STORE; +import static de.jplag.llvmir.LLVMIRTokenType.STRUCTURE; +import static de.jplag.llvmir.LLVMIRTokenType.SUBTRACTION; +import static de.jplag.llvmir.LLVMIRTokenType.SWITCH; +import static de.jplag.llvmir.LLVMIRTokenType.TYPE_DEFINITION; +import static de.jplag.llvmir.LLVMIRTokenType.VARIABLE_ARGUMENT; +import static de.jplag.llvmir.LLVMIRTokenType.VECTOR; +import static de.jplag.llvmir.LLVMIRTokenType.XOR; import de.jplag.antlr.AbstractAntlrListener; +import de.jplag.llvmir.grammar.LLVMIRParser.*; /** * Extracts tokens from the ANTLR parse tree. The token abstraction includes nesting tokens for functions and basic diff --git a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRParserAdapter.java b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRParserAdapter.java index edbe94148..0365236e0 100644 --- a/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRParserAdapter.java +++ b/languages/llvmir/src/main/java/de/jplag/llvmir/LLVMIRParserAdapter.java @@ -1,6 +1,9 @@ package de.jplag.llvmir; -import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.ParserRuleContext; import de.jplag.AbstractParser; import de.jplag.antlr.AbstractAntlrListener; diff --git a/languages/llvmir/src/test/java/de/jplag/llvmir/LLVMIRLanguageTest.java b/languages/llvmir/src/test/java/de/jplag/llvmir/LLVMIRLanguageTest.java index b34956af9..1a2c7e68b 100644 --- a/languages/llvmir/src/test/java/de/jplag/llvmir/LLVMIRLanguageTest.java +++ b/languages/llvmir/src/test/java/de/jplag/llvmir/LLVMIRLanguageTest.java @@ -1,6 +1,10 @@ package de.jplag.llvmir; -import static de.jplag.llvmir.LLVMIRTokenType.*; +import static de.jplag.llvmir.LLVMIRTokenType.CATCH_PAD; +import static de.jplag.llvmir.LLVMIRTokenType.CATCH_RETURN; +import static de.jplag.llvmir.LLVMIRTokenType.CATCH_SWITCH; +import static de.jplag.llvmir.LLVMIRTokenType.CLEAN_UP_PAD; +import static de.jplag.llvmir.LLVMIRTokenType.CLEAN_UP_RETURN; import java.util.Arrays; import java.util.List; diff --git a/languages/pom.xml b/languages/pom.xml index f2dba9a5d..819b1f491 100644 --- a/languages/pom.xml +++ b/languages/pom.xml @@ -9,8 +9,8 @@ languages pom + c cpp - cpp2 csharp emf-metamodel emf-metamodel-dynamic diff --git a/languages/python-3/src/main/java/de/jplag/python3/PythonLanguage.java b/languages/python-3/src/main/java/de/jplag/python3/PythonLanguage.java index 0140a37f7..b5a8fd73f 100644 --- a/languages/python-3/src/main/java/de/jplag/python3/PythonLanguage.java +++ b/languages/python-3/src/main/java/de/jplag/python3/PythonLanguage.java @@ -41,7 +41,7 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { return this.parser.parse(files); } } diff --git a/languages/python-3/src/main/java/de/jplag/python3/grammar/Python3LexerBase.java b/languages/python-3/src/main/java/de/jplag/python3/grammar/Python3LexerBase.java index a60956d7c..0e24adf20 100644 --- a/languages/python-3/src/main/java/de/jplag/python3/grammar/Python3LexerBase.java +++ b/languages/python-3/src/main/java/de/jplag/python3/grammar/Python3LexerBase.java @@ -1,8 +1,12 @@ package de.jplag.python3.grammar; -import java.util.*; +import java.util.Deque; +import java.util.LinkedList; -import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CommonToken; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.Token; abstract class Python3LexerBase extends Lexer { private LinkedList tokens = new LinkedList<>(); diff --git a/languages/python-3/src/main/java/de/jplag/python3/grammar/Python3ParserBase.java b/languages/python-3/src/main/java/de/jplag/python3/grammar/Python3ParserBase.java index 327dd044c..44b5926a4 100644 --- a/languages/python-3/src/main/java/de/jplag/python3/grammar/Python3ParserBase.java +++ b/languages/python-3/src/main/java/de/jplag/python3/grammar/Python3ParserBase.java @@ -1,6 +1,7 @@ package de.jplag.python3.grammar; -import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.TokenStream; public abstract class Python3ParserBase extends Parser { protected Python3ParserBase(TokenStream input) { diff --git a/languages/rlang/src/main/java/de/jplag/rlang/RLanguage.java b/languages/rlang/src/main/java/de/jplag/rlang/RLanguage.java index 182b856d2..d09e23b72 100644 --- a/languages/rlang/src/main/java/de/jplag/rlang/RLanguage.java +++ b/languages/rlang/src/main/java/de/jplag/rlang/RLanguage.java @@ -46,7 +46,7 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { return parserAdapter.parse(files); } } diff --git a/languages/rust/src/main/java/de/jplag/rust/RustLanguage.java b/languages/rust/src/main/java/de/jplag/rust/RustLanguage.java index 72d8fb89f..50f0826e0 100644 --- a/languages/rust/src/main/java/de/jplag/rust/RustLanguage.java +++ b/languages/rust/src/main/java/de/jplag/rust/RustLanguage.java @@ -47,7 +47,7 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { return parserAdapter.parse(files); } } diff --git a/languages/rust/src/main/java/de/jplag/rust/grammar/RustLexerBase.java b/languages/rust/src/main/java/de/jplag/rust/grammar/RustLexerBase.java index 87975ddba..cf99293cd 100644 --- a/languages/rust/src/main/java/de/jplag/rust/grammar/RustLexerBase.java +++ b/languages/rust/src/main/java/de/jplag/rust/grammar/RustLexerBase.java @@ -50,8 +50,9 @@ public boolean floatDotPossible() { private boolean lookAheadMatches(String expected) { for (int charIndex = 0; charIndex < expected.length(); charIndex++) { - if (_input.LA(charIndex + 1) != expected.charAt(charIndex)) + if (_input.LA(charIndex + 1) != expected.charAt(charIndex)) { return false; + } } return true; } @@ -61,9 +62,7 @@ private boolean lookAheadMatchesOneOf(String... expected) { } public boolean floatLiteralPossible() { - if (this.currentToken == null || this.currentToken.getType() != RustLexer.DOT) { - return true; - } else if (this.previousToken == null) { + if (this.currentToken == null || this.currentToken.getType() != RustLexer.DOT || (this.previousToken == null)) { return true; } diff --git a/languages/scala/src/main/scala/de/jplag/scala/ScalaLanguage.scala b/languages/scala/src/main/scala/de/jplag/scala/ScalaLanguage.scala index 47988c6d9..424b0f733 100644 --- a/languages/scala/src/main/scala/de/jplag/scala/ScalaLanguage.scala +++ b/languages/scala/src/main/scala/de/jplag/scala/ScalaLanguage.scala @@ -20,5 +20,5 @@ class ScalaLanguage extends de.jplag.Language { override def minimumTokenMatch = 8 - override def parse(files: util.Set[File]): java.util.List[Token] = this.parser.parse(files.asScala.toSet).asJava + override def parse(files: util.Set[File], normalize: Boolean): java.util.List[Token] = this.parser.parse(files.asScala.toSet).asJava } diff --git a/languages/scala/src/test/java/de/jplag/scala/ScalaLanguageTest.java b/languages/scala/src/test/java/de/jplag/scala/ScalaLanguageTest.java index fc659f083..2f0660b54 100644 --- a/languages/scala/src/test/java/de/jplag/scala/ScalaLanguageTest.java +++ b/languages/scala/src/test/java/de/jplag/scala/ScalaLanguageTest.java @@ -19,6 +19,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import de.jplag.ParsingException; import de.jplag.SharedTokenType; import de.jplag.Token; import de.jplag.TokenPrinter; @@ -57,7 +58,7 @@ void setup() { } @Test - void parseTestFiles() { + void parseTestFiles() throws ParsingException { for (String fileName : testFiles) { List tokens = language.parse(Set.of(new File(testFileLocation, fileName))); String output = TokenPrinter.printTokens(tokens, testFileLocation); diff --git a/languages/scheme/src/main/java/de/jplag/scheme/SchemeLanguage.java b/languages/scheme/src/main/java/de/jplag/scheme/SchemeLanguage.java index 08dec1df8..0ebbf4ef9 100644 --- a/languages/scheme/src/main/java/de/jplag/scheme/SchemeLanguage.java +++ b/languages/scheme/src/main/java/de/jplag/scheme/SchemeLanguage.java @@ -40,7 +40,7 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { return this.parser.parse(files); } } diff --git a/languages/scxml/pom.xml b/languages/scxml/pom.xml index 18d8cd1e6..37a3ab4dd 100644 --- a/languages/scxml/pom.xml +++ b/languages/scxml/pom.xml @@ -12,7 +12,7 @@ org.assertj assertj-core - 3.25.2 + 3.25.3 test diff --git a/languages/scxml/src/main/java/de/jplag/scxml/ScxmlLanguage.java b/languages/scxml/src/main/java/de/jplag/scxml/ScxmlLanguage.java index 6d83703e6..ec6316f4d 100644 --- a/languages/scxml/src/main/java/de/jplag/scxml/ScxmlLanguage.java +++ b/languages/scxml/src/main/java/de/jplag/scxml/ScxmlLanguage.java @@ -63,7 +63,7 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { return parser.parse(files); } diff --git a/languages/scxml/src/main/java/de/jplag/scxml/parser/HandcraftedScxmlTokenGenerator.java b/languages/scxml/src/main/java/de/jplag/scxml/parser/HandcraftedScxmlTokenGenerator.java index 2422e85f2..09bab656e 100644 --- a/languages/scxml/src/main/java/de/jplag/scxml/parser/HandcraftedScxmlTokenGenerator.java +++ b/languages/scxml/src/main/java/de/jplag/scxml/parser/HandcraftedScxmlTokenGenerator.java @@ -1,6 +1,14 @@ package de.jplag.scxml.parser; -import static de.jplag.scxml.ScxmlTokenType.*; +import static de.jplag.scxml.ScxmlTokenType.GUARDED_TRANSITION; +import static de.jplag.scxml.ScxmlTokenType.INITIAL_REGION; +import static de.jplag.scxml.ScxmlTokenType.INITIAL_STATE; +import static de.jplag.scxml.ScxmlTokenType.REGION; +import static de.jplag.scxml.ScxmlTokenType.STATE; +import static de.jplag.scxml.ScxmlTokenType.STATE_END; +import static de.jplag.scxml.ScxmlTokenType.TIMED_TRANSITION; +import static de.jplag.scxml.ScxmlTokenType.TRANSITION; +import static de.jplag.scxml.ScxmlTokenType.TRANSITION_END; import de.jplag.scxml.parser.model.State; import de.jplag.scxml.parser.model.Transition; diff --git a/languages/scxml/src/main/java/de/jplag/scxml/parser/SimpleScxmlTokenGenerator.java b/languages/scxml/src/main/java/de/jplag/scxml/parser/SimpleScxmlTokenGenerator.java index 2954bbf99..ba94ac2ed 100644 --- a/languages/scxml/src/main/java/de/jplag/scxml/parser/SimpleScxmlTokenGenerator.java +++ b/languages/scxml/src/main/java/de/jplag/scxml/parser/SimpleScxmlTokenGenerator.java @@ -1,6 +1,24 @@ package de.jplag.scxml.parser; -import static de.jplag.scxml.ScxmlTokenType.*; +import static de.jplag.scxml.ScxmlTokenType.ACTION_END; +import static de.jplag.scxml.ScxmlTokenType.ASSIGNMENT; +import static de.jplag.scxml.ScxmlTokenType.CANCEL; +import static de.jplag.scxml.ScxmlTokenType.ELSE; +import static de.jplag.scxml.ScxmlTokenType.ELSE_END; +import static de.jplag.scxml.ScxmlTokenType.ELSE_IF; +import static de.jplag.scxml.ScxmlTokenType.ELSE_IF_END; +import static de.jplag.scxml.ScxmlTokenType.FOREACH; +import static de.jplag.scxml.ScxmlTokenType.IF; +import static de.jplag.scxml.ScxmlTokenType.IF_END; +import static de.jplag.scxml.ScxmlTokenType.ON_ENTRY; +import static de.jplag.scxml.ScxmlTokenType.ON_EXIT; +import static de.jplag.scxml.ScxmlTokenType.RAISE; +import static de.jplag.scxml.ScxmlTokenType.SCRIPT; +import static de.jplag.scxml.ScxmlTokenType.SEND; +import static de.jplag.scxml.ScxmlTokenType.STATE; +import static de.jplag.scxml.ScxmlTokenType.STATE_END; +import static de.jplag.scxml.ScxmlTokenType.TRANSITION; +import static de.jplag.scxml.ScxmlTokenType.TRANSITION_END; import static java.util.Map.entry; import java.util.List; @@ -11,7 +29,14 @@ import de.jplag.scxml.parser.model.Statechart; import de.jplag.scxml.parser.model.StatechartElement; import de.jplag.scxml.parser.model.Transition; -import de.jplag.scxml.parser.model.executable_content.*; +import de.jplag.scxml.parser.model.executable_content.Action; +import de.jplag.scxml.parser.model.executable_content.Cancel; +import de.jplag.scxml.parser.model.executable_content.Else; +import de.jplag.scxml.parser.model.executable_content.ElseIf; +import de.jplag.scxml.parser.model.executable_content.ExecutableContent; +import de.jplag.scxml.parser.model.executable_content.If; +import de.jplag.scxml.parser.model.executable_content.Send; +import de.jplag.scxml.parser.model.executable_content.SimpleExecutableContent; import de.jplag.scxml.util.AbstractScxmlVisitor; /** diff --git a/languages/scxml/src/main/java/de/jplag/scxml/parser/model/executable_content/ExecutableContent.java b/languages/scxml/src/main/java/de/jplag/scxml/parser/model/executable_content/ExecutableContent.java index 17d4ee1e6..4b4632382 100644 --- a/languages/scxml/src/main/java/de/jplag/scxml/parser/model/executable_content/ExecutableContent.java +++ b/languages/scxml/src/main/java/de/jplag/scxml/parser/model/executable_content/ExecutableContent.java @@ -1,6 +1,10 @@ package de.jplag.scxml.parser.model.executable_content; -import static de.jplag.scxml.parser.model.executable_content.SimpleExecutableContent.Type.*; +import static de.jplag.scxml.parser.model.executable_content.SimpleExecutableContent.Type.ASSIGNMENT; +import static de.jplag.scxml.parser.model.executable_content.SimpleExecutableContent.Type.FOREACH; +import static de.jplag.scxml.parser.model.executable_content.SimpleExecutableContent.Type.LOG; +import static de.jplag.scxml.parser.model.executable_content.SimpleExecutableContent.Type.RAISE; +import static de.jplag.scxml.parser.model.executable_content.SimpleExecutableContent.Type.SCRIPT; import java.util.Set; diff --git a/languages/scxml/src/main/java/de/jplag/scxml/util/AbstractScxmlVisitor.java b/languages/scxml/src/main/java/de/jplag/scxml/util/AbstractScxmlVisitor.java index 85c7c6982..bd148d06c 100644 --- a/languages/scxml/src/main/java/de/jplag/scxml/util/AbstractScxmlVisitor.java +++ b/languages/scxml/src/main/java/de/jplag/scxml/util/AbstractScxmlVisitor.java @@ -10,7 +10,12 @@ import de.jplag.scxml.parser.model.Statechart; import de.jplag.scxml.parser.model.StatechartElement; import de.jplag.scxml.parser.model.Transition; -import de.jplag.scxml.parser.model.executable_content.*; +import de.jplag.scxml.parser.model.executable_content.Action; +import de.jplag.scxml.parser.model.executable_content.Else; +import de.jplag.scxml.parser.model.executable_content.ElseIf; +import de.jplag.scxml.parser.model.executable_content.ExecutableContent; +import de.jplag.scxml.parser.model.executable_content.If; +import de.jplag.scxml.parser.model.executable_content.SimpleExecutableContent; import de.jplag.scxml.sorting.RecursiveSortingStrategy; import de.jplag.scxml.sorting.SortingStrategy; diff --git a/languages/scxml/src/test/java/de/jplag/scxml/ScxmlParserTest.java b/languages/scxml/src/test/java/de/jplag/scxml/ScxmlParserTest.java index 13fad606b..42101cb83 100644 --- a/languages/scxml/src/test/java/de/jplag/scxml/ScxmlParserTest.java +++ b/languages/scxml/src/test/java/de/jplag/scxml/ScxmlParserTest.java @@ -21,7 +21,13 @@ import de.jplag.scxml.parser.model.State; import de.jplag.scxml.parser.model.Statechart; import de.jplag.scxml.parser.model.Transition; -import de.jplag.scxml.parser.model.executable_content.*; +import de.jplag.scxml.parser.model.executable_content.Cancel; +import de.jplag.scxml.parser.model.executable_content.Else; +import de.jplag.scxml.parser.model.executable_content.ElseIf; +import de.jplag.scxml.parser.model.executable_content.ExecutableContent; +import de.jplag.scxml.parser.model.executable_content.If; +import de.jplag.scxml.parser.model.executable_content.Send; +import de.jplag.scxml.parser.model.executable_content.SimpleExecutableContent; import de.jplag.scxml.util.StateBuilder; import de.jplag.testutils.FileUtil; diff --git a/languages/scxml/src/test/java/de/jplag/scxml/ScxmlTokenGeneratorTest.java b/languages/scxml/src/test/java/de/jplag/scxml/ScxmlTokenGeneratorTest.java index c11228c27..78e2dc594 100644 --- a/languages/scxml/src/test/java/de/jplag/scxml/ScxmlTokenGeneratorTest.java +++ b/languages/scxml/src/test/java/de/jplag/scxml/ScxmlTokenGeneratorTest.java @@ -1,7 +1,18 @@ package de.jplag.scxml; import static de.jplag.SharedTokenType.FILE_END; -import static de.jplag.scxml.ScxmlTokenType.*; +import static de.jplag.scxml.ScxmlTokenType.ACTION_END; +import static de.jplag.scxml.ScxmlTokenType.ASSIGNMENT; +import static de.jplag.scxml.ScxmlTokenType.CANCEL; +import static de.jplag.scxml.ScxmlTokenType.IF; +import static de.jplag.scxml.ScxmlTokenType.IF_END; +import static de.jplag.scxml.ScxmlTokenType.ON_ENTRY; +import static de.jplag.scxml.ScxmlTokenType.ON_EXIT; +import static de.jplag.scxml.ScxmlTokenType.SEND; +import static de.jplag.scxml.ScxmlTokenType.STATE; +import static de.jplag.scxml.ScxmlTokenType.STATE_END; +import static de.jplag.scxml.ScxmlTokenType.TRANSITION; +import static de.jplag.scxml.ScxmlTokenType.TRANSITION_END; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/languages/swift/src/main/java/de/jplag/swift/JPlagSwiftListener.java b/languages/swift/src/main/java/de/jplag/swift/JPlagSwiftListener.java index b52900c77..90e1e8249 100644 --- a/languages/swift/src/main/java/de/jplag/swift/JPlagSwiftListener.java +++ b/languages/swift/src/main/java/de/jplag/swift/JPlagSwiftListener.java @@ -1,10 +1,106 @@ package de.jplag.swift; -import static de.jplag.swift.SwiftTokenType.*; +import static de.jplag.swift.SwiftTokenType.ASSIGNMENT; +import static de.jplag.swift.SwiftTokenType.BREAK; +import static de.jplag.swift.SwiftTokenType.CATCH_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.CATCH_BODY_END; +import static de.jplag.swift.SwiftTokenType.CLASS_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.CLASS_BODY_END; +import static de.jplag.swift.SwiftTokenType.CLASS_DECLARATION; +import static de.jplag.swift.SwiftTokenType.CLOSURE_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.CLOSURE_BODY_END; +import static de.jplag.swift.SwiftTokenType.CONTINUE; +import static de.jplag.swift.SwiftTokenType.DEFER_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.DEFER_BODY_END; +import static de.jplag.swift.SwiftTokenType.DO_TRY_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.DO_TRY_BODY_END; +import static de.jplag.swift.SwiftTokenType.ENUM_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.ENUM_BODY_END; +import static de.jplag.swift.SwiftTokenType.ENUM_DECLARATION; +import static de.jplag.swift.SwiftTokenType.ENUM_LITERAL; +import static de.jplag.swift.SwiftTokenType.FALLTHROUGH; +import static de.jplag.swift.SwiftTokenType.FOR_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.FOR_BODY_END; +import static de.jplag.swift.SwiftTokenType.FUNCTION; +import static de.jplag.swift.SwiftTokenType.FUNCTION_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.FUNCTION_BODY_END; +import static de.jplag.swift.SwiftTokenType.FUNCTION_CALL; +import static de.jplag.swift.SwiftTokenType.FUNCTION_PARAMETER; +import static de.jplag.swift.SwiftTokenType.IF_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.IF_BODY_END; +import static de.jplag.swift.SwiftTokenType.IMPORT; +import static de.jplag.swift.SwiftTokenType.PROPERTY_ACCESSOR_BEGIN; +import static de.jplag.swift.SwiftTokenType.PROPERTY_ACCESSOR_END; +import static de.jplag.swift.SwiftTokenType.PROPERTY_DECLARATION; +import static de.jplag.swift.SwiftTokenType.PROTOCOL_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.PROTOCOL_BODY_END; +import static de.jplag.swift.SwiftTokenType.PROTOCOL_DECLARATION; +import static de.jplag.swift.SwiftTokenType.REPEAT_WHILE_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.REPEAT_WHILE_BODY_END; +import static de.jplag.swift.SwiftTokenType.RETURN; +import static de.jplag.swift.SwiftTokenType.STRUCT_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.STRUCT_BODY_END; +import static de.jplag.swift.SwiftTokenType.STRUCT_DECLARATION; +import static de.jplag.swift.SwiftTokenType.SWITCH_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.SWITCH_BODY_END; +import static de.jplag.swift.SwiftTokenType.SWITCH_CASE; +import static de.jplag.swift.SwiftTokenType.THROW; +import static de.jplag.swift.SwiftTokenType.WHILE_BODY_BEGIN; +import static de.jplag.swift.SwiftTokenType.WHILE_BODY_END; import org.antlr.v4.runtime.Token; -import de.jplag.swift.grammar.Swift5Parser.*; +import de.jplag.swift.grammar.Swift5Parser.Binary_operatorContext; +import de.jplag.swift.grammar.Swift5Parser.Break_statementContext; +import de.jplag.swift.grammar.Swift5Parser.Catch_clauseContext; +import de.jplag.swift.grammar.Swift5Parser.Class_bodyContext; +import de.jplag.swift.grammar.Swift5Parser.Class_declarationContext; +import de.jplag.swift.grammar.Swift5Parser.Closure_expressionContext; +import de.jplag.swift.grammar.Swift5Parser.Code_blockContext; +import de.jplag.swift.grammar.Swift5Parser.Constant_declarationContext; +import de.jplag.swift.grammar.Swift5Parser.Continue_statementContext; +import de.jplag.swift.grammar.Swift5Parser.Defer_statementContext; +import de.jplag.swift.grammar.Swift5Parser.DidSet_clauseContext; +import de.jplag.swift.grammar.Swift5Parser.Do_blockContext; +import de.jplag.swift.grammar.Swift5Parser.Else_clauseContext; +import de.jplag.swift.grammar.Swift5Parser.Enum_nameContext; +import de.jplag.swift.grammar.Swift5Parser.Fallthrough_statementContext; +import de.jplag.swift.grammar.Swift5Parser.For_in_statementContext; +import de.jplag.swift.grammar.Swift5Parser.Function_bodyContext; +import de.jplag.swift.grammar.Swift5Parser.Function_call_suffixContext; +import de.jplag.swift.grammar.Swift5Parser.Function_nameContext; +import de.jplag.swift.grammar.Swift5Parser.Function_resultContext; +import de.jplag.swift.grammar.Swift5Parser.Getter_clauseContext; +import de.jplag.swift.grammar.Swift5Parser.Getter_setter_blockContext; +import de.jplag.swift.grammar.Swift5Parser.Guard_statementContext; +import de.jplag.swift.grammar.Swift5Parser.If_statementContext; +import de.jplag.swift.grammar.Swift5Parser.Import_declarationContext; +import de.jplag.swift.grammar.Swift5Parser.InitializerContext; +import de.jplag.swift.grammar.Swift5Parser.Initializer_bodyContext; +import de.jplag.swift.grammar.Swift5Parser.Initializer_declarationContext; +import de.jplag.swift.grammar.Swift5Parser.ParameterContext; +import de.jplag.swift.grammar.Swift5Parser.Protocol_bodyContext; +import de.jplag.swift.grammar.Swift5Parser.Protocol_declarationContext; +import de.jplag.swift.grammar.Swift5Parser.Protocol_initializer_declarationContext; +import de.jplag.swift.grammar.Swift5Parser.Protocol_property_declarationContext; +import de.jplag.swift.grammar.Swift5Parser.Raw_value_assignmentContext; +import de.jplag.swift.grammar.Swift5Parser.Raw_value_style_enumContext; +import de.jplag.swift.grammar.Swift5Parser.Raw_value_style_enum_caseContext; +import de.jplag.swift.grammar.Swift5Parser.Raw_value_style_enum_membersContext; +import de.jplag.swift.grammar.Swift5Parser.Repeat_while_statementContext; +import de.jplag.swift.grammar.Swift5Parser.Return_statementContext; +import de.jplag.swift.grammar.Swift5Parser.Setter_clauseContext; +import de.jplag.swift.grammar.Swift5Parser.Struct_bodyContext; +import de.jplag.swift.grammar.Swift5Parser.Struct_declarationContext; +import de.jplag.swift.grammar.Swift5Parser.Switch_caseContext; +import de.jplag.swift.grammar.Swift5Parser.Switch_statementContext; +import de.jplag.swift.grammar.Swift5Parser.Throw_statementContext; +import de.jplag.swift.grammar.Swift5Parser.Union_style_enumContext; +import de.jplag.swift.grammar.Swift5Parser.Union_style_enum_caseContext; +import de.jplag.swift.grammar.Swift5Parser.Union_style_enum_membersContext; +import de.jplag.swift.grammar.Swift5Parser.Variable_declarationContext; +import de.jplag.swift.grammar.Swift5Parser.While_statementContext; +import de.jplag.swift.grammar.Swift5Parser.WillSet_clauseContext; import de.jplag.swift.grammar.Swift5ParserBaseListener; public class JPlagSwiftListener extends Swift5ParserBaseListener { diff --git a/languages/swift/src/main/java/de/jplag/swift/SwiftLanguage.java b/languages/swift/src/main/java/de/jplag/swift/SwiftLanguage.java index b02aa9094..87e13269f 100644 --- a/languages/swift/src/main/java/de/jplag/swift/SwiftLanguage.java +++ b/languages/swift/src/main/java/de/jplag/swift/SwiftLanguage.java @@ -47,7 +47,7 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { return parserAdapter.parse(files); } } diff --git a/languages/swift/src/main/java/de/jplag/swift/grammar/SwiftSupport.java b/languages/swift/src/main/java/de/jplag/swift/grammar/SwiftSupport.java index 45c1a0dca..9da0d5b09 100644 --- a/languages/swift/src/main/java/de/jplag/swift/grammar/SwiftSupport.java +++ b/languages/swift/src/main/java/de/jplag/swift/grammar/SwiftSupport.java @@ -162,7 +162,6 @@ public static boolean isOperatorCharacter(Token token) { public static boolean isOpNext(TokenStream tokens) { int start = tokens.index(); - Token lt = tokens.get(start); int stop = getLastOpTokenIndex(tokens); return stop != -1; // System.out.printf("isOpNext: i=%d t='%s'", start, lt.getText()); diff --git a/languages/text/src/main/java/de/jplag/text/NaturalLanguage.java b/languages/text/src/main/java/de/jplag/text/NaturalLanguage.java index a9b974840..572713009 100644 --- a/languages/text/src/main/java/de/jplag/text/NaturalLanguage.java +++ b/languages/text/src/main/java/de/jplag/text/NaturalLanguage.java @@ -45,7 +45,7 @@ public int minimumTokenMatch() { } @Override - public List parse(Set files) throws ParsingException { + public List parse(Set files, boolean normalize) throws ParsingException { return parserAdapter.parse(files); } } diff --git a/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptLanguage.java b/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptLanguage.java index d6f1fbb0a..9fa5ad514 100644 --- a/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptLanguage.java +++ b/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptLanguage.java @@ -39,7 +39,7 @@ public TypeScriptLanguageOptions getOptions() { } @Override - protected TypeScriptParserAdapter initializeParser() { + protected TypeScriptParserAdapter initializeParser(boolean normalize) { return new TypeScriptParserAdapter(getOptions().useStrictDefault()); } } diff --git a/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptListener.java b/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptListener.java index 31dc21378..5a65d7c42 100644 --- a/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptListener.java +++ b/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptListener.java @@ -36,47 +36,47 @@ import static de.jplag.typescript.TypeScriptTokenType.TRY_BEGIN; import static de.jplag.typescript.TypeScriptTokenType.WHILE_BEGIN; import static de.jplag.typescript.TypeScriptTokenType.WHILE_END; -import static de.jplag.typescript.grammar.TypeScriptParser.ArgumentsContext; -import static de.jplag.typescript.grammar.TypeScriptParser.ArrowFunctionDeclarationContext; -import static de.jplag.typescript.grammar.TypeScriptParser.AssignmentExpressionContext; -import static de.jplag.typescript.grammar.TypeScriptParser.BreakStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.CaseClauseContext; -import static de.jplag.typescript.grammar.TypeScriptParser.CatchProductionContext; -import static de.jplag.typescript.grammar.TypeScriptParser.ClassDeclarationContext; -import static de.jplag.typescript.grammar.TypeScriptParser.ConstructorDeclarationContext; -import static de.jplag.typescript.grammar.TypeScriptParser.ContinueStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.DefaultClauseContext; import static de.jplag.typescript.grammar.TypeScriptParser.Else; -import static de.jplag.typescript.grammar.TypeScriptParser.EnumDeclarationContext; -import static de.jplag.typescript.grammar.TypeScriptParser.EnumMemberContext; import static de.jplag.typescript.grammar.TypeScriptParser.Export; -import static de.jplag.typescript.grammar.TypeScriptParser.FinallyProductionContext; -import static de.jplag.typescript.grammar.TypeScriptParser.ForInStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.ForStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.ForVarStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.FunctionDeclarationContext; -import static de.jplag.typescript.grammar.TypeScriptParser.FunctionExpressionDeclarationContext; -import static de.jplag.typescript.grammar.TypeScriptParser.GetterSetterDeclarationExpressionContext; -import static de.jplag.typescript.grammar.TypeScriptParser.IfStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.ImportStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.InterfaceDeclarationContext; -import static de.jplag.typescript.grammar.TypeScriptParser.MethodDeclarationExpressionContext; -import static de.jplag.typescript.grammar.TypeScriptParser.NamespaceDeclarationContext; -import static de.jplag.typescript.grammar.TypeScriptParser.PostDecreaseExpressionContext; -import static de.jplag.typescript.grammar.TypeScriptParser.PostIncrementExpressionContext; -import static de.jplag.typescript.grammar.TypeScriptParser.PreDecreaseExpressionContext; -import static de.jplag.typescript.grammar.TypeScriptParser.PreIncrementExpressionContext; -import static de.jplag.typescript.grammar.TypeScriptParser.PropertyDeclarationExpressionContext; -import static de.jplag.typescript.grammar.TypeScriptParser.PropertySetterContext; -import static de.jplag.typescript.grammar.TypeScriptParser.PropertySignaturContext; -import static de.jplag.typescript.grammar.TypeScriptParser.ReturnStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.SwitchStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.ThrowStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.TryStatementContext; -import static de.jplag.typescript.grammar.TypeScriptParser.VariableDeclarationContext; -import static de.jplag.typescript.grammar.TypeScriptParser.WhileStatementContext; import de.jplag.antlr.AbstractAntlrListener; +import de.jplag.typescript.grammar.TypeScriptParser.ArgumentsContext; +import de.jplag.typescript.grammar.TypeScriptParser.ArrowFunctionDeclarationContext; +import de.jplag.typescript.grammar.TypeScriptParser.AssignmentExpressionContext; +import de.jplag.typescript.grammar.TypeScriptParser.BreakStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.CaseClauseContext; +import de.jplag.typescript.grammar.TypeScriptParser.CatchProductionContext; +import de.jplag.typescript.grammar.TypeScriptParser.ClassDeclarationContext; +import de.jplag.typescript.grammar.TypeScriptParser.ConstructorDeclarationContext; +import de.jplag.typescript.grammar.TypeScriptParser.ContinueStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.DefaultClauseContext; +import de.jplag.typescript.grammar.TypeScriptParser.EnumDeclarationContext; +import de.jplag.typescript.grammar.TypeScriptParser.EnumMemberContext; +import de.jplag.typescript.grammar.TypeScriptParser.FinallyProductionContext; +import de.jplag.typescript.grammar.TypeScriptParser.ForInStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.ForStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.ForVarStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.FunctionDeclarationContext; +import de.jplag.typescript.grammar.TypeScriptParser.FunctionExpressionDeclarationContext; +import de.jplag.typescript.grammar.TypeScriptParser.GetterSetterDeclarationExpressionContext; +import de.jplag.typescript.grammar.TypeScriptParser.IfStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.ImportStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.InterfaceDeclarationContext; +import de.jplag.typescript.grammar.TypeScriptParser.MethodDeclarationExpressionContext; +import de.jplag.typescript.grammar.TypeScriptParser.NamespaceDeclarationContext; +import de.jplag.typescript.grammar.TypeScriptParser.PostDecreaseExpressionContext; +import de.jplag.typescript.grammar.TypeScriptParser.PostIncrementExpressionContext; +import de.jplag.typescript.grammar.TypeScriptParser.PreDecreaseExpressionContext; +import de.jplag.typescript.grammar.TypeScriptParser.PreIncrementExpressionContext; +import de.jplag.typescript.grammar.TypeScriptParser.PropertyDeclarationExpressionContext; +import de.jplag.typescript.grammar.TypeScriptParser.PropertySetterContext; +import de.jplag.typescript.grammar.TypeScriptParser.PropertySignaturContext; +import de.jplag.typescript.grammar.TypeScriptParser.ReturnStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.SwitchStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.ThrowStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.TryStatementContext; +import de.jplag.typescript.grammar.TypeScriptParser.VariableDeclarationContext; +import de.jplag.typescript.grammar.TypeScriptParser.WhileStatementContext; /** * This class is responsible for mapping parsed TypeScript to the internal Token structure diff --git a/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptTokenType.java b/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptTokenType.java index da8b0f8da..1d31eb1ca 100644 --- a/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptTokenType.java +++ b/languages/typescript/src/main/java/de/jplag/typescript/TypeScriptTokenType.java @@ -46,6 +46,7 @@ public enum TypeScriptTokenType implements TokenType { private final String description; + @Override public String getDescription() { return this.description; } diff --git a/pom.xml b/pom.xml index 7109f205b..58bef6929 100644 --- a/pom.xml +++ b/pom.xml @@ -75,8 +75,8 @@ 21 21 2.43.0 - 2.0.11 - 5.10.1 + 2.0.12 + 5.10.2 2.7.7 4.13.1 @@ -86,7 +86,7 @@ 1.0.0 - 4.4.0-SNAPSHOT + 5.0.0 @@ -117,7 +117,7 @@ edu.stanford.nlp stanford-corenlp - 4.5.5 + 4.5.6 diff --git a/report-viewer/.husky/pre-commit b/report-viewer/.husky/pre-commit index b93aca0a8..b4eb63f65 100644 --- a/report-viewer/.husky/pre-commit +++ b/report-viewer/.husky/pre-commit @@ -1,5 +1,2 @@ -#!/usr/bin/env sh -. "$(dirname -- "$0")/_/husky.sh" - cd ./report-viewer -npx lint-staged +npx lint-staged \ No newline at end of file diff --git a/report-viewer/package-lock.json b/report-viewer/package-lock.json index e2a525c75..b272d1fce 100644 --- a/report-viewer/package-lock.json +++ b/report-viewer/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.0", "dependencies": { "@fortawesome/fontawesome-svg-core": "^6.5.1", + "@fortawesome/free-brands-svg-icons": "^6.5.1", "@fortawesome/free-regular-svg-icons": "^6.5.1", "@fortawesome/free-solid-svg-icons": "^6.5.1", "@fortawesome/vue-fontawesome": "^3.0.5", @@ -29,25 +30,25 @@ "@playwright/test": "^1.40.1", "@rushstack/eslint-patch": "^1.7.2", "@types/jsdom": "^21.1.6", - "@types/node": "^18.19.11", - "@vitejs/plugin-vue": "^5.0.3", - "@vue/eslint-config-prettier": "^8.0.0", + "@types/node": "^18.19.15", + "@vitejs/plugin-vue": "^5.0.4", + "@vue/eslint-config-prettier": "^9.0.0", "@vue/eslint-config-typescript": "^12.0.0", "@vue/test-utils": "^2.4.3", "@vue/tsconfig": "^0.5.1", "autoprefixer": "^10.4.16", "eslint": "^8.56.0", "eslint-plugin-vue": "^9.20.1", - "husky": "^8.0.0", + "husky": "^9.0.11", "jsdom": "^24.0.0", - "lint-staged": "^15.2.0", + "lint-staged": "^15.2.2", "npm-run-all": "^4.1.5", - "postcss": "^8.4.33", - "prettier": "^3.1.1", + "postcss": "^8.4.35", + "prettier": "^3.2.5", "prettier-plugin-tailwindcss": "^0.5.11", "tailwindcss": "^3.4.1", "typescript": "^5.3.3", - "vite": "^5.0.12", + "vite": "^5.1.1", "vitest": "^1.2.2", "vue-tsc": "^1.8.27" } @@ -551,6 +552,18 @@ "node": ">=6" } }, + "node_modules/@fortawesome/free-brands-svg-icons": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/@fortawesome/free-brands-svg-icons/-/free-brands-svg-icons-6.5.1.tgz", + "integrity": "sha512-093l7DAkx0aEtBq66Sf19MgoZewv1zeY9/4C7vSKPO4qMwEsW/2VYTUTpBtLwfb9T2R73tXaRDPmE4UqLCYHfg==", + "hasInstallScript": true, + "dependencies": { + "@fortawesome/fontawesome-common-types": "6.5.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@fortawesome/free-regular-svg-icons": { "version": "6.5.1", "resolved": "https://registry.npmjs.org/@fortawesome/free-regular-svg-icons/-/free-regular-svg-icons-6.5.1.tgz", @@ -1040,9 +1053,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.19.14", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.14.tgz", - "integrity": "sha512-EnQ4Us2rmOS64nHDWr0XqAD8DsO6f3XR6lf9UIIrZQpUzPVdN/oPuEzfDWNHSyXLvoGgjuEm/sPwFGSSs35Wtg==", + "version": "18.19.15", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.15.tgz", + "integrity": "sha512-AMZ2UWx+woHNfM11PyAEQmfSxi05jm9OlkxczuHeEqmvwPkYj6MWv44gbzDPefYOLysTOFyI3ziiy2ONmUZfpA==", "dev": true, "dependencies": { "undici-types": "~5.26.4" @@ -1257,9 +1270,9 @@ "dev": true }, "node_modules/@vitejs/plugin-vue": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-5.0.3.tgz", - "integrity": "sha512-b8S5dVS40rgHdDrw+DQi/xOM9ed+kSRZzfm1T74bMmBDCd8XO87NKlFYInzCtwvtWwXZvo1QxE2OSspTATWrbA==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-5.0.4.tgz", + "integrity": "sha512-WS3hevEszI6CEVEx28F8RjTX97k3KsrcY6kvTg7+Whm5y3oYvcqzVeGCU3hxSAn4uY2CLCkeokkGKpoctccilQ==", "dev": true, "engines": { "node": "^18.0.0 || >=20.0.0" @@ -1455,12 +1468,12 @@ "integrity": "sha512-+KpckaAQyfbvshdDW5xQylLni1asvNSGme1JFs8I1+/H5pHEhqUKMEQD/qn3Nx5+/nycBq11qAEi8lk+LXI2dA==" }, "node_modules/@vue/eslint-config-prettier": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@vue/eslint-config-prettier/-/eslint-config-prettier-8.0.0.tgz", - "integrity": "sha512-55dPqtC4PM/yBjhAr+yEw6+7KzzdkBuLmnhBrDfp4I48+wy+Giqqj9yUr5T2uD/BkBROjjmqnLZmXRdOx/VtQg==", + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@vue/eslint-config-prettier/-/eslint-config-prettier-9.0.0.tgz", + "integrity": "sha512-z1ZIAAUS9pKzo/ANEfd2sO+v2IUalz7cM/cTLOZ7vRFOPk5/xuRKQteOu1DErFLAh/lYGXMVZ0IfYKlyInuDVg==", "dev": true, "dependencies": { - "eslint-config-prettier": "^8.8.0", + "eslint-config-prettier": "^9.0.0", "eslint-plugin-prettier": "^5.0.0" }, "peerDependencies": { @@ -2754,9 +2767,9 @@ } }, "node_modules/eslint-config-prettier": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.10.0.tgz", - "integrity": "sha512-SM8AMJdeQqRYT9O9zguiruQZaN7+z+E4eAP9oiLNGKMtomwaB1E9dcgUD6ZAn/eQAb52USbvezbiljfZUhbJcg==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz", + "integrity": "sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==", "dev": true, "bin": { "eslint-config-prettier": "bin/cli.js" @@ -3525,15 +3538,15 @@ } }, "node_modules/husky": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/husky/-/husky-8.0.3.tgz", - "integrity": "sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg==", + "version": "9.0.11", + "resolved": "https://registry.npmjs.org/husky/-/husky-9.0.11.tgz", + "integrity": "sha512-AB6lFlbwwyIqMdHYhwPe+kjOC3Oc5P3nThEoW/AaO2BX3vJDjWPFxYLxokUZOo6RNX20He3AaT8sESs9NJcmEw==", "dev": true, "bin": { - "husky": "lib/bin.js" + "husky": "bin.mjs" }, "engines": { - "node": ">=14" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/typicode" @@ -4111,9 +4124,9 @@ "dev": true }, "node_modules/lint-staged": { - "version": "15.2.1", - "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-15.2.1.tgz", - "integrity": "sha512-dhwAPnM85VdshybV9FWI/9ghTvMLoQLEXgVMx+ua2DN7mdfzd/tRfoU2yhMcBac0RHkofoxdnnJUokr8s4zKmQ==", + "version": "15.2.2", + "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-15.2.2.tgz", + "integrity": "sha512-TiTt93OPh1OZOsb5B7k96A/ATl2AjIZo+vnzFZ6oHK5FuTk63ByDtxGQpHm+kFETjEWqgkF95M8FRXKR/LEBcw==", "dev": true, "dependencies": { "chalk": "5.3.0", @@ -5320,9 +5333,9 @@ } }, "node_modules/postcss": { - "version": "8.4.33", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.33.tgz", - "integrity": "sha512-Kkpbhhdjw2qQs2O2DGX+8m5OVqEcbB9HRBvuYM9pgrjEFUg30A9LmXNlTAUj4S9kgtGyrMbTzVjH7E+s5Re2yg==", + "version": "8.4.35", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.35.tgz", + "integrity": "sha512-u5U8qYpBCpN13BsiEB0CbR1Hhh4Gc0zLFuedrHJKMctHCHAGrMdG0PRM/KErzAL3CU6/eckEtmHNB3x6e3c0vA==", "funding": [ { "type": "opencollective", @@ -5465,9 +5478,9 @@ } }, "node_modules/prettier": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.4.tgz", - "integrity": "sha512-FWu1oLHKCrtpO1ypU6J0SbK2d9Ckwysq6bHj/uaCP26DxrPpppCLQRGVuqAxSTvhF00AcvDRyYrLNW7ocBhFFQ==", + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz", + "integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==", "dev": true, "bin": { "prettier": "bin/prettier.cjs" @@ -6900,13 +6913,13 @@ } }, "node_modules/vite": { - "version": "5.0.12", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.0.12.tgz", - "integrity": "sha512-4hsnEkG3q0N4Tzf1+t6NdN9dg/L3BM+q8SWgbSPnJvrgH2kgdyzfVJwbR1ic69/4uMJJ/3dqDZZE5/WwqW8U1w==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.1.1.tgz", + "integrity": "sha512-wclpAgY3F1tR7t9LL5CcHC41YPkQIpKUGeIuT8MdNwNZr6OqOTLs7JX5vIHAtzqLWXts0T+GDrh9pN2arneKqg==", "dev": true, "dependencies": { "esbuild": "^0.19.3", - "postcss": "^8.4.32", + "postcss": "^8.4.35", "rollup": "^4.2.0" }, "bin": { diff --git a/report-viewer/package.json b/report-viewer/package.json index e517d97c4..b1d1d8a06 100644 --- a/report-viewer/package.json +++ b/report-viewer/package.json @@ -16,10 +16,11 @@ "type-check": "vue-tsc --noEmit -p tsconfig.vitest.json --composite false", "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix --ignore-path .gitignore --max-warnings 0", "format": "prettier --write src/", - "prepare": "cd .. && husky install report-viewer/.husky" + "prepare": "cd .. && husky report-viewer/.husky" }, "dependencies": { "@fortawesome/fontawesome-svg-core": "^6.5.1", + "@fortawesome/free-brands-svg-icons": "^6.5.1", "@fortawesome/free-regular-svg-icons": "^6.5.1", "@fortawesome/free-solid-svg-icons": "^6.5.1", "@fortawesome/vue-fontawesome": "^3.0.5", @@ -40,25 +41,25 @@ "@playwright/test": "^1.40.1", "@rushstack/eslint-patch": "^1.7.2", "@types/jsdom": "^21.1.6", - "@types/node": "^18.19.11", - "@vitejs/plugin-vue": "^5.0.3", - "@vue/eslint-config-prettier": "^8.0.0", + "@types/node": "^18.19.15", + "@vitejs/plugin-vue": "^5.0.4", + "@vue/eslint-config-prettier": "^9.0.0", "@vue/eslint-config-typescript": "^12.0.0", "@vue/test-utils": "^2.4.3", "@vue/tsconfig": "^0.5.1", "autoprefixer": "^10.4.16", "eslint": "^8.56.0", "eslint-plugin-vue": "^9.20.1", - "husky": "^8.0.0", + "husky": "^9.0.11", "jsdom": "^24.0.0", - "lint-staged": "^15.2.0", + "lint-staged": "^15.2.2", "npm-run-all": "^4.1.5", - "postcss": "^8.4.33", - "prettier": "^3.1.1", + "postcss": "^8.4.35", + "prettier": "^3.2.5", "prettier-plugin-tailwindcss": "^0.5.11", "tailwindcss": "^3.4.1", "typescript": "^5.3.3", - "vite": "^5.0.12", + "vite": "^5.1.1", "vitest": "^1.2.2", "vue-tsc": "^1.8.27" } diff --git a/report-viewer/src/components/ComparisonsTable.vue b/report-viewer/src/components/ComparisonsTable.vue index 0b8bbbb18..f5607b316 100644 --- a/report-viewer/src/components/ComparisonsTable.vue +++ b/report-viewer/src/components/ComparisonsTable.vue @@ -76,13 +76,16 @@ '!bg-accent !bg-opacity-30 ': isHighlightedRow(item) }" > -
@@ -105,7 +108,7 @@ {{ (item.similarities[MetricType.MAXIMUM] * 100).toFixed(2) }}%
- +
@@ -170,7 +173,6 @@ import { generateColors } from '@/utils/ColorUtils' import ToolTipComponent from './ToolTipComponent.vue' import { MetricType, metricToolTips } from '@/model/MetricType' import NameElement from './NameElement.vue' -import { router } from '@/router' import ComparisonTableFilter from './ComparisonTableFilter.vue' library.add(faUserGroup) @@ -195,7 +197,7 @@ const props = defineProps({ }) const displayedComparisons = computed(() => { - const comparisons = getFilteredComparisons(getSortedComparisons(props.topComparisons)) + const comparisons = getFilteredComparisons(getSortedComparisons(Array.from(props.topComparisons))) let index = 1 comparisons.forEach((c) => { c.id = index++ @@ -250,7 +252,7 @@ function getSortedComparisons(comparisons: ComparisonListElement[]) { comparisons.forEach((c) => { c.sortingPlace = index++ }) - return props.topComparisons + return comparisons } function getClusterFor(clusterIndex: number) { diff --git a/report-viewer/src/components/NameElement.vue b/report-viewer/src/components/NameElement.vue index a3445bf49..52a89dcfc 100644 --- a/report-viewer/src/components/NameElement.vue +++ b/report-viewer/src/components/NameElement.vue @@ -33,6 +33,7 @@ const props = defineProps({ function changeAnonymous(event: Event) { event.stopPropagation() + event.preventDefault() if (store().isAnonymous(props.id)) { store().removeAnonymous([props.id]) } else { diff --git a/report-viewer/src/components/RepositoryReference.vue b/report-viewer/src/components/RepositoryReference.vue new file mode 100644 index 000000000..c05d85bd2 --- /dev/null +++ b/report-viewer/src/components/RepositoryReference.vue @@ -0,0 +1,25 @@ + + + diff --git a/report-viewer/src/components/TextInformation.vue b/report-viewer/src/components/TextInformation.vue index c7372be1b..6044737f9 100644 --- a/report-viewer/src/components/TextInformation.vue +++ b/report-viewer/src/components/TextInformation.vue @@ -2,8 +2,8 @@ A container displaying simple text information --> - - diff --git a/report-viewer/src/model/Comparison.ts b/report-viewer/src/model/Comparison.ts index 6fafbc60f..c171fd1ac 100644 --- a/report-viewer/src/model/Comparison.ts +++ b/report-viewer/src/model/Comparison.ts @@ -1,5 +1,5 @@ import type { Match } from './Match' -import type { SubmissionFile } from '@/stores/state' +import type { SubmissionFile } from '@/model/File' import { MatchInSingleFile } from './MatchInSingleFile' import type { MetricType } from './MetricType' @@ -13,6 +13,8 @@ export class Comparison { private _filesOfFirstSubmission: SubmissionFile[] private _filesOfSecondSubmission: SubmissionFile[] private _allMatches: Array + private readonly _firstSimilarity?: number + private readonly _secondSimilarity?: number constructor( firstSubmissionId: string, @@ -20,7 +22,9 @@ export class Comparison { similarities: Record, filesOfFirstSubmission: SubmissionFile[], filesOfSecondSubmission: SubmissionFile[], - allMatches: Array + allMatches: Array, + firstSimilarity?: number, + secondSimilarity?: number ) { this._firstSubmissionId = firstSubmissionId this._secondSubmissionId = secondSubmissionId @@ -28,6 +32,8 @@ export class Comparison { this._filesOfFirstSubmission = filesOfFirstSubmission this._filesOfSecondSubmission = filesOfSecondSubmission this._allMatches = allMatches + this._firstSimilarity = firstSimilarity + this._secondSimilarity = secondSimilarity } /** @@ -86,6 +92,14 @@ export class Comparison { return this._similarities } + get firstSimilarity(): number | undefined { + return this._firstSimilarity + } + + get secondSimilarity(): number | undefined { + return this._secondSimilarity + } + private groupMatchesByFileName(index: 1 | 2): Map> { const acc = new Map>() this._allMatches.forEach((val) => { diff --git a/report-viewer/src/model/File.ts b/report-viewer/src/model/File.ts new file mode 100644 index 000000000..443c3eccb --- /dev/null +++ b/report-viewer/src/model/File.ts @@ -0,0 +1,31 @@ +/** + * Internal representation of a single file. + */ +export interface File { + /** + * The name of the file. + */ + fileName: string + /** + * The files content. + */ + data: string +} + +/** + * Internal representation of a single file from a submission. + */ +export interface SubmissionFile extends File { + /** + * The id of the submission. + */ + submissionId: string + /** + * Number of total tokens in the file. + */ + tokenCount?: number + /** + * Number of tokens in the file that are matched. + */ + matchedTokenCount: number +} diff --git a/report-viewer/src/model/Language.ts b/report-viewer/src/model/Language.ts index 0a833f3e7..1334d56e1 100644 --- a/report-viewer/src/model/Language.ts +++ b/report-viewer/src/model/Language.ts @@ -4,8 +4,10 @@ enum ParserLanguage { JAVA = 'Javac based AST plugin', PYTHON = 'Python3 Parser', - CPP = 'C/C++ Scanner [basic markup]', - CPP2 = 'C/C++ Parser', + C = 'C Scanner', + CPP_OLD = 'C/C++ Scanner [basic markup]', + CPP = 'C++ Parser', + CPP_2 = 'C/C++ Parser', C_SHARP = 'C# 6 Parser', EMF_METAMODEL_DYNAMIC = 'emf-dynamic', EMF_METAMODEL = 'EMF metamodel', diff --git a/report-viewer/src/model/factories/BaseFactory.ts b/report-viewer/src/model/factories/BaseFactory.ts index 29f94d95b..4a01d60c9 100644 --- a/report-viewer/src/model/factories/BaseFactory.ts +++ b/report-viewer/src/model/factories/BaseFactory.ts @@ -1,10 +1,12 @@ import { store } from '@/stores/store' -import { ZipFileHandler } from '@/utils/fileHandling/ZipFileHandler' +import { ZipFileHandler } from '@/model/fileHandling/ZipFileHandler' /** * This class provides some basic functionality for the factories. */ export class BaseFactory { + public static zipFileName = 'results.zip' + /** * Returns the content of a file through the stored loading type. * @param path - Path to the file @@ -17,16 +19,15 @@ export class BaseFactory { return this.getFileFromStore(path) } if (store().state.localModeUsed) { - if (store().state.zipModeUsed) { - await new ZipFileHandler().handleFile(await this.getLocalFile('results.zip')) - return this.getFileFromStore(path) - } else { - return await (await this.getLocalFile(`/files/${path}`)).text() - } + return await (await this.getLocalFile(`/files/${path}`)).text() } else if (store().state.zipModeUsed) { return this.getFileFromStore(path) } else if (store().state.singleModeUsed) { return store().state.singleFillRawContent + } else if (await this.useLocalZipMode()) { + await new ZipFileHandler().handleFile(await this.getLocalFile(this.zipFileName)) + store().setLoadingType('zip') + return this.getFileFromStore(path) } throw new Error('No loading type specified') } @@ -49,12 +50,26 @@ export class BaseFactory { * @return Content of the file * @throws Error if the file could not be found */ - protected static async getLocalFile(path: string): Promise { + public static async getLocalFile(path: string): Promise { const request = await fetch(`${window.location.origin}${import.meta.env.BASE_URL}${path}`) if (request.status == 200) { - return request.blob() + const blob = await request.blob() + // Check that file is not the index.html + if (blob.type == 'text/html') { + throw new Error(`Could not find ${path} in local files.`) + } + return blob } else { throw new Error(`Could not find ${path} in local files.`) } } + + public static async useLocalZipMode() { + try { + await this.getLocalFile(this.zipFileName) + return true + } catch (e) { + return false + } + } } diff --git a/report-viewer/src/model/factories/ComparisonFactory.ts b/report-viewer/src/model/factories/ComparisonFactory.ts index 8ff287efd..d6257e93e 100644 --- a/report-viewer/src/model/factories/ComparisonFactory.ts +++ b/report-viewer/src/model/factories/ComparisonFactory.ts @@ -10,13 +10,8 @@ import { MetricType } from '../MetricType' * Factory class for creating Comparison objects */ export class ComparisonFactory extends BaseFactory { - public static async getComparison(id1: string, id2: string): Promise { - const filePath = store().getComparisonFileName(id1, id2) - if (!filePath) { - throw new Error('Comparison file not specified') - } - - return await this.extractComparison(JSON.parse(await this.getFile(filePath))) + public static async getComparison(fileName: string): Promise { + return await this.extractComparison(JSON.parse(await this.getFile(fileName))) } /** @@ -26,14 +21,26 @@ export class ComparisonFactory extends BaseFactory { private static async extractComparison(json: Record): Promise { const firstSubmissionId = json.id1 as string const secondSubmissionId = json.id2 as string - if (store().state.localModeUsed && !store().state.zipModeUsed) { - await this.loadSubmissionFilesFromLocal(firstSubmissionId) - await this.loadSubmissionFilesFromLocal(secondSubmissionId) - } + await this.getFile(`submissionFileIndex.json`) + .then(async () => { + await this.loadSubmissionFiles(firstSubmissionId) + await this.loadSubmissionFiles(secondSubmissionId) + }) + .catch(() => {}) const filesOfFirstSubmission = store().filesOfSubmission(firstSubmissionId) const filesOfSecondSubmission = store().filesOfSubmission(secondSubmissionId) const matches = json.matches as Array> + matches.forEach((match) => { + store().getSubmissionFile( + firstSubmissionId, + slash(match.file1 as string) + ).matchedTokenCount += match.tokens as number + store().getSubmissionFile( + secondSubmissionId, + slash(match.file2 as string) + ).matchedTokenCount += match.tokens as number + }) const unColoredMatches = matches.map((match) => this.getMatch(match)) @@ -43,7 +50,9 @@ export class ComparisonFactory extends BaseFactory { this.extractSimilarities(json), filesOfFirstSubmission, filesOfSecondSubmission, - this.colorMatches(unColoredMatches) + this.colorMatches(unColoredMatches), + json.first_similarity as number | undefined, + json.second_similarity as number | undefined ) } @@ -76,20 +85,25 @@ export class ComparisonFactory extends BaseFactory { return similarities } - private static async getSubmissionFileListFromLocal(submissionId: string): Promise { - return JSON.parse( - await this.getLocalFile(`files/submissionFileIndex.json`).then((file) => file.text()) - ).submission_file_indexes[submissionId].map((file: string) => slash(file)) + private static async getSubmissionFileList( + submissionId: string + ): Promise> { + return JSON.parse(await this.getFile(`submissionFileIndex.json`)).submission_file_indexes[ + submissionId + ] } - private static async loadSubmissionFilesFromLocal(submissionId: string) { + private static async loadSubmissionFiles(submissionId: string) { try { - const fileList = await this.getSubmissionFileListFromLocal(submissionId) - for (const filePath of fileList) { + const fileList = await this.getSubmissionFileList(submissionId) + const fileNames = Object.keys(fileList) + for (const filePath of fileNames) { store().saveSubmissionFile({ fileName: slash(filePath), submissionId: submissionId, - data: await this.getLocalFile(`files/files/${filePath}`).then((file) => file.text()) + data: await this.getSubmissionFileContent(submissionId, slash(filePath)), + tokenCount: fileList[filePath].token_count, + matchedTokenCount: 0 }) } } catch (e) { @@ -97,6 +111,13 @@ export class ComparisonFactory extends BaseFactory { } } + private static async getSubmissionFileContent(submissionId: string, fileName: string) { + if (store().state.localModeUsed && !store().state.zipModeUsed) { + return await this.getLocalFile('files/' + fileName).then((file) => file.text()) + } + return store().getSubmissionFile(submissionId, fileName).data + } + private static getMatch(match: Record): Match { return { firstFile: slash(match.file1 as string), diff --git a/report-viewer/src/model/factories/OverviewFactory.ts b/report-viewer/src/model/factories/OverviewFactory.ts index 2e6b0a812..47ac75639 100644 --- a/report-viewer/src/model/factories/OverviewFactory.ts +++ b/report-viewer/src/model/factories/OverviewFactory.ts @@ -271,7 +271,7 @@ export class OverviewFactory extends BaseFactory { "The result's version(" + jsonVersion.toString() + ') is older than the minimal support version of the report viewer(' + - reportViewerVersion.toString() + + minimalVersion.toString() + '). ' + 'Can not read the report.' ) diff --git a/report-viewer/src/utils/fileHandling/FileHandler.ts b/report-viewer/src/model/fileHandling/FileHandler.ts similarity index 100% rename from report-viewer/src/utils/fileHandling/FileHandler.ts rename to report-viewer/src/model/fileHandling/FileHandler.ts diff --git a/report-viewer/src/utils/fileHandling/JsonFileHandler.ts b/report-viewer/src/model/fileHandling/JsonFileHandler.ts similarity index 100% rename from report-viewer/src/utils/fileHandling/JsonFileHandler.ts rename to report-viewer/src/model/fileHandling/JsonFileHandler.ts diff --git a/report-viewer/src/utils/fileHandling/ZipFileHandler.ts b/report-viewer/src/model/fileHandling/ZipFileHandler.ts similarity index 98% rename from report-viewer/src/utils/fileHandling/ZipFileHandler.ts rename to report-viewer/src/model/fileHandling/ZipFileHandler.ts index 59774aa5a..2e4d67d5f 100644 --- a/report-viewer/src/utils/fileHandling/ZipFileHandler.ts +++ b/report-viewer/src/model/fileHandling/ZipFileHandler.ts @@ -29,7 +29,8 @@ export class ZipFileHandler extends FileHandler { store().saveSubmissionFile({ submissionId: slash(submissionFileName), fileName: slash(fullPathFileName), - data: data + data: data, + matchedTokenCount: NaN }) }) } else { diff --git a/report-viewer/src/model/ui/DistributionChartConfig.ts b/report-viewer/src/model/ui/DistributionChartConfig.ts new file mode 100644 index 000000000..84222054f --- /dev/null +++ b/report-viewer/src/model/ui/DistributionChartConfig.ts @@ -0,0 +1,9 @@ +import type { MetricType } from '../MetricType' + +/** + * Configuration for the distribution chart. + */ +export interface DistributionChartConfig { + metric: MetricType + xScale: 'linear' | 'logarithmic' +} diff --git a/report-viewer/src/model/ui/ToolTip.ts b/report-viewer/src/model/ui/ToolTip.ts index c8149957c..363296b8b 100644 --- a/report-viewer/src/model/ui/ToolTip.ts +++ b/report-viewer/src/model/ui/ToolTip.ts @@ -2,3 +2,5 @@ export type ToolTipLabel = { displayValue: string tooltip: string } + +export type ToolTipDirection = 'top' | 'bottom' | 'left' | 'right' diff --git a/report-viewer/src/router/index.ts b/report-viewer/src/router/index.ts index 48afa3ffe..644c5c914 100644 --- a/report-viewer/src/router/index.ts +++ b/report-viewer/src/router/index.ts @@ -23,7 +23,7 @@ const router = createRouter({ component: OverviewViewWrapper }, { - path: '/comparison/:firstId/:secondId', + path: '/comparison/:comparisonFileName', name: 'ComparisonView', component: ComparisonViewWrapper, props: true @@ -62,7 +62,7 @@ function redirectOnError( router.push({ name: 'ErrorView', params: { - message: prefix + error.message, + message: prefix + (error.message ?? error), to: redirectRoute, routerInfo: redirectRouteTitle } diff --git a/report-viewer/src/stores/state.ts b/report-viewer/src/stores/state.ts index 7f71e818a..4e8c898fe 100644 --- a/report-viewer/src/stores/state.ts +++ b/report-viewer/src/stores/state.ts @@ -1,4 +1,6 @@ +import type { SubmissionFile } from '@/model/File' import type { MetricType } from '@/model/MetricType' +import type { DistributionChartConfig } from '@/model/ui/DistributionChartConfig' /** * Local store. Stores the state of the application. @@ -16,7 +18,7 @@ export interface State { * Stored files if zip mode is used. Stores the files as key - file name, value - file string */ files: Record - submissions: Record> + submissions: Record> /** * Indicates whether local mode is used. */ @@ -42,39 +44,6 @@ export interface State { uploadedFileName: string } -/** - * Internal representation of a single file. - */ -export interface File { - /** - * The name of the file. - */ - fileName: string - /** - * The files content. - */ - data: string -} - -/** - * Internal representation of a single file from a submission. - */ -export interface SubmissionFile extends File { - /** - * The id of the submission. - */ - submissionId: string -} - -/** - * Load configuration is used to indicate which mode is used. - */ -export interface LoadConfiguration { - local: boolean - zip: boolean - single: boolean -} - export interface UIState { useDarkMode: boolean comparisonTableSortingMetric: MetricType @@ -83,9 +52,10 @@ export interface UIState { } /** - * Configuration for the distribution chart. + * Load configuration is used to indicate which mode is used. */ -export interface DistributionChartConfig { - metric: MetricType - xScale: 'linear' | 'logarithmic' +export interface LoadConfiguration { + local: boolean + zip: boolean + single: boolean } diff --git a/report-viewer/src/stores/store.ts b/report-viewer/src/stores/store.ts index 6828e2ee1..9829d33d4 100644 --- a/report-viewer/src/stores/store.ts +++ b/report-viewer/src/stores/store.ts @@ -1,6 +1,7 @@ import { defineStore } from 'pinia' -import type { State, SubmissionFile, File, LoadConfiguration, UIState } from './state' +import type { State, UIState } from './state' import { MetricType } from '@/model/MetricType' +import type { SubmissionFile, File } from '@/model/File' /** * The store is a global state management system. It is used to store the state of the application. @@ -23,7 +24,7 @@ const store = defineStore('store', { uploadedFileName: '' }, uiState: { - useDarkMode: false, + useDarkMode: window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches, comparisonTableSortingMetric: MetricType.AVERAGE, comparisonTableClusterSorting: false, distributionChartConfig: { @@ -40,11 +41,16 @@ const store = defineStore('store', { filesOfSubmission: (state) => (submissionId: string): SubmissionFile[] => { - return Array.from(state.state.submissions[submissionId], ([name, value]) => ({ - submissionId, - fileName: name, - data: value - })) + return Array.from(state.state.submissions[submissionId].values()) + }, + /** + * @param submissionID the name of the submission + * @returns files a single file in the submission of the given name + */ + getSubmissionFile: + (state) => + (submissionId: string, fileName: string): SubmissionFile => { + return state.state.submissions[submissionId].get(fileName) as SubmissionFile }, /** * @param name the name of the submission @@ -83,7 +89,7 @@ const store = defineStore('store', { }, /** * @param id the id to check for - * @returns whether this submission should be anonymised + * @returns whether this submission should be anonymized */ isAnonymous: (state) => (submissionId: string) => { return state.state.anonymous.has(submissionId) @@ -185,17 +191,17 @@ const store = defineStore('store', { } this.state.submissions[submissionFile.submissionId].set( submissionFile.fileName, - submissionFile.data + submissionFile ) }, /** * Sets the loading type * @param payload Type used to input JPlag results */ - setLoadingType(payload: LoadConfiguration) { - this.state.localModeUsed = payload.local - this.state.zipModeUsed = payload.zip - this.state.singleModeUsed = payload.single + setLoadingType(loadingType: 'zip' | 'local' | 'single') { + this.state.localModeUsed = loadingType == 'local' + this.state.zipModeUsed = loadingType == 'zip' + this.state.singleModeUsed = loadingType == 'single' }, /** * Sets the raw content of the single file mode diff --git a/report-viewer/src/utils/CodeHighlighter.ts b/report-viewer/src/utils/CodeHighlighter.ts index 2d4b101f6..913e6d959 100644 --- a/report-viewer/src/utils/CodeHighlighter.ts +++ b/report-viewer/src/utils/CodeHighlighter.ts @@ -38,8 +38,11 @@ function getHighlightLanguage(lang: ParserLanguage) { switch (lang) { case ParserLanguage.PYTHON: return 'python' + case ParserLanguage.C: + return 'c' case ParserLanguage.CPP: - case ParserLanguage.CPP2: + case ParserLanguage.CPP_OLD: + case ParserLanguage.CPP_2: return 'cpp' case ParserLanguage.C_SHARP: return 'csharp' diff --git a/report-viewer/src/version.json b/report-viewer/src/version.json index 7b68fe18a..ed3b025ad 100644 --- a/report-viewer/src/version.json +++ b/report-viewer/src/version.json @@ -1,12 +1,12 @@ { "report_viewer_version": { - "major": 0, + "major": 5, "minor": 0, "patch": 0 }, "minimal_report_version": { "major": 4, - "minor": 0, + "minor": 2, "patch": 0 } } diff --git a/report-viewer/src/viewWrapper/ClusterViewWrapper.vue b/report-viewer/src/viewWrapper/ClusterViewWrapper.vue index 132cf4a78..6c3e2bbde 100644 --- a/report-viewer/src/viewWrapper/ClusterViewWrapper.vue +++ b/report-viewer/src/viewWrapper/ClusterViewWrapper.vue @@ -6,6 +6,8 @@ >
+ +