From 918c7093f27aae6cba8056e9363242b62a3b3aa8 Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 3 Sep 2024 15:33:57 +0200 Subject: [PATCH 1/5] Added token position tests for annotations --- .../resources/de/jplag/java/tokenPositions/Anno_1.java | 3 +++ .../resources/de/jplag/java/tokenPositions/Anno_2.java | 3 +++ .../resources/de/jplag/java/tokenPositions/Anno_3.java | 6 ++++++ .../resources/de/jplag/java/tokenPositions/Anno_4.java | 7 +++++++ .../de/jplag/java/tokenPositions/JAnnoTBegin-End_1.java | 3 +++ .../de/jplag/java/tokenPositions/JAnnoTBegin-End_2.java | 4 ++++ 6 files changed, 26 insertions(+) create mode 100644 languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_1.java create mode 100644 languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_2.java create mode 100644 languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_3.java create mode 100644 languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_4.java create mode 100644 languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_1.java create mode 100644 languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_2.java diff --git a/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_1.java b/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_1.java new file mode 100644 index 000000000..e78c1a3f9 --- /dev/null +++ b/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_1.java @@ -0,0 +1,3 @@ +>@SuppressWarnings +$| J_ANNO 17 +>class Anno {} \ No newline at end of file diff --git a/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_2.java b/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_2.java new file mode 100644 index 000000000..54f9b612a --- /dev/null +++ b/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_2.java @@ -0,0 +1,3 @@ +>@SuppressWarnings("SomeWarning") +$| J_ANNO 17 +>class Anno {} \ No newline at end of file diff --git a/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_3.java b/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_3.java new file mode 100644 index 000000000..522a18dae --- /dev/null +++ b/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_3.java @@ -0,0 +1,6 @@ +>class Anno { +> @Deprecated +$ | J_ANNO 11 +> public void testFunction() { +> } +>} \ No newline at end of file diff --git a/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_4.java b/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_4.java new file mode 100644 index 000000000..92db1e255 --- /dev/null +++ b/languages/java/src/test/resources/de/jplag/java/tokenPositions/Anno_4.java @@ -0,0 +1,7 @@ +>import javax.annotation.processing.Generated; +> +>class Anno { +> public void testMethod(@Generated String param) { +$ | J_ANNO 10 +> } +>} diff --git a/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_1.java b/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_1.java new file mode 100644 index 000000000..b072549bb --- /dev/null +++ b/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_1.java @@ -0,0 +1,3 @@ +>public @interface MyAnnotation {} +$| J_ANNO_T_BEGIN 30 +$ | J_ANNO_T_END 1 \ No newline at end of file diff --git a/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_2.java b/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_2.java new file mode 100644 index 000000000..13ea969ca --- /dev/null +++ b/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_2.java @@ -0,0 +1,4 @@ +>public @interface MyAnnotation { +$| J_ANNO_T_BEGIN 30 +>} +$| J_ANNO_T_END 1 From 500bbde74751f5206fbb3ffeefc99fea87039c7f Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Thu, 12 Sep 2024 12:44:17 +0200 Subject: [PATCH 2/5] Fixed annotation token positions in java module --- .../java/de/jplag/java/TokenGeneratingTreeScanner.java | 8 +++++--- .../de/jplag/java/tokenPositions/JAnnoTBegin-End_1.java | 2 +- .../de/jplag/java/tokenPositions/JAnnoTBegin-End_2.java | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java b/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java index 9589da81b..c1c306aa8 100644 --- a/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java +++ b/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java @@ -136,7 +136,8 @@ public Void visitClass(ClassTree node, Void unused) { } } - long start = positions.getStartPosition(ast, node); + long start = positions.getEndPosition(ast, node.getModifiers()) + 1; + long nameLength = node.getSimpleName().length(); long end = positions.getEndPosition(ast, node) - 1; CodeSemantics semantics = CodeSemantics.createControl(); if (node.getKind() == Tree.Kind.ENUM) { @@ -146,7 +147,7 @@ public Void visitClass(ClassTree node, Void unused) { } else if (node.getKind() == Tree.Kind.RECORD) { addToken(JavaTokenType.J_RECORD_BEGIN, start, 1, semantics); } else if (node.getKind() == Tree.Kind.ANNOTATION_TYPE) { - addToken(JavaTokenType.J_ANNO_T_BEGIN, start, 10, semantics); + addToken(JavaTokenType.J_ANNO_T_BEGIN, start - 2 /*@ is final modifier for annotations*/, (start - 2) + 11 + nameLength, semantics); } else if (node.getKind() == Tree.Kind.CLASS) { addToken(JavaTokenType.J_CLASS_BEGIN, start, 5, semantics); } @@ -508,7 +509,8 @@ public Void visitMethodInvocation(MethodInvocationTree node, Void unused) { @Override public Void visitAnnotation(AnnotationTree node, Void unused) { long start = positions.getStartPosition(ast, node); - addToken(JavaTokenType.J_ANNO, start, 1, new CodeSemantics()); + String annotationName = node.getAnnotationType().toString(); + addToken(JavaTokenType.J_ANNO, start, annotationName.length() + 1, new CodeSemantics()); return super.visitAnnotation(node, null); } diff --git a/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_1.java b/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_1.java index b072549bb..119972961 100644 --- a/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_1.java +++ b/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_1.java @@ -1,3 +1,3 @@ >public @interface MyAnnotation {} -$| J_ANNO_T_BEGIN 30 +$ | J_ANNO_T_BEGIN 23 $ | J_ANNO_T_END 1 \ No newline at end of file diff --git a/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_2.java b/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_2.java index 13ea969ca..0d0d89d1e 100644 --- a/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_2.java +++ b/languages/java/src/test/resources/de/jplag/java/tokenPositions/JAnnoTBegin-End_2.java @@ -1,4 +1,4 @@ >public @interface MyAnnotation { -$| J_ANNO_T_BEGIN 30 +$ | J_ANNO_T_BEGIN 23 >} $| J_ANNO_T_END 1 From 0f0efea80baadd2caa8cf1036d65e53d9fcfb8ab Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 29 Oct 2024 15:44:30 +0100 Subject: [PATCH 3/5] Fixed token order test by ignoring certain token types. --- .../test/java/de/jplag/testutils/LanguageModuleTest.java | 7 ++++++- .../src/test/java/de/jplag/java/JavaLanguageTest.java | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/language-testutils/src/test/java/de/jplag/testutils/LanguageModuleTest.java b/language-testutils/src/test/java/de/jplag/testutils/LanguageModuleTest.java index f659ba2b2..1681903a5 100644 --- a/language-testutils/src/test/java/de/jplag/testutils/LanguageModuleTest.java +++ b/language-testutils/src/test/java/de/jplag/testutils/LanguageModuleTest.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.junit.jupiter.api.AfterAll; @@ -247,7 +248,7 @@ final List getTokenPositionTestData() { @MethodSource("getAllTestData") @DisplayName("Test that the tokens map to ascending line numbers") final void testMonotoneTokenOrder(TestData data) throws ParsingException, IOException { - List tokens = parseTokens(data); + List tokens = parseTokens(data).stream().filter(it -> !getIgnoredTokensForMonotoneTokenOrder().contains(it.getType())).toList(); for (int i = 0; i < tokens.size() - 2; i++) { Token first = tokens.get(i); @@ -338,4 +339,8 @@ private > C ignoreEmptyTestType(C data) { protected File getTestFileLocation() { return new File(DEFAULT_TEST_CODE_PATH_BASE.toFile(), this.language.getIdentifier()); } + + protected List getIgnoredTokensForMonotoneTokenOrder() { + return Collections.emptyList(); + } } diff --git a/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java b/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java index a4f89d9bb..03a0cd06c 100644 --- a/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java +++ b/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java @@ -29,10 +29,13 @@ import static de.jplag.java.JavaTokenType.J_TRY_END; import static de.jplag.java.JavaTokenType.J_VARDEF; +import de.jplag.TokenType; import de.jplag.testutils.LanguageModuleTest; import de.jplag.testutils.datacollector.TestDataCollector; import de.jplag.testutils.datacollector.TestSourceIgnoredLinesCollector; +import java.util.List; + public class JavaLanguageTest extends LanguageModuleTest { public JavaLanguageTest() { super(new JavaLanguage(), JavaTokenType.class); @@ -84,4 +87,9 @@ protected void configureIgnoredLines(TestSourceIgnoredLinesCollector collector) collector.ignoreLinesByPrefix("})"); collector.ignoreByCondition(line -> line.contains("else") && !line.contains("if")); } + + @Override + protected List getIgnoredTokensForMonotoneTokenOrder() { + return List.of(JavaTokenType.J_ANNO); + } } From 009f6004b57518746166ee302c88df3506b6f0c1 Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 29 Oct 2024 15:44:55 +0100 Subject: [PATCH 4/5] Spotless --- .../main/java/de/jplag/java/TokenGeneratingTreeScanner.java | 2 +- .../java/src/test/java/de/jplag/java/JavaLanguageTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java b/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java index c1c306aa8..8bfbeb266 100644 --- a/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java +++ b/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java @@ -147,7 +147,7 @@ public Void visitClass(ClassTree node, Void unused) { } else if (node.getKind() == Tree.Kind.RECORD) { addToken(JavaTokenType.J_RECORD_BEGIN, start, 1, semantics); } else if (node.getKind() == Tree.Kind.ANNOTATION_TYPE) { - addToken(JavaTokenType.J_ANNO_T_BEGIN, start - 2 /*@ is final modifier for annotations*/, (start - 2) + 11 + nameLength, semantics); + addToken(JavaTokenType.J_ANNO_T_BEGIN, start - 2 /* @ is final modifier for annotations */, (start - 2) + 11 + nameLength, semantics); } else if (node.getKind() == Tree.Kind.CLASS) { addToken(JavaTokenType.J_CLASS_BEGIN, start, 5, semantics); } diff --git a/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java b/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java index 03a0cd06c..8e9320553 100644 --- a/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java +++ b/languages/java/src/test/java/de/jplag/java/JavaLanguageTest.java @@ -29,13 +29,13 @@ import static de.jplag.java.JavaTokenType.J_TRY_END; import static de.jplag.java.JavaTokenType.J_VARDEF; +import java.util.List; + import de.jplag.TokenType; import de.jplag.testutils.LanguageModuleTest; import de.jplag.testutils.datacollector.TestDataCollector; import de.jplag.testutils.datacollector.TestSourceIgnoredLinesCollector; -import java.util.List; - public class JavaLanguageTest extends LanguageModuleTest { public JavaLanguageTest() { super(new JavaLanguage(), JavaTokenType.class); From caa1eb12f5f0f5b8c24a9ae281d43b55e0ccc213 Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Mon, 4 Nov 2024 10:53:22 +0100 Subject: [PATCH 5/5] Fixed formatting --- .../main/java/de/jplag/java/TokenGeneratingTreeScanner.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java b/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java index 8bfbeb266..503c84fb0 100644 --- a/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java +++ b/languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java @@ -147,7 +147,9 @@ public Void visitClass(ClassTree node, Void unused) { } else if (node.getKind() == Tree.Kind.RECORD) { addToken(JavaTokenType.J_RECORD_BEGIN, start, 1, semantics); } else if (node.getKind() == Tree.Kind.ANNOTATION_TYPE) { - addToken(JavaTokenType.J_ANNO_T_BEGIN, start - 2 /* @ is final modifier for annotations */, (start - 2) + 11 + nameLength, semantics); + // The start position for the is calculated that way, because the @ is the final element in the modifier list for + // annotations + addToken(JavaTokenType.J_ANNO_T_BEGIN, start - 2, (start - 2) + 11 + nameLength, semantics); } else if (node.getKind() == Tree.Kind.CLASS) { addToken(JavaTokenType.J_CLASS_BEGIN, start, 5, semantics); }