From 941e5ee3cb46e2330658282ae2459d8bd8964c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Ma=CC=8Artensson?= Date: Wed, 10 Feb 2021 16:02:49 +0100 Subject: [PATCH 1/9] Add position to ReferenceType --- javalang/parser.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/javalang/parser.py b/javalang/parser.py index 50b0aff..2b9a8aa 100644 --- a/javalang/parser.py +++ b/javalang/parser.py @@ -473,11 +473,16 @@ def parse_type(self): @parse_debug def parse_basic_type(self): - return tree.BasicType(name=self.accept(BasicType)) + token = self.tokens.look() + basic_type = tree.BasicType(name=self.accept(BasicType)) + basic_type._position = token.position + return basic_type @parse_debug def parse_reference_type(self): + token = self.tokens.look() reference_type = tree.ReferenceType() + reference_type._position = token.position tail = reference_type while True: @@ -516,11 +521,14 @@ def parse_type_argument(self): pattern_type = None base_type = None + token = self.tokens.look() if self.try_accept('?'): - if self.tokens.look().value in ('extends', 'super'): - pattern_type = self.tokens.next().value + if token.value in ('extends', 'super'): + pattern_type = tokens.next().value else: - return tree.TypeArgument(pattern_type='?') + type_argument = tree.TypeArgument(pattern_type='?') + type_argument._position = token.position + return type_argument if self.would_accept(BasicType): base_type = self.parse_basic_type() @@ -532,8 +540,10 @@ def parse_type_argument(self): base_type.dimensions += self.parse_array_dimension() - return tree.TypeArgument(type=base_type, + type_argument = tree.TypeArgument(type=base_type, pattern_type=pattern_type) + type_argument._position = token.position + return type_argument @parse_debug def parse_nonwildcard_type_arguments(self): From c3a186b38bfb5aabc6fa7a443f408a3851fe4e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Ma=CC=8Artensson?= Date: Mon, 1 Mar 2021 13:27:35 +0100 Subject: [PATCH 2/9] Expose Identifiers instead of str, giving Position data --- javalang/parser.py | 15 ++++++++++----- javalang/test/test_java_8_syntax.py | 2 +- javalang/test/test_package_declaration.py | 10 +++++----- javalang/tree.py | 7 +++++++ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/javalang/parser.py b/javalang/parser.py index 2b9a8aa..929855b 100644 --- a/javalang/parser.py +++ b/javalang/parser.py @@ -229,7 +229,10 @@ def is_annotation_declaration(self, i=0): @parse_debug def parse_identifier(self): - return self.accept(Identifier) + token = self.tokens.look() + identifier = tree.Identifier(value=self.accept(Identifier)) + identifier._position = token.position + return identifier @parse_debug def parse_qualified_identifier(self): @@ -242,7 +245,7 @@ def parse_qualified_identifier(self): if not self.try_accept('.'): break - return '.'.join(qualified_identifier) + return qualified_identifier @parse_debug def parse_qualified_identifier_list(self): @@ -335,7 +338,7 @@ def parse_import_declaration(self): self.accept(';') break - return tree.Import(path='.'.join(qualified_identifier), + return tree.Import(path=qualified_identifier, static=static, wildcard=import_all) @@ -522,6 +525,7 @@ def parse_type_argument(self): base_type = None token = self.tokens.look() + if self.try_accept('?'): if token.value in ('extends', 'super'): pattern_type = tokens.next().value @@ -540,6 +544,7 @@ def parse_type_argument(self): base_type.dimensions += self.parse_array_dimension() + token = self.tokens.look() type_argument = tree.TypeArgument(type=base_type, pattern_type=pattern_type) type_argument._position = token.position @@ -1147,7 +1152,7 @@ def parse_formal_parameters(self): while True: modifiers, annotations = self.parse_variable_modifiers() - + token = self.tokens.look() parameter_type = self.parse_type() varargs = False @@ -2013,7 +2018,7 @@ def parse_primary(self): identifier_suffix.type = tree.ReferenceType(name=qualified_identifier.pop()) identifier_suffix._position = token.position - identifier_suffix.qualifier = '.'.join(qualified_identifier) + identifier_suffix.qualifier = qualified_identifier return identifier_suffix diff --git a/javalang/test/test_java_8_syntax.py b/javalang/test/test_java_8_syntax.py index 0a8c8fd..cb55af1 100644 --- a/javalang/test/test_java_8_syntax.py +++ b/javalang/test/test_java_8_syntax.py @@ -26,7 +26,7 @@ def filter_type_in_method(clazz, the_type, method_name): for path, node in clazz.filter(the_type): for p in reversed(path): if isinstance(p, tree.MethodDeclaration): - if p.name == method_name: + if p.name.value == method_name: yield path, node diff --git a/javalang/test/test_package_declaration.py b/javalang/test/test_package_declaration.py index 880ac67..c3d10ad 100644 --- a/javalang/test/test_package_declaration.py +++ b/javalang/test/test_package_declaration.py @@ -11,7 +11,7 @@ def testPackageDeclarationOnly(self): source_file = "source/package-info/NoAnnotationNoJavadoc.java" ast = self.get_ast(source_file) - self.failUnless(ast.package.name == "org.javalang.test") + self.failUnless(ast.package.as_java() == "org.javalang.test") self.failIf(ast.package.annotations) self.failIf(ast.package.documentation) @@ -19,7 +19,7 @@ def testAnnotationOnly(self): source_file = "source/package-info/AnnotationOnly.java" ast = self.get_ast(source_file) - self.failUnless(ast.package.name == "org.javalang.test") + self.failUnless(ast.package.as_java() == "org.javalang.test") self.failUnless(ast.package.annotations) self.failIf(ast.package.documentation) @@ -27,7 +27,7 @@ def testJavadocOnly(self): source_file = "source/package-info/JavadocOnly.java" ast = self.get_ast(source_file) - self.failUnless(ast.package.name == "org.javalang.test") + self.failUnless(ast.package.as_java() == "org.javalang.test") self.failIf(ast.package.annotations) self.failUnless(ast.package.documentation) @@ -35,7 +35,7 @@ def testAnnotationThenJavadoc(self): source_file = "source/package-info/AnnotationJavadoc.java" ast = self.get_ast(source_file) - self.failUnless(ast.package.name == "org.javalang.test") + self.failUnless(ast.package.as_java() == "org.javalang.test") self.failUnless(ast.package.annotations) self.failIf(ast.package.documentation) @@ -43,7 +43,7 @@ def testJavadocThenAnnotation(self): source_file = "source/package-info/JavadocAnnotation.java" ast = self.get_ast(source_file) - self.failUnless(ast.package.name == "org.javalang.test") + self.failUnless(ast.package.as_java() == "org.javalang.test") self.failUnless(ast.package.annotations) self.failUnless(ast.package.documentation) diff --git a/javalang/tree.py b/javalang/tree.py index 77e7a37..5a2619a 100644 --- a/javalang/tree.py +++ b/javalang/tree.py @@ -33,6 +33,9 @@ def constructors(self): class PackageDeclaration(Declaration, Documented): attrs = ("name",) + def as_java(self): + return '.'.join(map(lambda q: q.value, self.name)) + class ClassDeclaration(TypeDeclaration): attrs = ("type_parameters", "extends", "implements") @@ -278,3 +281,7 @@ class EnumConstantDeclaration(Declaration, Documented): class AnnotationMethod(Declaration): attrs = ("name", "return_type", "dimensions", "default") +# ------------------------------------------------------------------------------ + +class Identifier(Node): + attrs = ("value",) From 6512daa1d94157a60c2a656de32024fcef18996a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Ma=CC=8Artensson?= Date: Mon, 1 Mar 2021 14:22:57 +0100 Subject: [PATCH 3/9] Introduce QualifiedIdentifier --- javalang/parser.py | 2 +- javalang/tree.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/javalang/parser.py b/javalang/parser.py index 929855b..afe8be2 100644 --- a/javalang/parser.py +++ b/javalang/parser.py @@ -245,7 +245,7 @@ def parse_qualified_identifier(self): if not self.try_accept('.'): break - return qualified_identifier + return tree.QualifiedIdentifier(values=qualified_identifier) @parse_debug def parse_qualified_identifier_list(self): diff --git a/javalang/tree.py b/javalang/tree.py index 5a2619a..824d809 100644 --- a/javalang/tree.py +++ b/javalang/tree.py @@ -34,7 +34,7 @@ class PackageDeclaration(Declaration, Documented): attrs = ("name",) def as_java(self): - return '.'.join(map(lambda q: q.value, self.name)) + return self.name.as_java() class ClassDeclaration(TypeDeclaration): attrs = ("type_parameters", "extends", "implements") @@ -282,6 +282,11 @@ class AnnotationMethod(Declaration): attrs = ("name", "return_type", "dimensions", "default") # ------------------------------------------------------------------------------ +class QualifiedIdentifier(Node): + attrs = ("values",) + + def as_java(self): + return '.'.join(map(lambda q: q.value, self.values)) class Identifier(Node): attrs = ("value",) From 7e759343ae1d9f39c869c0387da3d02aed033bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Ma=CC=8Artensson?= Date: Mon, 1 Mar 2021 14:39:18 +0100 Subject: [PATCH 4/9] Use QualifiedIdentifier --- javalang/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/javalang/parser.py b/javalang/parser.py index afe8be2..bff82a7 100644 --- a/javalang/parser.py +++ b/javalang/parser.py @@ -338,7 +338,7 @@ def parse_import_declaration(self): self.accept(';') break - return tree.Import(path=qualified_identifier, + return tree.Import(path=tree.QualifiedIdentifier(values=qualified_identifier), static=static, wildcard=import_all) @@ -2018,7 +2018,7 @@ def parse_primary(self): identifier_suffix.type = tree.ReferenceType(name=qualified_identifier.pop()) identifier_suffix._position = token.position - identifier_suffix.qualifier = qualified_identifier + identifier_suffix.qualifier = tree.QualifiedIdentifier(values=qualified_identifier) return identifier_suffix From a2856a7e143763f6ef40b602a7285af6eccfc735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Ma=CC=8Artensson?= Date: Mon, 1 Mar 2021 18:14:51 +0100 Subject: [PATCH 5/9] MethodInvocation was initiated with str sometimes, use Identifier --- javalang/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javalang/parser.py b/javalang/parser.py index bff82a7..5fdd71a 100644 --- a/javalang/parser.py +++ b/javalang/parser.py @@ -2268,7 +2268,7 @@ def parse_selector(self): token = self.tokens.look() if isinstance(token, Identifier): - identifier = self.tokens.next().value + identifier = self.tokens.next() arguments = None if self.would_accept('('): From 29f083ea6c91dd97e74d70e80d626da4d0419fda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Ma=CC=8Artensson?= Date: Wed, 3 Mar 2021 23:13:36 +0100 Subject: [PATCH 6/9] Add end_separator to Invocation --- javalang/parser.py | 8 +++++--- javalang/tree.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/javalang/parser.py b/javalang/parser.py index 5fdd71a..e88cc33 100644 --- a/javalang/parser.py +++ b/javalang/parser.py @@ -2104,7 +2104,8 @@ def parse_explicit_generic_invocation_suffix(self): identifier = self.parse_identifier() arguments = self.parse_arguments() return tree.MethodInvocation(member=identifier, - arguments=arguments) + arguments=arguments, + end_separator=self.tokens.last()) # ------------------------------------------------------------------------------ # -- Creators -- @@ -2197,7 +2198,7 @@ def parse_identifier_suffix(self): elif self.would_accept('('): arguments = self.parse_arguments() - return tree.MethodInvocation(arguments=arguments) + return tree.MethodInvocation(arguments=arguments, end_separator=self.tokens.last()) elif self.try_accept('.', 'class'): return tree.ClassReference() @@ -2275,7 +2276,8 @@ def parse_selector(self): arguments = self.parse_arguments() return tree.MethodInvocation(member=identifier, - arguments=arguments) + arguments=arguments, + end_separator=self.tokens.last()) else: return tree.MemberReference(member=identifier) elif self.would_accept('super', '::'): diff --git a/javalang/tree.py b/javalang/tree.py index 824d809..e90a1ac 100644 --- a/javalang/tree.py +++ b/javalang/tree.py @@ -230,7 +230,7 @@ class MemberReference(Primary): attrs = ("member",) class Invocation(Primary): - attrs = ("type_arguments", "arguments") + attrs = ("type_arguments", "arguments", "end_separator") class ExplicitConstructorInvocation(Invocation): attrs = () From d95aa9740f8681d5fd1753548022910d4a3c8840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Ma=CC=8Artensson?= Date: Fri, 5 Mar 2021 10:28:04 +0100 Subject: [PATCH 7/9] Add end_separator to Declaration --- javalang/parser.py | 6 ++++-- javalang/tree.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/javalang/parser.py b/javalang/parser.py index e88cc33..01e04d8 100644 --- a/javalang/parser.py +++ b/javalang/parser.py @@ -905,7 +905,8 @@ def parse_method_declarator_rest(self): return tree.MethodDeclaration(parameters=formal_parameters, throws=throws, body=body, - return_type=tree.Type(dimensions=additional_dimensions)) + return_type=tree.Type(dimensions=additional_dimensions), + end_separator=self.tokens.last()) @parse_debug def parse_void_method_declarator_rest(self): @@ -923,7 +924,8 @@ def parse_void_method_declarator_rest(self): return tree.MethodDeclaration(parameters=formal_parameters, throws=throws, - body=body) + body=body, + end_separator=self.tokens.last()) @parse_debug def parse_constructor_declarator_rest(self): diff --git a/javalang/tree.py b/javalang/tree.py index e90a1ac..ae26776 100644 --- a/javalang/tree.py +++ b/javalang/tree.py @@ -13,7 +13,7 @@ class Documented(Node): attrs = ("documentation",) class Declaration(Node): - attrs = ("modifiers", "annotations") + attrs = ("modifiers", "annotations", "end_separator") class TypeDeclaration(Declaration, Documented): attrs = ("name", "body") From 2ec19f4f2f3ab76c48bcba8c9462364fd5ebea65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Ma=CC=8Artensson?= Date: Sat, 6 Mar 2021 16:25:11 +0100 Subject: [PATCH 8/9] Allow int[]::new style class constructor reference --- javalang/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javalang/parser.py b/javalang/parser.py index 01e04d8..daa1447 100644 --- a/javalang/parser.py +++ b/javalang/parser.py @@ -2195,7 +2195,7 @@ def parse_array_creator_rest(self): def parse_identifier_suffix(self): if self.try_accept('[', ']'): array_dimension = [None] + self.parse_array_dimension() - self.accept('.', 'class') + self.try_accept('.', 'class') return tree.ClassReference(type=tree.Type(dimensions=array_dimension)) elif self.would_accept('('): From c63075dddae588ca25d50e39def6156a2b9510d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joachim=20Ma=CC=8Artensson?= Date: Sat, 6 Mar 2021 17:00:29 +0100 Subject: [PATCH 9/9] assign MemberReference.member full KeyWord --- javalang/parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/javalang/parser.py b/javalang/parser.py index daa1447..26d273d 100644 --- a/javalang/parser.py +++ b/javalang/parser.py @@ -1909,7 +1909,8 @@ def parse_method_reference(self): if self.would_accept('<'): type_arguments = self.parse_nonwildcard_type_arguments() if self.would_accept('new'): - method_reference = tree.MemberReference(member=self.accept('new')) + self.accept('new') + method_reference = tree.MemberReference(member=self.tokens.last()) else: method_reference = self.parse_expression() return method_reference, type_arguments