Add position to ReferenceType

javalang/parser.py

@@ -229,7 +229,10 @@ def is_annotation_declaration(self, i=0):
 
     @parse_debug
     def parse_identifier(self):
-        return self.accept(Identifier)
+        token = self.tokens.look()
+        identifier = tree.Identifier(value=self.accept(Identifier))
+        identifier._position = token.position
+        return identifier
 
     @parse_debug
     def parse_qualified_identifier(self):
@@ -242,7 +245,7 @@ def parse_qualified_identifier(self):
             if not self.try_accept('.'):
                 break
 
-        return '.'.join(qualified_identifier)
+        return tree.QualifiedIdentifier(values=qualified_identifier)
 
     @parse_debug
     def parse_qualified_identifier_list(self):
@@ -335,7 +338,7 @@ def parse_import_declaration(self):
                 self.accept(';')
                 break
 
-        return tree.Import(path='.'.join(qualified_identifier),
+        return tree.Import(path=tree.QualifiedIdentifier(values=qualified_identifier),
                            static=static,
                            wildcard=import_all)
 
@@ -473,11 +476,16 @@ def parse_type(self):
 
     @parse_debug
     def parse_basic_type(self):
-        return tree.BasicType(name=self.accept(BasicType))
+        token = self.tokens.look()
+        basic_type = tree.BasicType(name=self.accept(BasicType))
+        basic_type._position = token.position
+        return basic_type
 
     @parse_debug
     def parse_reference_type(self):
+        token = self.tokens.look()
         reference_type = tree.ReferenceType()
+        reference_type._position = token.position
         tail = reference_type
 
         while True:
@@ -516,11 +524,15 @@ def parse_type_argument(self):
         pattern_type = None
         base_type = None
 
+        token = self.tokens.look()
+
         if self.try_accept('?'):
-            if self.tokens.look().value in ('extends', 'super'):
-                pattern_type = self.tokens.next().value
+            if token.value in ('extends', 'super'):
+                pattern_type = tokens.next().value
             else:
-                return tree.TypeArgument(pattern_type='?')
+                type_argument = tree.TypeArgument(pattern_type='?')
+                type_argument._position = token.position
+                return type_argument
 
         if self.would_accept(BasicType):
             base_type = self.parse_basic_type()
@@ -532,8 +544,11 @@ def parse_type_argument(self):
 
         base_type.dimensions += self.parse_array_dimension()
 
-        return tree.TypeArgument(type=base_type,
+        token = self.tokens.look()
+        type_argument = tree.TypeArgument(type=base_type,
                                  pattern_type=pattern_type)
+        type_argument._position = token.position
+        return type_argument
 
     @parse_debug
     def parse_nonwildcard_type_arguments(self):
@@ -890,7 +905,8 @@ def parse_method_declarator_rest(self):
         return tree.MethodDeclaration(parameters=formal_parameters,
                                      throws=throws,
                                      body=body,
-                                     return_type=tree.Type(dimensions=additional_dimensions))
+                                     return_type=tree.Type(dimensions=additional_dimensions),
+                                     end_separator=self.tokens.last())
 
     @parse_debug
     def parse_void_method_declarator_rest(self):
@@ -908,7 +924,8 @@ def parse_void_method_declarator_rest(self):
 
         return tree.MethodDeclaration(parameters=formal_parameters,
                                       throws=throws,
-                                      body=body)
+                                      body=body,
+                                      end_separator=self.tokens.last())
 
     @parse_debug
     def parse_constructor_declarator_rest(self):
@@ -1137,7 +1154,7 @@ def parse_formal_parameters(self):
 
         while True:
             modifiers, annotations = self.parse_variable_modifiers()
-            
+
             token = self.tokens.look()
             parameter_type = self.parse_type()
             varargs = False
@@ -1892,7 +1909,8 @@ def parse_method_reference(self):
         if self.would_accept('<'):
             type_arguments = self.parse_nonwildcard_type_arguments()
         if self.would_accept('new'):
-            method_reference = tree.MemberReference(member=self.accept('new'))
+            self.accept('new')
+            method_reference = tree.MemberReference(member=self.tokens.last())
         else:
             method_reference = self.parse_expression()
         return method_reference, type_arguments
@@ -2003,7 +2021,7 @@ def parse_primary(self):
                 identifier_suffix.type = tree.ReferenceType(name=qualified_identifier.pop())
 
             identifier_suffix._position = token.position
-            identifier_suffix.qualifier = '.'.join(qualified_identifier)
+            identifier_suffix.qualifier = tree.QualifiedIdentifier(values=qualified_identifier)
 
             return identifier_suffix
 
@@ -2089,7 +2107,8 @@ def parse_explicit_generic_invocation_suffix(self):
             identifier = self.parse_identifier()
             arguments = self.parse_arguments()
             return tree.MethodInvocation(member=identifier,
-                                         arguments=arguments)
+                                         arguments=arguments,
+                                         end_separator=self.tokens.last())
 
 # ------------------------------------------------------------------------------
 # -- Creators --
@@ -2177,12 +2196,12 @@ def parse_array_creator_rest(self):
     def parse_identifier_suffix(self):
         if self.try_accept('[', ']'):
             array_dimension = [None] + self.parse_array_dimension()
-            self.accept('.', 'class')
+            self.try_accept('.', 'class')
             return tree.ClassReference(type=tree.Type(dimensions=array_dimension))
 
         elif self.would_accept('('):
             arguments = self.parse_arguments()
-            return tree.MethodInvocation(arguments=arguments)
+            return tree.MethodInvocation(arguments=arguments, end_separator=self.tokens.last())
 
         elif self.try_accept('.', 'class'):
             return tree.ClassReference()
@@ -2253,14 +2272,15 @@ def parse_selector(self):
 
             token = self.tokens.look()
             if isinstance(token, Identifier):
-                identifier = self.tokens.next().value
+                identifier = self.tokens.next()
                 arguments = None
 
                 if self.would_accept('('):
                     arguments = self.parse_arguments()
 
                     return tree.MethodInvocation(member=identifier,
-                                                 arguments=arguments)
+                                                 arguments=arguments,
+                                                 end_separator=self.tokens.last())
                 else:
                     return tree.MemberReference(member=identifier)
             elif self.would_accept('super', '::'):

javalang/test/test_java_8_syntax.py

@@ -26,7 +26,7 @@ def filter_type_in_method(clazz, the_type, method_name):
     for path, node in clazz.filter(the_type):
         for p in reversed(path):
             if isinstance(p, tree.MethodDeclaration):
-                if p.name == method_name:
+                if p.name.value == method_name:
                     yield path, node
 
 

javalang/test/test_package_declaration.py

@@ -11,39 +11,39 @@ def testPackageDeclarationOnly(self):
         source_file = "source/package-info/NoAnnotationNoJavadoc.java"
         ast = self.get_ast(source_file)
 
-        self.failUnless(ast.package.name == "org.javalang.test")
+        self.failUnless(ast.package.as_java() == "org.javalang.test")
         self.failIf(ast.package.annotations)
         self.failIf(ast.package.documentation)
 
     def testAnnotationOnly(self):
         source_file = "source/package-info/AnnotationOnly.java"
         ast = self.get_ast(source_file)
 
-        self.failUnless(ast.package.name == "org.javalang.test")
+        self.failUnless(ast.package.as_java() == "org.javalang.test")
         self.failUnless(ast.package.annotations)
         self.failIf(ast.package.documentation)
 
     def testJavadocOnly(self):
         source_file = "source/package-info/JavadocOnly.java"
         ast = self.get_ast(source_file)
 
-        self.failUnless(ast.package.name == "org.javalang.test")
+        self.failUnless(ast.package.as_java() == "org.javalang.test")
         self.failIf(ast.package.annotations)
         self.failUnless(ast.package.documentation)
 
     def testAnnotationThenJavadoc(self):
         source_file = "source/package-info/AnnotationJavadoc.java"
         ast = self.get_ast(source_file)
 
-        self.failUnless(ast.package.name == "org.javalang.test")
+        self.failUnless(ast.package.as_java() == "org.javalang.test")
         self.failUnless(ast.package.annotations)
         self.failIf(ast.package.documentation)
 
     def testJavadocThenAnnotation(self):
         source_file = "source/package-info/JavadocAnnotation.java"
         ast = self.get_ast(source_file)
 
-        self.failUnless(ast.package.name == "org.javalang.test")
+        self.failUnless(ast.package.as_java() == "org.javalang.test")
         self.failUnless(ast.package.annotations)
         self.failUnless(ast.package.documentation)
 

javalang/tree.py

@@ -13,7 +13,7 @@ class Documented(Node):
     attrs = ("documentation",)
 
 class Declaration(Node):
-    attrs = ("modifiers", "annotations")
+    attrs = ("modifiers", "annotations", "end_separator")
 
 class TypeDeclaration(Declaration, Documented):
     attrs = ("name", "body")
@@ -33,6 +33,9 @@ def constructors(self):
 class PackageDeclaration(Declaration, Documented):
     attrs = ("name",)
 
+    def as_java(self):
+        return self.name.as_java()
+
 class ClassDeclaration(TypeDeclaration):
     attrs = ("type_parameters", "extends", "implements")
 
@@ -227,7 +230,7 @@ class MemberReference(Primary):
     attrs = ("member",)
 
 class Invocation(Primary):
-    attrs = ("type_arguments", "arguments")
+    attrs = ("type_arguments", "arguments", "end_separator")
 
 class ExplicitConstructorInvocation(Invocation):
     attrs = ()
@@ -278,3 +281,12 @@ class EnumConstantDeclaration(Declaration, Documented):
 class AnnotationMethod(Declaration):
     attrs = ("name", "return_type", "dimensions", "default")
 
+# ------------------------------------------------------------------------------
+class QualifiedIdentifier(Node):
+    attrs = ("values",)
+
+    def as_java(self):
+        return '.'.join(map(lambda q: q.value, self.values))
+
+class Identifier(Node):
+    attrs = ("value",)