Implement WhiteSpace as well and merge with pytree.

jedi/parser/pytree.py

@@ -16,6 +16,7 @@ import sys
 import os
 
 from . import pgen2
+from . import tokenize
 
 _type_reprs = {}
 
@@ -44,12 +45,12 @@ python_grammar_no_print_statement = python_grammar.copy()
 del python_grammar_no_print_statement.keywords["print"]
 
 
-from jedi.parser import representation as er
+from jedi.parser import representation as pr
 _ast_mapping = {
-    #'simple_stmt': ExprStmt,
-    'classdef': er.Class,
-    'funcdef': er.Function,
-    'file_input': er.SubModule,
+    #'simple_stmt': pr.ExprStmt,
+    'classdef': pr.Class,
+    'funcdef': pr.Function,
+    'file_input': pr.SubModule,
 }
 
 ast_mapping = dict((getattr(python_symbols, k), v) for k, v in _ast_mapping.items())
@@ -229,4 +230,11 @@ def convert(grammar, raw_node):
     else:
         print('leaf', raw_node, type_repr(type))
         prefix, start_pos = context
-        return Leaf(type, value, start_pos, prefix)
+        if type == tokenize.NAME:
+            return pr.Name(value, start_pos, prefix)
+        elif type in (tokenize.STRING, tokenize.NUMBER):
+            return pr.Name(value, start_pos, prefix)
+        elif type in (tokenize.NEWLINE, tokenize.ENDMARKER):
+            return pr.Whitespace(value, start_pos, prefix)
+        else:
+            return pr.Operator(value, start_pos, prefix)

jedi/parser/representation.py

@@ -176,11 +176,19 @@ class _Leaf(Base):
         self.start_pos = start_pos
         self.prefix = prefix
 
+    @property
+    def end_pos(self):
+        return self.start_pos[0], self.start_pos[1] + len(self.value)
+
     def get_code(self):
         return self.prefix + self.value
 
     def __repr__(self):
-        return "<%s: `%s`>" % (type(self).__name__, self.value)
+        return "<%s: %s>" % (type(self).__name__, repr(self.value))
+
+
+class Whitespace(_Leaf):
+    """Contains NEWLINE and ENDMARKER tokens."""
 
 
 class Name(_Leaf):
@@ -199,7 +207,7 @@ class Name(_Leaf):
         return self.value
 
     def __repr__(self):
-        return "<%s: %s@%s,%s>" % (type(self).__name__, self._string,
+        return "<%s: %s@%s,%s>" % (type(self).__name__, self.value,
                                    self.start_pos[0], self.start_pos[1])
 
     def get_definition(self):

test/test_new_parser.py

@@ -23,5 +23,5 @@ def test_basic():
     print()
     x = d.parse_string('\na #pass\n')
     print(repr(x))
-    print(x)
+    print(x.get_code())
     assert False