Start actual testing of the new parser.

jedi/parser/pytree.py

@@ -126,6 +126,9 @@ class Node(Base):
     if sys.version_info > (3, 0):
         __str__ = __unicode__
 
+    def get_code(self):
+        return str(self)
+
     @property
     def prefix(self):
         """
@@ -186,7 +189,7 @@ def convert(grammar, raw_node):
         if type == tokenize.NAME:
             return pr.Name(value, start_pos, prefix)
         elif type in (tokenize.STRING, tokenize.NUMBER):
-            return pr.Name(value, start_pos, prefix)
+            return pr.Literal(value, start_pos, prefix)
         elif type in (tokenize.NEWLINE, tokenize.ENDMARKER):
             return pr.Whitespace(value, start_pos, prefix)
         else:

jedi/parser/representation.py

@@ -219,11 +219,14 @@ class Literal(_Leaf):
         return literal_eval(self.value)
 
     def __repr__(self):
+        # TODO remove?
+        """
         if is_py3:
             s = self.literal
         else:
             s = self.literal.encode('ascii', 'replace')
-        return "<%s: %s>" % (type(self).__name__, s)
+        """
+        return "<%s: %s>" % (type(self).__name__, self.value)
 
 
 class Operator(_Leaf):
@@ -278,7 +281,7 @@ class Simple(Base):
         return self.children[-1].end_pos
 
     def get_code(self):
-        return "".join(str(c) for c in self.children)
+        return "".join(c.get_code() for c in self.children)
 
     def __repr__(self):
         code = self.get_code().replace('\n', ' ')

test/test_new_parser.py

@@ -7,7 +7,7 @@ from jedi.parser.pgen2 import Driver
 def test_basic():
     def compare(string):
         """Generates the AST object and then regenerates the code."""
-        return d.parse_string(string).get_code() == string
+        assert d.parse_string(string).get_code() == string
 
     #if self.options["print_function"]:
     #    python_grammar = pygram.python_grammar_no_print_statement
@@ -18,6 +18,10 @@ def test_basic():
     logger = logging.getLogger("RefactoringTool")
     d = Driver(pytree.python_grammar, convert=pytree.convert, logger=logger)
 
+    compare('\na #pass\n')
+    compare('wblabla* 1\t\n')
+    compare('def x(a, b:3): pass\n')
+
     print(d)
     tree = d.parse_string('wblabla* 1\t\n')
     print(repr(tree))