fix python 2.7 issues. the parser now only takes unicode inputs

jedi/evaluate/finder.py

@@ -372,12 +372,13 @@ def get_names_of_scope(evaluator, scope, position=None, star_search=True, includ
     This function is used to include names from outer scopes. For example, when
     the current scope is function:
 
+    >>> from jedi._compatibility import u
     >>> from jedi.parser import Parser
-    >>> parser = Parser('''
+    >>> parser = Parser(u('''
     ... x = ['a', 'b', 'c']
     ... def func():
     ...     y = None
-    ... ''')
+    ... '''))
     >>> scope = parser.module.subscopes[0]
     >>> scope
     <Function: func@3-4>

jedi/parser/representation.py

@@ -18,8 +18,9 @@ which is being used in a function definition.
 The easiest way to play with this module is to use :class:`parsing.Parser`.
 :attr:`parsing.Parser.module` holds an instance of :class:`SubModule`:
 
+>>> from jedi._compatibility import u
 >>> from jedi.parser import Parser
->>> parser = Parser('import os', 'example.py')
+>>> parser = Parser(u('import os'), 'example.py')
 >>> submodule = parser.module
 >>> submodule
 <SubModule: example.py@1-1>
@@ -283,12 +284,13 @@ class Scope(Simple, IsScope):
         """
         Get all defined names in this scope.
 
+        >>> from jedi._compatibility import u
         >>> from jedi.parser import Parser
-        >>> parser = Parser('''
+        >>> parser = Parser(u('''
         ... a = x
         ... b = y
         ... b.c = z
-        ... ''')
+        ... '''))
         >>> parser.module.get_defined_names()
         [<Name: a@2,0>, <Name: b@3,0>]
 

test/test_parser/test_get_code.py

@@ -2,9 +2,10 @@ import difflib
 
 import pytest
 
-import jedi.parser as parser
+from jedi._compatibility import u
+from jedi.parser import Parser
 
-code_basic_features = '''
+code_basic_features = u('''
 """A mod docstring"""
 
 def a_function(a_argument, a_default = "default"):
@@ -21,7 +22,7 @@ to""" + "huhu"
         return str(a_result)
     else
         return None
-'''
+''')
 
 
 def diff_code_assert(a, b, n=4):
@@ -43,7 +44,7 @@ def diff_code_assert(a, b, n=4):
 def test_basic_parsing():
     """Validate the parsing features"""
 
-    prs = parser.Parser(code_basic_features)
+    prs = Parser(code_basic_features)
     diff_code_assert(
         code_basic_features,
         prs.module.get_code2()
@@ -52,6 +53,34 @@ def test_basic_parsing():
 
 @pytest.mark.skipif('True', reason='Not yet working.')
 def test_operators():
-    src = '5  * 3'
+    src = u('5  * 3')
-    prs = parser.Parser(src)
+    prs = Parser(src)
     diff_code_assert(src, prs.module.get_code())
+
+
+def test_get_code():
+    """Use the same code that the parser also generates, to compare"""
+    s = u('''"""a docstring"""
+class SomeClass(object, mixin):
+    def __init__(self):
+        self.xy = 3.0
+        """statement docstr"""
+    def some_method(self):
+        return 1
+    def yield_method(self):
+        while hasattr(self, 'xy'):
+            yield True
+        for x in [1, 2]:
+            yield x
+    def empty(self):
+        pass
+class Empty:
+    pass
+class WithDocstring:
+    """class docstr"""
+    pass
+def method_with_docstring():
+    """class docstr"""
+    pass
+''')
+    assert Parser(s).module.get_code() == s

test/test_parser/test_parsing.py

@@ -1,3 +1,4 @@
+from jedi._compatibility import u
 from jedi.parser import Parser
 from jedi.parser.user_context import UserContextParser
 from jedi.parser import representation as pr
@@ -5,19 +6,19 @@ from jedi.parser import representation as pr
 
 def test_user_statement_on_import():
     """github #285"""
-    s = "from datetime import (\n" \
+    s = u("from datetime import (\n"
-        "    time)"
+          "    time)")
 
     for pos in [(2, 1), (2, 4)]:
-        u = UserContextParser(s, None, pos, None).user_stmt()
+        p = UserContextParser(s, None, pos, None).user_stmt()
-        assert isinstance(u, pr.Import)
+        assert isinstance(p, pr.Import)
-        assert u.defunct is False
+        assert p.defunct is False
-        assert [str(n) for n in u.get_defined_names()] == ['time']
+        assert [str(n) for n in p.get_defined_names()] == ['time']
 
 
 class TestCallAndName():
     def get_call(self, source):
-        stmt = Parser(source, no_docstr=True).module.statements[0]
+        stmt = Parser(u(source), no_docstr=True).module.statements[0]
         return stmt.expression_list()[0]
 
     def test_name_and_call_positions(self):
@@ -52,7 +53,7 @@ class TestCallAndName():
 
 class TestSubscopes():
     def get_sub(self, source):
-        return Parser(source).module.subscopes[0]
+        return Parser(u(source)).module.subscopes[0]
 
     def test_subscope_names(self):
         name = self.get_sub('class Foo: pass').name
@@ -71,7 +72,7 @@ class TestImports():
         return Parser(source).module.imports[0]
 
     def test_import_names(self):
-        imp = self.get_import('import math\n')
+        imp = self.get_import(u('import math\n'))
         names = imp.get_defined_names()
         assert len(names) == 1
         assert str(names[0]) == 'math'
@@ -83,13 +84,13 @@ class TestImports():
 
 
 def test_module():
-    module = Parser('asdf', 'example.py', no_docstr=True).module
+    module = Parser(u('asdf'), 'example.py', no_docstr=True).module
     name = module.name
     assert str(name) == 'example'
     assert name.start_pos == (0, 0)
     assert name.end_pos == (0, 0)
 
-    module = Parser('asdf', no_docstr=True).module
+    module = Parser(u('asdf'), no_docstr=True).module
     name = module.name
     assert str(name) == ''
     assert name.start_pos == (0, 0)

test/test_parser/test_parsing_representation.py

@@ -1,29 +0,0 @@
-from jedi.parser import Parser
-
-def test_get_code():
-    """Use the same code that the parser also generates, to compare"""
-    s = \
-'''"""a docstring"""
-class SomeClass(object, mixin):
-    def __init__(self):
-        self.xy = 3.0
-        """statement docstr"""
-    def some_method(self):
-        return 1
-    def yield_method(self):
-        while hasattr(self, 'xy'):
-            yield True
-        for x in [1, 2]:
-            yield x
-    def empty(self):
-        pass
-class Empty:
-    pass
-class WithDocstring:
-    """class docstr"""
-    pass
-def method_with_docstring():
-    """class docstr"""
-    pass
-'''
-    assert Parser(s).module.get_code() == s

test/test_parser/test_token.py

@@ -1,4 +1,5 @@
-import jedi.parser as parser
+from jedi import parser
+from jedi._compatibility import u
 
 try:
     import unittest2 as unittest
@@ -8,18 +9,18 @@ except ImportError:  # pragma: no cover
 
 class TokenTest(unittest.TestCase):
     def test_end_pos_one_line(self):
-        parsed = parser.Parser('''
+        parsed = parser.Parser(u('''
 def testit():
     a = "huhu"
-''')
+'''))
         tok = parsed.module.subscopes[0].statements[0].token_list[2]
         self.assertEqual(tok.end_pos, (3, 14))
 
     def test_end_pos_multi_line(self):
-        parsed = parser.Parser('''
+        parsed = parser.Parser(u('''
 def testit():
     a = """huhu
 asdfasdf""" + "h"
-''')
+'''))
         tok = parsed.module.subscopes[0].statements[0].token_list[2]
         self.assertEqual(tok.end_pos, (4, 11))

test/test_regression.py

@@ -10,6 +10,7 @@ from .helpers import TestCase, cwd_at
 
 import pytest
 import jedi
+from jedi._compatibility import u
 from jedi import Script
 from jedi import api
 from jedi.evaluate import imports
@@ -113,7 +114,7 @@ class TestRegression(TestCase):
 
     def test_end_pos(self):
         # jedi issue #150
-        s = "x()\nx( )\nx(  )\nx (  )"
+        s = u("x()\nx( )\nx(  )\nx (  )")
         parser = Parser(s)
         for i, s in enumerate(parser.module.statements, 3):
             for c in s.expression_list():