VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

Refactored the parser calls. Now it's possible to use jedi.parser.python.parse to quickly parse something.

Browse Source
This commit is contained in:
Dave Halter
2017-03-14 00:38:58 +01:00
parent 9b5e6d16da
commit 97fc3bc23c
23 changed files with 126 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
test/test_parser/test_diff_parser.py
View File

@@ -4,10 +4,9 @@ import pytest
import jedi
from jedi import debug
from jedi._compatibility import u
from jedi.common import splitlines
from jedi import cache
from jedi.parser import load_grammar
from jedi.parser.python import load_grammar
from jedi.parser.diff import DiffParser
from jedi.parser import ParserWithRecovery
@@ -45,7 +44,7 @@ class Differ(object):
def initialize(self, source):
debug.dbg('differ: initialize', color='YELLOW')
grammar = load_grammar()
self.parser = ParserWithRecovery(grammar, u(source))
self.parser = ParserWithRecovery(grammar, source)
return self.parser.module
def parse(self, source, copies=0, parsers=0, expect_error_leaves=False):

25
test/test_parser/test_get_code.py
View File

@@ -2,10 +2,9 @@ import difflib
import pytest
from jedi._compatibility import u
from jedi.parser import ParserWithRecovery, load_grammar
from jedi.parser.python import parse
code_basic_features = u('''
code_basic_features = '''
"""A mod docstring"""
def a_function(a_argument, a_default = "default"):
@@ -22,7 +21,7 @@ to""" + "huhu"
return str(a_result)
else
return None
''')
'''
def diff_code_assert(a, b, n=4):
@@ -44,22 +43,22 @@ def diff_code_assert(a, b, n=4):
def test_basic_parsing():
"""Validate the parsing features"""
prs = ParserWithRecovery(load_grammar(), code_basic_features)
m = parse(code_basic_features)
diff_code_assert(
code_basic_features,
prs.module.get_code()
m.get_code()
)
def test_operators():
src = u('5 * 3')
prs = ParserWithRecovery(load_grammar(), src)
diff_code_assert(src, prs.module.get_code())
src = '5 * 3'
module = parse(src)
diff_code_assert(src, module.get_code())
def test_get_code():
"""Use the same code that the parser also generates, to compare"""
s = u('''"""a docstring"""
s = '''"""a docstring"""
class SomeClass(object, mixin):
def __init__(self):
self.xy = 3.0
@@ -81,8 +80,8 @@ class WithDocstring:
def method_with_docstring():
"""class docstr"""
pass
''')
assert ParserWithRecovery(load_grammar(), s).module.get_code() == s
'''
assert parse(s).get_code() == s
def test_end_newlines():
@@ -92,7 +91,7 @@ def test_end_newlines():
line the parser needs.
"""
def test(source, end_pos):
module = ParserWithRecovery(load_grammar(), u(source)).module
module = parse(source)
assert module.get_code() == source
assert module.end_pos == end_pos

2
test/test_parser/test_old_fast_parser.py
View File

@@ -10,7 +10,7 @@ from textwrap import dedent
import jedi
from jedi._compatibility import u
from jedi.parser import load_grammar
from jedi.parser.python import load_grammar
from jedi.parser.diff import FastParser
from jedi.parser.utils import save_parser

8
test/test_parser/test_param_splitting.py
View File

@@ -5,7 +5,7 @@ instead of simple parser objects.
from textwrap import dedent
from jedi.parser import Parser, load_grammar
from jedi.parser.python import parse
def assert_params(param_string, **wanted_dct):
@@ -14,12 +14,12 @@ def assert_params(param_string, **wanted_dct):
pass
''') % param_string
parser = Parser(load_grammar(), dedent(source))
funcdef = parser.get_parsed_node().subscopes[0]
module = parse(source)
funcdef = module.subscopes[0]
dct = dict((p.name.value, p.default and p.default.get_code())
for p in funcdef.params)
assert dct == wanted_dct
assert parser.get_parsed_node().get_code() == source
assert module.get_code() == source
def test_split_params_with_separation_star():

62
test/test_parser/test_parser.py
View File

@@ -4,18 +4,19 @@ from textwrap import dedent
import jedi
from jedi._compatibility import u, is_py3
from jedi.parser import ParserWithRecovery, load_grammar
from jedi.parser import ParserWithRecovery
from jedi.parser.python import parse, load_grammar
from jedi.parser import tree as pt
def test_user_statement_on_import():
"""github #285"""
s = u("from datetime import (\n"
" time)")
s = "from datetime import (\n" \
" time)"
for pos in [(2, 1), (2, 4)]:
p = ParserWithRecovery(load_grammar(), s)
stmt = p.module.get_statement_for_position(pos)
p = parse(s)
stmt = p.get_statement_for_position(pos)
assert isinstance(stmt, pt.Import)
assert [str(n) for n in stmt.get_defined_names()] == ['time']
@@ -23,7 +24,7 @@ def test_user_statement_on_import():
class TestCallAndName():
def get_call(self, source):
# Get the simple_stmt and then the first one.
simple_stmt = ParserWithRecovery(load_grammar(), u(source)).module.children[0]
simple_stmt = parse(source).children[0]
return simple_stmt.children[0]
def test_name_and_call_positions(self):
@@ -58,7 +59,7 @@ class TestCallAndName():
class TestSubscopes():
def get_sub(self, source):
return ParserWithRecovery(load_grammar(), u(source)).module.subscopes[0]
return parse(source).subscopes[0]
def test_subscope_names(self):
name = self.get_sub('class Foo: pass').name
@@ -74,7 +75,7 @@ class TestSubscopes():
class TestImports():
def get_import(self, source):
return ParserWithRecovery(load_grammar(), source).module.imports[0]
return parse(source).imports[0]
def test_import_names(self):
imp = self.get_import(u('import math\n'))
@@ -103,25 +104,25 @@ def test_module():
def test_end_pos():
s = u(dedent('''
x = ['a', 'b', 'c']
def func():
y = None
'''))
parser = ParserWithRecovery(load_grammar(), s)
scope = parser.module.subscopes[0]
s = dedent('''
x = ['a', 'b', 'c']
def func():
y = None
''')
parser = parse(s)
scope = parser.subscopes[0]
assert scope.start_pos == (3, 0)
assert scope.end_pos == (5, 0)
def test_carriage_return_statements():
source = u(dedent('''
source = dedent('''
foo = 'ns1!'
# this is a namespace package
'''))
''')
source = source.replace('\n', '\r\n')
stmt = ParserWithRecovery(load_grammar(), source).module.statements[0]
stmt = parse(source).statements[0]
assert '#' not in stmt.get_code()
@@ -129,7 +130,7 @@ def test_incomplete_list_comprehension():
""" Shouldn't raise an error, same bug as #418. """
# With the old parser this actually returned a statement. With the new
# parser only valid statements generate one.
assert ParserWithRecovery(load_grammar(), u('(1 for def')).module.statements == []
assert parse('(1 for def').statements == []
def test_hex_values_in_docstring():
@@ -141,7 +142,7 @@ def test_hex_values_in_docstring():
return 1
'''
doc = ParserWithRecovery(load_grammar(), dedent(u(source))).module.subscopes[0].raw_doc
doc = parse(source).subscopes[0].raw_doc
if is_py3:
assert doc == '\xff'
else:
@@ -160,7 +161,7 @@ def test_error_correction_with():
def test_newline_positions():
endmarker = ParserWithRecovery(load_grammar(), u('a\n')).module.children[-1]
endmarker = parse('a\n').children[-1]
assert endmarker.end_pos == (2, 0)
new_line = endmarker.get_previous_leaf()
assert new_line.start_pos == (1, 1)
@@ -173,8 +174,8 @@ def test_end_pos_error_correction():
grammar needs it. However, they are removed again. We still want the right
end_pos, even if something breaks in the parser (error correction).
"""
s = u('def x():\n .')
m = ParserWithRecovery(load_grammar(), s).module
s = 'def x():\n .'
m = parse(s)
func = m.children[0]
assert func.type == 'funcdef'
assert func.end_pos == (2, 2)
@@ -208,19 +209,17 @@ def test_unicode_string():
def test_backslash_dos_style():
grammar = load_grammar()
m = ParserWithRecovery(grammar, u('\\\r\n')).module
assert m
assert parse('\\\r\n')
def test_started_lambda_stmt():
p = ParserWithRecovery(load_grammar(), u'lambda a, b: a i')
assert p.get_parsed_node().children[0].type == 'error_node'
m = parse(u'lambda a, b: a i')
assert m.children[0].type == 'error_node'
def test_python2_octal():
parser = ParserWithRecovery(load_grammar(), u'0660')
first = parser.get_parsed_node().children[0]
module = parse('0660')
first = module.children[0]
if is_py3:
assert first.type == 'error_node'
else:
@@ -228,8 +227,7 @@ def test_python2_octal():
def test_python3_octal():
parser = ParserWithRecovery(load_grammar(), u'0o660')
module = parser.get_parsed_node()
module = parse('0o660')
if is_py3:
assert module.children[0].children[0].type == 'number'
else:

6
test/test_parser/test_parser_tree.py
View File

@@ -5,7 +5,7 @@ from textwrap import dedent
import pytest
from jedi._compatibility import u, unicode
from jedi.parser import ParserWithRecovery, load_grammar
from jedi.parser.python import parse
from jedi.parser import tree as pt
@@ -27,9 +27,9 @@ class TestsFunctionAndLambdaParsing(object):
@pytest.fixture(params=FIXTURES)
def node(self, request):
parsed = ParserWithRecovery(load_grammar(), dedent(u(request.param[0])))
parsed = parse(dedent(request.param[0]))
request.keywords['expected'] = request.param[1]
return parsed.module.subscopes[0]
return parsed.subscopes[0]
@pytest.fixture()
def expected(self, request, node):

8
test/test_parser/test_pgen2.py
View File

@@ -8,9 +8,9 @@ test_grammar.py files from both Python 2 and Python 3.
from textwrap import dedent
from jedi._compatibility import unicode, is_py3
from jedi.parser import Parser, load_grammar, ParseError
from jedi._compatibility import is_py3
from jedi.parser.python import parse as _parse, load_grammar
from jedi.parser import ParseError
import pytest
from test.helpers import TestCase
@@ -19,7 +19,7 @@ from test.helpers import TestCase
def parse(code, version='3.4'):
code = dedent(code) + "\n\n"
grammar = load_grammar(version=version)
return Parser(grammar, unicode(code), 'file_input').get_parsed_node()
return _parse(code, grammar, error_recovery=False)
class TestDriver(TestCase):

39
test/test_parser/test_tokenize.py
View File

@@ -2,9 +2,10 @@
from textwrap import dedent
from jedi._compatibility import u, is_py3, py_version
from jedi._compatibility import is_py3, py_version
from jedi.parser.token import NAME, OP, NEWLINE, STRING, INDENT, ERRORTOKEN, ENDMARKER
from jedi.parser import ParserWithRecovery, load_grammar, tokenize
from jedi.parser import tokenize
from jedi.parser.python import parse
from jedi.common import splitlines
from jedi.parser.tokenize import TokenInfo
@@ -17,25 +18,25 @@ def _get_token_list(string):
class TokenTest(unittest.TestCase):
def test_end_pos_one_line(self):
parsed = ParserWithRecovery(load_grammar(), dedent(u('''
parsed = parse(dedent('''
def testit():
a = "huhu"
''')))
tok = parsed.module.subscopes[0].statements[0].children[2]
'''))
tok = parsed.subscopes[0].statements[0].children[2]
assert tok.end_pos == (3, 14)
def test_end_pos_multi_line(self):
parsed = ParserWithRecovery(load_grammar(), dedent(u('''
parsed = parse(dedent('''
def testit():
a = """huhu
asdfasdf""" + "h"
''')))
tok = parsed.module.subscopes[0].statements[0].children[2].children[0]
'''))
tok = parsed.subscopes[0].statements[0].children[2].children[0]
assert tok.end_pos == (4, 11)
def test_simple_no_whitespace(self):
# Test a simple one line string, no preceding whitespace
simple_docstring = u('"""simple one line docstring"""')
simple_docstring = '"""simple one line docstring"""'
tokens = tokenize.source_tokens(simple_docstring)
token_list = list(tokens)
_, value, _, prefix = token_list[0]
@@ -44,7 +45,7 @@ class TokenTest(unittest.TestCase):
def test_simple_with_whitespace(self):
# Test a simple one line string with preceding whitespace and newline
simple_docstring = u(' """simple one line docstring""" \r\n')
simple_docstring = ' """simple one line docstring""" \r\n'
tokens = tokenize.source_tokens(simple_docstring)
token_list = list(tokens)
assert token_list[0][0] == INDENT
@@ -58,12 +59,12 @@ class TokenTest(unittest.TestCase):
def test_function_whitespace(self):
# Test function definition whitespace identification
fundef = dedent(u('''
fundef = dedent('''
def test_whitespace(*args, **kwargs):
x = 1
if x > 0:
print(True)
'''))
''')
tokens = tokenize.source_tokens(fundef)
token_list = list(tokens)
for _, value, _, prefix in token_list:
@@ -83,7 +84,7 @@ class TokenTest(unittest.TestCase):
def test_tokenize_multiline_I(self):
# Make sure multiline string having newlines have the end marker on the
# next line
fundef = u('''""""\n''')
fundef = '''""""\n'''
tokens = tokenize.source_tokens(fundef)
token_list = list(tokens)
assert token_list == [TokenInfo(ERRORTOKEN, '""""\n', (1, 0), ''),
@@ -92,7 +93,7 @@ class TokenTest(unittest.TestCase):
def test_tokenize_multiline_II(self):
# Make sure multiline string having no newlines have the end marker on
# same line
fundef = u('''""""''')
fundef = '''""""'''
tokens = tokenize.source_tokens(fundef)
token_list = list(tokens)
assert token_list == [TokenInfo(ERRORTOKEN, '""""', (1, 0), ''),
@@ -101,17 +102,17 @@ class TokenTest(unittest.TestCase):
def test_tokenize_multiline_III(self):
# Make sure multiline string having newlines have the end marker on the
# next line even if several newline
fundef = u('''""""\n\n''')
fundef = '''""""\n\n'''
tokens = tokenize.source_tokens(fundef)
token_list = list(tokens)
assert token_list == [TokenInfo(ERRORTOKEN, '""""\n\n', (1, 0), ''),
TokenInfo(ENDMARKER, '', (3, 0), '')]
def test_identifier_contains_unicode(self):
fundef = dedent(u('''
fundef = dedent('''
def 我あφ():
pass
'''))
''')
tokens = tokenize.source_tokens(fundef)
token_list = list(tokens)
unicode_token = token_list[1]
@@ -133,8 +134,8 @@ class TokenTest(unittest.TestCase):
]
for s in string_tokens:
parsed = ParserWithRecovery(load_grammar(), u('''a = %s\n''' % s))
simple_stmt = parsed.module.children[0]
module = parse('''a = %s\n''' % s)
simple_stmt = module.children[0]
expr_stmt = simple_stmt.children[0]
assert len(expr_stmt.children) == 3
string_tok = expr_stmt.children[2]