diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 10f05a4d..0fde756c 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -13,6 +13,7 @@ except ImportError: is_py3 = sys.version_info[0] >= 3 is_py33 = is_py3 and sys.version_info.minor >= 3 +is_py34 = is_py3 and sys.version_info.minor >= 4 is_py35 = is_py3 and sys.version_info.minor >= 5 is_py26 = not is_py3 and sys.version_info[1] < 7 diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index efe5ab48..b5788b1f 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -4,6 +4,7 @@ Helpers for the API import re from collections import namedtuple +from jedi._compatibility import u from jedi.evaluate.helpers import call_of_leaf from jedi import parser from jedi.parser import tokenize, token @@ -52,7 +53,7 @@ def get_stack_at_position(grammar, code_lines, module, pos): user_stmt = module.get_statement_for_position(pos) if user_stmt is not None and user_stmt.type in ('indent', 'dedent'): - code = '' + code = u('') else: if user_stmt is None: user_stmt = module.get_leaf_for_position(pos, include_prefixes=True) @@ -72,7 +73,7 @@ def get_stack_at_position(grammar, code_lines, module, pos): code = _get_code(code_lines, user_stmt.start_pos, pos) if code == ';': # ; cannot be parsed. - code = '' + code = u('') # Remove whitespace at the end. Necessary, because the tokenizer will parse # an error token (there's no new line at the end in our case). This doesn't @@ -99,7 +100,7 @@ def get_stack_at_position(grammar, code_lines, module, pos): try: p.parse(tokenizer=tokenize_without_endmarker(code)) except EndMarkerReached: - return Stack(p.pgen_parser.stack) + return Stack(p.stack) class Stack(list): diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index a5be29a0..d8c8f9ab 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -8,7 +8,7 @@ import os import inspect import types -from jedi._compatibility import is_py3, builtins, unicode +from jedi._compatibility import is_py3, builtins, unicode, is_py34 from jedi.parser import ParserWithRecovery, load_grammar from jedi.parser import tree as pt from jedi.evaluate.helpers import FakeName @@ -36,10 +36,11 @@ NOT_CLASS_TYPES = ( if is_py3: NOT_CLASS_TYPES += ( - types.DynamicClassAttribute, types.MappingProxyType, types.SimpleNamespace ) + if is_py34: + NOT_CLASS_TYPES += (types.DynamicClassAttribute,) def _load_faked_module(module): diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index fd4dc4a9..3d92b70b 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -98,7 +98,6 @@ class Parser(object): def __init__(self, grammar, source, start_symbol='file_input', tokenizer=None, start_parsing=True): # Todo Remove start_parsing (with False) - start_number = grammar.symbol2number[start_symbol] self._used_names = {} self._scope_names_stack = [{}] @@ -114,11 +113,6 @@ class Parser(object): source += '\n' self._added_newline = True - self.pgen_parser = PgenParser( - grammar, self.convert_node, self.convert_leaf, - self.error_recovery, start_number - ) - self._start_symbol = start_symbol self._grammar = grammar @@ -133,7 +127,16 @@ class Parser(object): if self._parsed is not None: return self._parsed - self._parsed = self.pgen_parser.parse(tokenizer) + start_number = self._grammar.symbol2number[self._start_symbol] + pgen_parser = PgenParser( + self._grammar, self.convert_node, self.convert_leaf, + self.error_recovery, start_number + ) + + try: + self._parsed = pgen_parser.parse(tokenizer) + finally: + self.stack = pgen_parser.stack if self._start_symbol == 'file_input' != self._parsed.type: # If there's only one statement, we get back a non-module. That's diff --git a/jedi/parser/grammar2.7.txt b/jedi/parser/grammar2.7.txt index b2950143..515dea64 100644 --- a/jedi/parser/grammar2.7.txt +++ b/jedi/parser/grammar2.7.txt @@ -57,7 +57,7 @@ break_stmt: 'break' continue_stmt: 'continue' return_stmt: 'return' [testlist] yield_stmt: yield_expr -raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]] +raise_stmt: 'raise' [test [',' test [',' test]]] import_stmt: import_name | import_from import_name: 'import' dotted_as_names # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS @@ -68,7 +68,7 @@ dotted_as_name: dotted_name ['as' NAME] import_as_names: import_as_name (',' import_as_name)* [','] dotted_as_names: dotted_as_name (',' dotted_as_name)* dotted_name: NAME ('.' NAME)* -global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* +global_stmt: 'global' NAME (',' NAME)* exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] diff --git a/jedi/parser/utils.py b/jedi/parser/utils.py index a854aa40..65986442 100644 --- a/jedi/parser/utils.py +++ b/jedi/parser/utils.py @@ -6,10 +6,7 @@ import json import hashlib import gc import shutil -try: - import cPickle as pickle -except ImportError: - import pickle +import pickle from jedi import settings from jedi import debug