From cd8932fbfc4bfbc44c426bddd2637a3669231336 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 24 May 2017 00:37:36 -0400 Subject: [PATCH] Add a latest grammar to the evaluator and use it to avoid importing from parso import parse. --- jedi/evaluate/__init__.py | 3 +++ jedi/evaluate/compiled/fake.py | 18 +++++++++--------- jedi/evaluate/compiled/mixed.py | 1 - jedi/evaluate/docstrings.py | 4 ++-- jedi/evaluate/imports.py | 3 +-- jedi/evaluate/pep0484.py | 15 ++++++++------- test/run.py | 8 ++++++-- test/test_evaluate/test_buildout_detection.py | 2 +- test/test_parso_integration/test_basic.py | 2 +- test/test_regression.py | 1 - 10 files changed, 31 insertions(+), 26 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 1c41bf54..d8844166 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -64,6 +64,8 @@ import copy import sys from parso.python import tree +from parso import load_python_grammar + from jedi import debug from jedi.common import unite from jedi.evaluate import representation as er @@ -87,6 +89,7 @@ from jedi import parser_utils class Evaluator(object): def __init__(self, grammar, sys_path=None): self.grammar = grammar + self.latest_grammar = load_python_grammar('3.6') self.memoize_cache = {} # for memoize decorators # To memorize modules -> equals `sys.modules`. self.modules = {} # like `sys.modules`. diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 8145f055..60dbefe4 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -9,10 +9,10 @@ import inspect import types from itertools import chain -from jedi._compatibility import is_py3, builtins, unicode, is_py34 -from parso.python import parse from parso.python import tree +from jedi._compatibility import is_py3, builtins, unicode, is_py34 + modules = {} @@ -47,7 +47,7 @@ class FakeDoesNotExist(Exception): pass -def _load_faked_module(module): +def _load_faked_module(grammar, module): module_name = module.__name__ if module_name == '__builtin__' and not is_py3: module_name = 'builtins' @@ -62,7 +62,7 @@ def _load_faked_module(module): except IOError: modules[module_name] = None return - modules[module_name] = m = parse(unicode(source)) + modules[module_name] = m = grammar.parse(unicode(source)) if module_name == 'builtins' and not is_py3: # There are two implementations of `open` for either python 2/3. @@ -105,12 +105,12 @@ def get_module(obj): return builtins -def _faked(module, obj, name): +def _faked(grammar, module, obj, name): # Crazy underscore actions to try to escape all the internal madness. if module is None: module = get_module(obj) - faked_mod = _load_faked_module(module) + faked_mod = _load_faked_module(grammar, module) if faked_mod is None: return None, None @@ -168,8 +168,8 @@ def memoize_faked(obj): @memoize_faked -def _get_faked(module, obj, name=None): - result, fake_module = _faked(module, obj, name) +def _get_faked(grammar, module, obj, name=None): + result, fake_module = _faked(grammar, module, obj, name) if result is None: # We're not interested in classes. What we want is functions. raise FakeDoesNotExist @@ -197,7 +197,7 @@ def get_faked(evaluator, module, obj, name=None, parent_context=None): else: raise FakeDoesNotExist - faked, fake_module = _get_faked(module and module.obj, obj, name) + faked, fake_module = _get_faked(evaluator.latest_grammar, module and module.obj, obj, name) if module is not None: module.get_used_names = fake_module.get_used_names return faked diff --git a/jedi/evaluate/compiled/mixed.py b/jedi/evaluate/compiled/mixed.py index fb0ce5aa..4a2c9739 100644 --- a/jedi/evaluate/compiled/mixed.py +++ b/jedi/evaluate/compiled/mixed.py @@ -5,7 +5,6 @@ Used only for REPL Completion. import inspect import os -from parso.python import parse from jedi import settings from jedi.evaluate import compiled from jedi.cache import underscore_memoization diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index b3ccef73..0a7854e4 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -22,7 +22,6 @@ from jedi._compatibility import u from jedi.common import unite from jedi.evaluate import context from jedi.evaluate.cache import memoize_default -from parso.python import parse from jedi.common import indent_block from jedi.evaluate.iterable import SequenceLiteralContext, FakeSequence @@ -132,7 +131,8 @@ def _evaluate_for_statement_string(module_context, string): # Take the default grammar here, if we load the Python 2.7 grammar here, it # will be impossible to use `...` (Ellipsis) as a token. Docstring types # don't need to conform with the current grammar. - module = parse(code.format(indent_block(string))) + grammar = module_context.evaluator.latest_grammar + module = grammar.parse(code.format(indent_block(string))) try: funcdef = next(module.iter_funcdefs()) # First pick suite, then simple_stmt and then the node, diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 13c4ca05..0e014a53 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -16,7 +16,6 @@ import os import pkgutil import sys -from parso.python import parse from parso.python import tree from parso.tree import search_ancestor from parso.cache import parser_cache @@ -471,7 +470,7 @@ def _load_module(evaluator, path=None, code=None, sys_path=None, parent_module=N if path is not None and path.endswith(('.py', '.zip', '.egg')) \ and dotted_path not in settings.auto_import_modules: - module_node = parse( + module_node = evaluator.grammar.parse( code=code, path=path, cache=True, diff_cache=True, cache_path=settings.cache_directory) diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 0d126986..12964231 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -20,10 +20,12 @@ x support for type hint comments for functions, `# type: (int, str) -> int`. """ import itertools - import os +import re + from parso import ParserSyntaxError -from parso.python import parse, tree +from parso.python import tree + from jedi.common import unite from jedi.evaluate.cache import memoize_default from jedi.evaluate import compiled @@ -31,7 +33,6 @@ from jedi.evaluate.context import LazyTreeContext from jedi import debug from jedi import _compatibility from jedi import parser_utils -import re def _evaluate_for_annotation(context, annotation, index=None): @@ -63,7 +64,7 @@ def _fix_forward_reference(context, node): if isinstance(evaled_node, compiled.CompiledObject) and \ isinstance(evaled_node.obj, str): try: - new_node = parse( + new_node = context.evaluator.grammar.parse( _compatibility.unicode(evaled_node.obj), start_symbol='eval_input', error_recovery=False @@ -110,7 +111,7 @@ def infer_return_types(function_context): _typing_module = None -def _get_typing_replacement_module(): +def _get_typing_replacement_module(grammar): """ The idea is to return our jedi replacement for the PEP-0484 typing module as discussed at https://github.com/davidhalter/jedi/issues/663 @@ -121,7 +122,7 @@ def _get_typing_replacement_module(): os.path.abspath(os.path.join(__file__, "../jedi_typing.py")) with open(typing_path) as f: code = _compatibility.unicode(f.read()) - _typing_module = parse(code) + _typing_module = grammar.parse(code) return _typing_module @@ -155,7 +156,7 @@ def py__getitem__(context, typ, node): from jedi.evaluate.representation import ModuleContext typing = ModuleContext( context.evaluator, - module_node=_get_typing_replacement_module(), + module_node=_get_typing_replacement_module(context.evaluator.latest_grammar), path=None ) factories = typing.py__getattribute__("factory") diff --git a/test/run.py b/test/run.py index 4354ea1a..558304bc 100755 --- a/test/run.py +++ b/test/run.py @@ -117,10 +117,11 @@ from ast import literal_eval from io import StringIO from functools import reduce +import parso + import jedi from jedi import debug from jedi._compatibility import unicode, is_py3 -from parso.python import parse from jedi.api.classes import Definition from jedi.api.completion import get_user_scope from jedi import parser_utils @@ -132,6 +133,9 @@ TEST_ASSIGNMENTS = 2 TEST_USAGES = 3 +grammar36 = parso.load_python_grammar('3.6') + + class IntegrationTestCase(object): def __init__(self, test_type, correct, line_nr, column, start, line, path=None, skip=None): @@ -188,7 +192,7 @@ class IntegrationTestCase(object): should_be = set() for match in re.finditer('(?:[^ ]+)', correct): string = match.group(0) - parser = parse(string, start_symbol='eval_input', error_recovery=False) + parser = grammar36.parse(string, start_symbol='eval_input', error_recovery=False) parser_utils.move(parser.get_root_node(), self.line_nr) element = parser.get_root_node() module_context = script._get_module() diff --git a/test/test_evaluate/test_buildout_detection.py b/test/test_evaluate/test_buildout_detection.py index 4fceda44..6e45939a 100644 --- a/test/test_evaluate/test_buildout_detection.py +++ b/test/test_evaluate/test_buildout_detection.py @@ -8,7 +8,7 @@ from jedi.evaluate.sys_path import (_get_parent_dir_with_file, _check_module) from jedi.evaluate import Evaluator from jedi.evaluate.representation import ModuleContext -from parso.python import parse +from parso import parse from parso import load_python_grammar from ..helpers import cwd_at diff --git a/test/test_parso_integration/test_basic.py b/test/test_parso_integration/test_basic.py index e8fa48bc..13051a42 100644 --- a/test/test_parso_integration/test_basic.py +++ b/test/test_parso_integration/test_basic.py @@ -1,6 +1,6 @@ from textwrap import dedent -from parso.python import parse +from parso import parse import jedi diff --git a/test/test_regression.py b/test/test_regression.py index 32671b73..a8f73351 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -11,7 +11,6 @@ import pytest from jedi import Script from jedi import api -from jedi import common from jedi.evaluate import imports from parso.python import parse from .helpers import TestCase, cwd_at