diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index a76ec36d..2827d46e 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -16,7 +16,7 @@ import sys from jedi._compatibility import unicode from jedi.parser import load_grammar from jedi.parser import tree -from jedi.parser.user_context import UserContextParser +from jedi.parser.fast import FastParser from jedi import debug from jedi import settings from jedi import common @@ -116,13 +116,11 @@ class Script(object): if not (0 <= column <= line_len): raise ValueError('`column` parameter is not in a valid range.') self._pos = line, column + self._path = path cache.clear_time_caches() debug.reset_time() self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2]) - self._parser = UserContextParser(self._grammar, self._source, path, - self._pos, - self._parsed_callback) if sys_path is None: venv = os.getenv('VIRTUAL_ENV') if venv: @@ -130,12 +128,14 @@ class Script(object): self._evaluator = Evaluator(self._grammar, sys_path=sys_path) debug.speed('init') - def _parsed_callback(self, parser): + def _get_module(self): + cache.invalidate_star_import_cache(self._path) + parser = FastParser(self._grammar, self._source, self._path) + cache.save_parser(self._path, parser, pickling=False) + module = self._evaluator.wrap(parser.module) imports.add_module(self._evaluator, unicode(module.name), module) - - def _get_module(self): - return self._parser.module() + return parser.module @property def source_path(self): @@ -160,7 +160,7 @@ class Script(object): """ debug.speed('completions start') completion = Completion( - self._evaluator, self._parser, self._code_lines, + self._evaluator, self._get_module(), self._code_lines, self._pos, self.call_signatures ) completions = completion.completions() diff --git a/jedi/api/completion.py b/jedi/api/completion.py index a1a3ef20..e4fdc32f 100644 --- a/jedi/api/completion.py +++ b/jedi/api/completion.py @@ -1,4 +1,3 @@ -import re from itertools import chain from jedi.parser import token @@ -74,10 +73,9 @@ def get_user_scope(module, position): class Completion: - def __init__(self, evaluator, parser, code_lines, position, call_signatures_method): + def __init__(self, evaluator, module, code_lines, position, call_signatures_method): self._evaluator = evaluator - self._parser = parser - self._module = evaluator.wrap(parser.module()) + self._module = evaluator.wrap(module) self._code_lines = code_lines # The first step of completions is to get the name diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index fe0a6960..25755604 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -13,7 +13,9 @@ from __future__ import absolute_import import string import re +from collections import namedtuple from io import StringIO + from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, opmap, NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT) from jedi._compatibility import is_py3 @@ -148,6 +150,9 @@ ALWAYS_BREAK_TOKENS = (';', 'import', 'class', 'def', 'try', 'except', 'finally', 'while', 'return') +Token = namedtuple('Token', ['type', 'string', 'start_pos', 'prefix']) + + def source_tokens(source, use_exact_op_types=False): """Generate tokens from a the source code (string).""" source = source diff --git a/jedi/parser/user_context.py b/jedi/parser/user_context.py index 5501fb45..b286bfa5 100644 --- a/jedi/parser/user_context.py +++ b/jedi/parser/user_context.py @@ -27,15 +27,7 @@ class UserContextParser(object): @cache.underscore_memoization def _parser(self): - cache.invalidate_star_import_cache(self._path) - if self._use_fast_parser: - parser = FastParser(self._grammar, self._source, self._path) - # Don't pickle that module, because the main module is changing quickly - cache.save_parser(self._path, parser, pickling=False) - else: - parser = ParserWithRecovery(self._grammar, self._source, self._path) - self._parser_done_callback(parser) - return parser + pass def module(self): return self._parser().module diff --git a/test/test_parser/test_fast_parser.py b/test/test_parser/test_fast_parser.py index 607fae07..ac7c5a07 100644 --- a/test/test_parser/test_fast_parser.py +++ b/test/test_parser/test_fast_parser.py @@ -38,7 +38,7 @@ def test_class_in_docstr(): Regression test for a problem with classes in docstrings. """ a = '"\nclasses\n"' - jedi.Script(a, 1, 0)._parser + jedi.Script(a, 1, 0)._get_module() b = a + '\nimport os' assert jedi.Script(b, 4, 8).goto_assignments() @@ -450,7 +450,7 @@ def test_string_literals(): """) script = jedi.Script(dedent(source)) - script._parser.module().end_pos == (6, 0) + script._get_module().end_pos == (6, 0) assert script.completions() @@ -468,7 +468,7 @@ def test_decorator_string_issue(): s = jedi.Script(source) assert s.completions() - assert s._parser.module().get_code() == source + assert s._get_module().get_code() == source def test_round_trip():