Move the parsing away from user_context to api.py.

jedi/api/__init__.py

@@ -16,7 +16,7 @@ import sys
 from jedi._compatibility import unicode
 from jedi.parser import load_grammar
 from jedi.parser import tree
-from jedi.parser.user_context import UserContextParser
+from jedi.parser.fast import FastParser
 from jedi import debug
 from jedi import settings
 from jedi import common
@@ -116,13 +116,11 @@ class Script(object):
         if not (0 <= column <= line_len):
             raise ValueError('`column` parameter is not in a valid range.')
         self._pos = line, column
+        self._path = path
 
         cache.clear_time_caches()
         debug.reset_time()
         self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2])
-        self._parser = UserContextParser(self._grammar, self._source, path,
-                                         self._pos,
-                                         self._parsed_callback)
         if sys_path is None:
             venv = os.getenv('VIRTUAL_ENV')
             if venv:
@@ -130,12 +128,14 @@ class Script(object):
         self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
         debug.speed('init')
 
-    def _parsed_callback(self, parser):
+    def _get_module(self):
+        cache.invalidate_star_import_cache(self._path)
+        parser = FastParser(self._grammar, self._source, self._path)
+        cache.save_parser(self._path, parser, pickling=False)
+
         module = self._evaluator.wrap(parser.module)
         imports.add_module(self._evaluator, unicode(module.name), module)
-
+        return parser.module
-    def _get_module(self):
-        return self._parser.module()
 
     @property
     def source_path(self):
@@ -160,7 +160,7 @@ class Script(object):
         """
         debug.speed('completions start')
         completion = Completion(
-            self._evaluator, self._parser, self._code_lines,
+            self._evaluator, self._get_module(), self._code_lines,
             self._pos, self.call_signatures
         )
         completions = completion.completions()

jedi/api/completion.py

@@ -1,4 +1,3 @@
-import re
 from itertools import chain
 
 from jedi.parser import token
@@ -74,10 +73,9 @@ def get_user_scope(module, position):
 
 
 class Completion:
-    def __init__(self, evaluator, parser, code_lines, position, call_signatures_method):
+    def __init__(self, evaluator, module, code_lines, position, call_signatures_method):
         self._evaluator = evaluator
-        self._parser = parser
+        self._module = evaluator.wrap(module)
-        self._module = evaluator.wrap(parser.module())
         self._code_lines = code_lines
 
         # The first step of completions is to get the name

jedi/parser/tokenize.py

@@ -13,7 +13,9 @@ from __future__ import absolute_import
 
 import string
 import re
+from collections import namedtuple
 from io import StringIO
+
 from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, opmap,
                                NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT)
 from jedi._compatibility import is_py3
@@ -148,6 +150,9 @@ ALWAYS_BREAK_TOKENS = (';', 'import', 'class', 'def', 'try', 'except',
                        'finally', 'while', 'return')
 
 
+Token = namedtuple('Token', ['type', 'string', 'start_pos', 'prefix'])
+
+
 def source_tokens(source, use_exact_op_types=False):
     """Generate tokens from a the source code (string)."""
     source = source

jedi/parser/user_context.py

@@ -27,15 +27,7 @@ class UserContextParser(object):
 
     @cache.underscore_memoization
     def _parser(self):
-        cache.invalidate_star_import_cache(self._path)
+        pass
-        if self._use_fast_parser:
-            parser = FastParser(self._grammar, self._source, self._path)
-            # Don't pickle that module, because the main module is changing quickly
-            cache.save_parser(self._path, parser, pickling=False)
-        else:
-            parser = ParserWithRecovery(self._grammar, self._source, self._path)
-        self._parser_done_callback(parser)
-        return parser
 
     def module(self):
         return self._parser().module

test/test_parser/test_fast_parser.py

@@ -38,7 +38,7 @@ def test_class_in_docstr():
     Regression test for a problem with classes in docstrings.
     """
     a = '"\nclasses\n"'
-    jedi.Script(a, 1, 0)._parser
+    jedi.Script(a, 1, 0)._get_module()
 
     b = a + '\nimport os'
     assert jedi.Script(b, 4, 8).goto_assignments()
@@ -450,7 +450,7 @@ def test_string_literals():
     """)
 
     script = jedi.Script(dedent(source))
-    script._parser.module().end_pos == (6, 0)
+    script._get_module().end_pos == (6, 0)
     assert script.completions()
 
 
@@ -468,7 +468,7 @@ def test_decorator_string_issue():
 
     s = jedi.Script(source)
     assert s.completions()
-    assert s._parser.module().get_code() == source
+    assert s._get_module().get_code() == source
 
 
 def test_round_trip():