Make jedi work with the next parso release

jedi/api/classes.py

@@ -404,8 +404,9 @@ class Completion(BaseDefinition):
             append = '('
 
         if self._name.api_type == 'param' and self._stack is not None:
-            node_names = list(self._stack.get_node_names(self._evaluator.grammar._pgen_grammar))
+            nonterminals = list(self._stack._list_nonterminals())
-            if 'trailer' in node_names and 'argument' not in node_names:
+            if 'trailer' in nonterminals and 'argument' not in nonterminals:
+                # TODO this doesn't work for nested calls.
                 append += '='
 
         name = self._name.string_name

jedi/api/completion.py

@@ -1,4 +1,4 @@
-from parso.python import token
+from parso.python.token import PythonTokenTypes
 from parso.python import tree
 from parso.tree import search_ancestor, Leaf
 
@@ -122,11 +122,11 @@ class Completion:
         grammar = self._evaluator.grammar
 
         try:
-            self.stack = helpers.get_stack_at_position(
+            self.stack = stack = helpers.get_stack_at_position(
                 grammar, self._code_lines, self._module_node, self._position
             )
         except helpers.OnErrorLeaf as e:
-            self.stack = None
+            self.stack = stack = None
             if e.error_leaf.value == '.':
                 # After ErrorLeaf's that are dots, we will not do any
                 # completions since this probably just confuses the user.
@@ -135,10 +135,10 @@ class Completion:
 
             return self._global_completions()
 
-        allowed_keywords, allowed_tokens = \
+        allowed_transitions = \
-            helpers.get_possible_completion_types(grammar._pgen_grammar, self.stack)
+            list(stack._allowed_transition_names_and_token_types())
 
-        if 'if' in allowed_keywords:
+        if 'if' in allowed_transitions:
             leaf = self._module_node.get_leaf_for_position(self._position, include_prefixes=True)
             previous_leaf = leaf.get_previous_leaf()
 
@@ -164,50 +164,52 @@ class Completion:
                     # Compare indents
                     if stmt.start_pos[1] == indent:
                         if type_ == 'if_stmt':
-                            allowed_keywords += ['elif', 'else']
+                            allowed_transitions += ['elif', 'else']
                         elif type_ == 'try_stmt':
-                            allowed_keywords += ['except', 'finally', 'else']
+                            allowed_transitions += ['except', 'finally', 'else']
                         elif type_ == 'for_stmt':
-                            allowed_keywords.append('else')
+                            allowed_transitions.append('else')
 
-        completion_names = list(self._get_keyword_completion_names(allowed_keywords))
+        completion_names = list(self._get_keyword_completion_names(allowed_transitions))
 
-        if token.NAME in allowed_tokens or token.INDENT in allowed_tokens:
+        if any(t in allowed_transitions for t in (PythonTokenTypes.NAME,
+                                                  PythonTokenTypes.INDENT)):
             # This means that we actually have to do type inference.
 
-            symbol_names = list(self.stack.get_node_names(grammar._pgen_grammar))
+            nonterminals = stack._list_nonterminals()
 
-            nodes = list(self.stack.get_nodes())
+            nodes = stack._list_all_nodes()
 
             if nodes and nodes[-1] in ('as', 'def', 'class'):
                 # No completions for ``with x as foo`` and ``import x as foo``.
                 # Also true for defining names as a class or function.
                 return list(self._get_class_context_completions(is_function=True))
-            elif "import_stmt" in symbol_names:
+            elif "import_stmt" in nonterminals:
-                level, names = self._parse_dotted_names(nodes, "import_from" in symbol_names)
+                level, names = self._parse_dotted_names(nodes, "import_from" in nonterminals)
 
-                only_modules = not ("import_from" in symbol_names and 'import' in nodes)
+                only_modules = not ("import_from" in nonterminals and 'import' in nodes)
                 completion_names += self._get_importer_names(
                     names,
                     level,
                     only_modules=only_modules,
                 )
-            elif symbol_names[-1] in ('trailer', 'dotted_name') and nodes[-1] == '.':
+            elif nonterminals[-1] in ('trailer', 'dotted_name') and nodes[-1] == '.':
                 dot = self._module_node.get_leaf_for_position(self._position)
                 completion_names += self._trailer_completions(dot.get_previous_leaf())
             else:
                 completion_names += self._global_completions()
                 completion_names += self._get_class_context_completions(is_function=False)
 
-            if 'trailer' in symbol_names:
+            if 'trailer' in nonterminals:
                 call_signatures = self._call_signatures_method()
                 completion_names += get_call_signature_param_names(call_signatures)
 
         return completion_names
 
-    def _get_keyword_completion_names(self, keywords_):
+    def _get_keyword_completion_names(self, allowed_transitions):
-        for k in keywords_:
+        for k in allowed_transitions:
-            yield keywords.KeywordName(self._evaluator, k)
+            if isinstance(k, str) and k.isalpha():
+                yield keywords.KeywordName(self._evaluator, k)
 
     def _global_completions(self):
         context = get_user_scope(self._module_context, self._position)

jedi/api/helpers.py

@@ -127,7 +127,7 @@ def get_stack_at_position(grammar, code_lines, module_node, pos):
     try:
         p.parse(tokens=tokenize_without_endmarker(code))
     except EndMarkerReached:
-        return Stack(p.pgen_parser.stack)
+        return p.stack
     raise SystemError("This really shouldn't happen. There's a bug in Jedi.")
 
 
@@ -153,7 +153,6 @@ def get_possible_completion_types(pgen_grammar, stack):
                 t, v = pgen_grammar.labels[label_index]
                 assert t >= 256
                 # See if it's a symbol and if we're in its first set
-                inversed_keywords
                 itsdfa = pgen_grammar.dfas[t]
                 itsstates, itsfirst = itsdfa
                 for first_label_index in itsfirst.keys():

test/test_api/test_completion.py

@@ -137,3 +137,7 @@ def test_async(Script, environment):
     names = [c.name for c in comps]
     assert 'foo' in names
     assert 'hey' in names
+
+
+def test_with_stmt_error_recovery(Script):
+    assert Script('with open('') as foo: foo.\na', line=1).completions()

tox.ini

@@ -24,7 +24,7 @@ setenv =
     env37: JEDI_TEST_ENVIRONMENT=37
 commands =
 # Overwrite the parso version (only used sometimes).
-#    pip install git+https://github.com/davidhalter/parso.git
+    pip install git+https://github.com/davidhalter/parso.git
     py.test {posargs:jedi test}
 [testenv:py27]
 deps =