Changes for 3.8: sync_comp_for instead of comp_for

Please also look at the changes for parso in its b5d50392a4058919c0018666cdfc8c3eaaea9cb5 commit

jedi/evaluate/__init__.py

@@ -410,12 +410,12 @@ class Evaluator(object):
                 if parser_utils.is_scope(node):
                     return node
                 elif node.type in ('argument', 'testlist_comp'):
-                    if node.children[1].type == 'comp_for':
+                    if node.children[1].type in ('comp_for', 'sync_comp_for'):
                         return node.children[1]
                 elif node.type == 'dictorsetmaker':
                     for n in node.children[1:4]:
                         # In dictionaries it can be pretty much anything.
-                        if n.type == 'comp_for':
+                        if n.type in ('comp_for', 'sync_comp_for'):
                             return n
 
         def from_scope_node(scope_node, is_nested=True, node_is_object=False):
@@ -441,7 +441,7 @@ class Evaluator(object):
                 return func
             elif scope_node.type == 'classdef':
                 return ClassContext(self, parent_context, scope_node)
-            elif scope_node.type == 'comp_for':
+            elif scope_node.type in ('comp_for', 'sync_comp_for'):
                 if node.start_pos >= scope_node.children[-1].start_pos:
                     return parent_context
                 return CompForContext.from_comp_for(parent_context, scope_node)

jedi/evaluate/arguments.py

@@ -233,10 +233,13 @@ class TreeArguments(AbstractArguments):
                         named_args.append((c[0].value, LazyTreeContext(self.context, c[2]),))
                     else:  # Generator comprehension.
                         # Include the brackets with the parent.
+                        sync_comp_for = el.children[1]
+                        if sync_comp_for.type == 'comp_for':
+                            sync_comp_for = sync_comp_for.children[1]
                         comp = iterable.GeneratorComprehension(
                             self._evaluator,
                             defining_context=self.context,
-                            comp_for_node=el.children[1],
+                            sync_comp_for_node=sync_comp_for,
                             entry_node=el.children[0],
                         )
                         yield None, LazyKnownContext(comp)

jedi/evaluate/base_context.py

@@ -8,7 +8,7 @@ just one.
 """
 from functools import reduce
 from operator import add
-from parso.python.tree import ExprStmt, CompFor
+from parso.python.tree import ExprStmt, SyncCompFor
 
 from jedi import debug
 from jedi._compatibility import zip_longest, unicode
@@ -330,7 +330,7 @@ class ContextualizedName(ContextualizedNode):
                 is_star_expr = False
             elif node.type == 'star_expr':
                 is_star_expr = True
-            elif isinstance(node, (ExprStmt, CompFor)):
+            elif isinstance(node, (ExprStmt, SyncCompFor)):
                 break
 
             compare = node

jedi/evaluate/context/instance.py

@@ -215,7 +215,7 @@ class AbstractInstanceContext(Context):
             elif scope.type == 'classdef':
                 class_context = ClassContext(self.evaluator, parent_context, scope)
                 return class_context
-            elif scope.type == 'comp_for':
+            elif scope.type in ('comp_for', 'sync_comp_for'):
                 # Comprehensions currently don't have a special scope in Jedi.
                 return self.create_instance_context(class_context, scope)
             else:

jedi/evaluate/context/iterable.py

@@ -38,7 +38,7 @@ from jedi.evaluate.filters import ParserTreeFilter, LazyAttributeOverwrite, \
     publish_method
 from jedi.evaluate.base_context import ContextSet, NO_CONTEXTS, \
     TreeContext, ContextualizedNode, iterate_contexts, HelperContextMixin
-from jedi.parser_utils import get_comp_fors
+from jedi.parser_utils import get_sync_comp_fors
 
 
 class IterableMixin(object):
@@ -107,12 +107,17 @@ class CompForContext(TreeContext):
 def comprehension_from_atom(evaluator, context, atom):
     bracket = atom.children[0]
     test_list_comp = atom.children[1]
+
     if bracket == '{':
         if atom.children[1].children[1] == ':':
+            sync_comp_for = test_list_comp.children[3]
+            if sync_comp_for.type == 'comp_for':
+                sync_comp_for = sync_comp_for.children[1]
+
             return DictComprehension(
                 evaluator,
                 context,
-                comp_for_node=test_list_comp.children[3],
+                sync_comp_for_node=sync_comp_for,
                 key_node=test_list_comp.children[0],
                 value_node=test_list_comp.children[2],
             )
@@ -122,10 +127,15 @@ def comprehension_from_atom(evaluator, context, atom):
         cls = GeneratorComprehension
     elif bracket == '[':
         cls = ListComprehension
+
+    sync_comp_for = test_list_comp.children[1]
+    if sync_comp_for.type == 'comp_for':
+        sync_comp_for = sync_comp_for.children[1]
+
     return cls(
         evaluator,
         defining_context=context,
-        comp_for_node=test_list_comp.children[1],
+        sync_comp_for_node=sync_comp_for,
         entry_node=test_list_comp.children[0],
     )
 
@@ -138,16 +148,16 @@ class ComprehensionMixin(object):
     def _nested(self, comp_fors, parent_context=None):
         comp_for = comp_fors[0]
 
-        is_async = 'async' == comp_for.children[comp_for.children.index('for') - 1]
+        is_async = comp_for.parent.type == 'comp_for'
 
-        input_node = comp_for.children[comp_for.children.index('in') + 1]
+        input_node = comp_for.children[3]
         parent_context = parent_context or self._defining_context
         input_types = parent_context.eval_node(input_node)
         # TODO: simulate await if self.is_async
 
         cn = ContextualizedNode(parent_context, input_node)
         iterated = input_types.iterate(cn, is_async=is_async)
-        exprlist = comp_for.children[comp_for.children.index('for') + 1]
+        exprlist = comp_for.children[1]
         for i, lazy_context in enumerate(iterated):
             types = lazy_context.infer()
             dct = unpack_tuple_to_dict(parent_context, types, exprlist)
@@ -169,7 +179,7 @@ class ComprehensionMixin(object):
     @evaluator_method_cache(default=[])
     @to_list
     def _iterate(self):
-        comp_fors = tuple(get_comp_fors(self._comp_for_node))
+        comp_fors = tuple(get_sync_comp_fors(self._sync_comp_for_node))
         for result in self._nested(comp_fors):
             yield result
 
@@ -178,7 +188,7 @@ class ComprehensionMixin(object):
             yield LazyKnownContexts(set_)
 
     def __repr__(self):
-        return "<%s of %s>" % (type(self).__name__, self._comp_for_node)
+        return "<%s of %s>" % (type(self).__name__, self._sync_comp_for_node)
 
 
 class _DictMixin(object):
@@ -219,10 +229,11 @@ class Sequence(LazyAttributeOverwrite, IterableMixin):
 
 
 class _BaseComprehension(ComprehensionMixin):
-    def __init__(self, evaluator, defining_context, comp_for_node, entry_node):
+    def __init__(self, evaluator, defining_context, sync_comp_for_node, entry_node):
+        assert sync_comp_for_node.type == 'sync_comp_for'
         super(_BaseComprehension, self).__init__(evaluator)
         self._defining_context = defining_context
-        self._comp_for_node = comp_for_node
+        self._sync_comp_for_node = sync_comp_for_node
         self._entry_node = entry_node
 
 
@@ -250,10 +261,11 @@ class GeneratorComprehension(_BaseComprehension, GeneratorBase):
 class DictComprehension(ComprehensionMixin, Sequence):
     array_type = u'dict'
 
-    def __init__(self, evaluator, defining_context, comp_for_node, key_node, value_node):
+    def __init__(self, evaluator, defining_context, sync_comp_for_node, key_node, value_node):
+        assert sync_comp_for_node.type == 'sync_comp_for'
         super(DictComprehension, self).__init__(evaluator)
         self._defining_context = defining_context
-        self._comp_for_node = comp_for_node
+        self._sync_comp_for_node = sync_comp_for_node
         self._entry_node = key_node
         self._value_node = value_node
 

jedi/evaluate/finder.py

@@ -134,11 +134,12 @@ class NameFinder(object):
         ``filters``), until a name fits.
         """
         names = []
+        # TODO why is this paragraph needed?
         if self._context.predefined_names and isinstance(self._name, tree.Name):
             node = self._name
             while node is not None and not is_scope(node):
                 node = node.parent
-                if node.type in ("if_stmt", "for_stmt", "comp_for"):
+                if node.type in ("if_stmt", "for_stmt", "comp_for", 'sync_comp_for'):
                     try:
                         name_dict = self._context.predefined_names[node]
                         types = name_dict[self._string_name]
@@ -199,7 +200,7 @@ class NameFinder(object):
             flow_scope = self._name
             base_nodes = [self._name_context.tree_node]
 
-            if any(b.type == 'comp_for' for b in base_nodes):
+            if any(b.type in ('comp_for', 'sync_comp_for') for b in base_nodes):
                 return contexts
             while True:
                 flow_scope = get_parent_scope(flow_scope, include_flows=True)

jedi/evaluate/syntax_tree.py

@@ -247,7 +247,7 @@ def eval_atom(context, atom):
                 except IndexError:
                     pass
 
-            if comp_for.type == 'comp_for':
+            if comp_for.type in ('comp_for', 'sync_comp_for'):
                 return ContextSet([iterable.comprehension_from_atom(
                     context.evaluator, context, atom
                 )])
@@ -588,7 +588,7 @@ def tree_name_to_contexts(evaluator, context, tree_name):
         if types:
             return types
 
-    if typ in ('for_stmt', 'comp_for'):
+    if typ in ('for_stmt', 'comp_for', 'sync_comp_for'):
         try:
             types = context.predefined_names[node][tree_name.value]
         except KeyError:

jedi/parser_utils.py

@@ -54,11 +54,13 @@ def get_executable_nodes(node, last_added=False):
     return result
 
 
-def get_comp_fors(comp_for):
+def get_sync_comp_fors(comp_for):
     yield comp_for
     last = comp_for.children[-1]
     while True:
         if last.type == 'comp_for':
+            yield last.children[1]  # Ignore the async.
+        elif last.type == 'sync_comp_for':
             yield last
         elif not last.type == 'comp_if':
             break
@@ -216,7 +218,12 @@ def get_following_comment_same_line(node):
 
 
 def is_scope(node):
-    return node.type in ('file_input', 'classdef', 'funcdef', 'lambdef', 'comp_for')
+    t = node.type
+    if t == 'comp_for':
+        # Starting with Python 3.8, async is outside of the statement.
+        return node.children[1].type != 'sync_comp_for'
+
+    return t in ('file_input', 'classdef', 'funcdef', 'lambdef', 'sync_comp_for')
 
 
 def get_parent_scope(node, include_flows=False):