Merge branch 'master' into python3

CHANGELOG.rst

@@ -8,6 +8,10 @@ Unreleased
 
 - Added an option to pass environment variables to ``Environment``
 - ``Project(...).path`` exists now
+- Support for Python 3.9
+
+This will probably the last release that supports Python 2 and Python 3.5.
+``0.18.0`` will be Python 3.6+.
 
 0.17.1 (2020-06-20)
 +++++++++++++++++++

jedi/api/environment.py

@@ -61,7 +61,7 @@ class Environment(_BaseEnvironment):
     """
     _subprocess = None
 
-    def __init__(self, executable, env_vars={}):
+    def __init__(self, executable, env_vars=None):
         self._start_executable = executable
         self._env_vars = env_vars
         # Initialize the environment
@@ -126,7 +126,7 @@ class _SameEnvironmentMixin(object):
         self._start_executable = self.executable = sys.executable
         self.path = sys.prefix
         self.version_info = _VersionInfo(*sys.version_info[:3])
-        self._env_vars = {}
+        self._env_vars = None
 
 
 class SameEnvironment(_SameEnvironmentMixin, Environment):
@@ -353,7 +353,7 @@ def get_system_environment(version, *, env_vars={}):
     raise InvalidPythonEnvironment("Cannot find executable python%s." % version)
 
 
-def create_environment(path, *, safe=True, env_vars={}):
+def create_environment(path, *, safe=True, env_vars=None):
     """
     Make it possible to manually create an Environment object by specifying a
     Virtualenv path or an executable path and optional environment variables.

jedi/api/interpreter.py

@@ -3,6 +3,9 @@ TODO Some parts of this module are still not well documented.
 """
 
 from jedi.inference import compiled
+from jedi.inference.base_value import ValueSet
+from jedi.inference.filters import ParserTreeFilter, MergedFilter
+from jedi.inference.names import TreeNameDefinition
 from jedi.inference.compiled import mixed
 from jedi.inference.compiled.access import create_access_path
 from jedi.inference.context import ModuleContext
@@ -19,10 +22,37 @@ class NamespaceObject(object):
         self.__dict__ = dct
 
 
+class MixedTreeName(TreeNameDefinition):
+    def infer(self):
+        """
+        In IPython notebook it is typical that some parts of the code that is
+        provided was already executed. In that case if something is not properly
+        inferred, it should still infer from the variables it already knows.
+        """
+        inferred = super(MixedTreeName, self).infer()
+        if not inferred:
+            for compiled_value in self.parent_context.mixed_values:
+                for f in compiled_value.get_filters():
+                    values = ValueSet.from_sets(
+                        n.infer() for n in f.get(self.string_name)
+                    )
+                    if values:
+                        return values
+        return inferred
+
+
+class MixedParserTreeFilter(ParserTreeFilter):
+    name_class = MixedTreeName
+
+
 class MixedModuleContext(ModuleContext):
     def __init__(self, tree_module_value, namespaces):
         super().__init__(tree_module_value)
-        self._namespace_objects = [NamespaceObject(n) for n in namespaces]
+        self.mixed_values = [
+            self._get_mixed_object(
+                _create(self.inference_state, NamespaceObject(n))
+            ) for n in namespaces
+        ]
 
     def _get_mixed_object(self, compiled_value):
         return mixed.MixedObject(
@@ -30,12 +60,16 @@ class MixedModuleContext(ModuleContext):
             tree_value=self._value
         )
 
-    def get_filters(self, *args, **kwargs):
+    def get_filters(self, until_position=None, origin_scope=None):
-        for filter in self._value.as_context().get_filters(*args, **kwargs):
+        yield MergedFilter(
-            yield filter
+            MixedParserTreeFilter(
+                parent_context=self,
+                until_position=until_position,
+                origin_scope=origin_scope
+            ),
+            self.get_global_filter(),
+        )
 
-        for namespace_obj in self._namespace_objects:
+        for mixed_object in self.mixed_values:
-            compiled_value = _create(self.inference_state, namespace_obj)
+            for filter in mixed_object.get_filters(until_position, origin_scope):
-            mixed_object = self._get_mixed_object(compiled_value)
-            for filter in mixed_object.get_filters(*args, **kwargs):
                 yield filter

jedi/inference/base_value.py

@@ -176,6 +176,9 @@ class Value(HelperValueMixin):
                 message="TypeError: '%s' object is not iterable" % self)
         return iter([])
 
+    def py__next__(self, contextualized_node=None):
+        return self.py__iter__(contextualized_node)
+
     def get_signatures(self):
         return []
 

jedi/inference/compiled/mixed.py

@@ -69,12 +69,22 @@ class MixedObject(ValueWrapper):
         else:
             return self.compiled_value.get_safe_value(default)
 
+    @property
+    def array_type(self):
+        return self.compiled_value.array_type
+
+    def get_key_values(self):
+        return self.compiled_value.get_key_values()
+
     def py__simple_getitem__(self, index):
         python_object = self.compiled_value.access_handle.access._obj
         if type(python_object) in ALLOWED_GETITEM_TYPES:
             return self.compiled_value.py__simple_getitem__(index)
         return self._wrapped_value.py__simple_getitem__(index)
 
+    def negate(self):
+        return self.compiled_value.negate()
+
     def _as_context(self):
         if self.parent_context is None:
             return MixedModuleContext(self)

jedi/inference/compiled/subprocess/__init__.py

@@ -165,21 +165,12 @@ class InferenceStateSubprocess(_InferenceStateProcess):
 class CompiledSubprocess(object):
     is_crashed = False
 
-    def __init__(self, executable, env_vars={}):
+    def __init__(self, executable, env_vars=None):
         self._executable = executable
-        self._env_vars = dict(env_vars)
+        self._env_vars = env_vars
         self._inference_state_deletion_queue = queue.deque()
         self._cleanup_callable = lambda: None
 
-        # Use explicit envionment to ensure reliable results (#1540)
-        if os.name == 'nt':
-            # if SYSTEMROOT (or case variant) exists in environment,
-            # ensure it goes to subprocess
-            for k, v in os.environ.items():
-                if 'SYSTEMROOT' == k.upper():
-                    self._env_vars.update({k: os.environ[k]})
-                    break  # don't risk multiple entries
-
     def __repr__(self):
         pid = os.getpid()
         return '<%s _executable=%r, is_crashed=%r, pid=%r>' % (

jedi/inference/value/function.py

@@ -201,7 +201,7 @@ class MethodValue(FunctionValue):
 
 
 class BaseFunctionExecutionContext(ValueContext, TreeContextMixin):
-    def _infer_annotations(self):
+    def infer_annotations(self):
         raise NotImplementedError
 
     @inference_state_method_cache(default=NO_VALUES)
@@ -215,7 +215,7 @@ class BaseFunctionExecutionContext(ValueContext, TreeContextMixin):
             value_set = NO_VALUES
             returns = get_yield_exprs(self.inference_state, funcdef)
         else:
-            value_set = self._infer_annotations()
+            value_set = self.infer_annotations()
             if value_set:
                 # If there are annotations, prefer them over anything else.
                 # This will make it faster.
@@ -367,7 +367,7 @@ class FunctionExecutionContext(BaseFunctionExecutionContext):
             arguments=self._arguments
         )
 
-    def _infer_annotations(self):
+    def infer_annotations(self):
         from jedi.inference.gradual.annotation import infer_return_types
         return infer_return_types(self._value, self._arguments)
 
@@ -379,7 +379,7 @@ class FunctionExecutionContext(BaseFunctionExecutionContext):
 
 
 class AnonymousFunctionExecution(BaseFunctionExecutionContext):
-    def _infer_annotations(self):
+    def infer_annotations(self):
         # I don't think inferring anonymous executions is a big thing.
         # Anonymous contexts are mostly there for the user to work in. ~ dave
         return NO_VALUES

jedi/inference/value/instance.py

@@ -255,21 +255,20 @@ class _BaseTreeInstance(AbstractInstanceValue):
 
         def iterate():
             for generator in self.execute_function_slots(iter_slot_names):
-                if generator.is_instance() and not generator.is_compiled():
+                for lazy_value in generator.py__next__(contextualized_node):
-                    # `__next__` logic.
-                    name = '__next__'
-                    next_slot_names = generator.get_function_slot_names(name)
-                    if next_slot_names:
-                        yield LazyKnownValues(
-                            generator.execute_function_slots(next_slot_names)
-                        )
-                    else:
-                        debug.warning('Instance has no __next__ function in %s.', generator)
-                else:
-                    for lazy_value in generator.py__iter__():
                     yield lazy_value
         return iterate()
 
+    def py__next__(self, contextualized_node=None):
+        name = u'__next__'
+        next_slot_names = self.get_function_slot_names(name)
+        if next_slot_names:
+            yield LazyKnownValues(
+                self.execute_function_slots(next_slot_names)
+            )
+        else:
+            debug.warning('Instance has no __next__ function in %s.', self)
+
     def py__call__(self, arguments):
         names = self.get_function_slot_names('__call__')
         if not names:

jedi/inference/value/iterable.py

@@ -20,6 +20,9 @@ from jedi.inference.value.dynamic_arrays import check_array_additions
 
 
 class IterableMixin(object):
+    def py__next__(self, contextualized_node=None):
+        return self.py__iter__(contextualized_node)
+
     def py__stop_iteration_returns(self):
         return ValueSet([compiled.builtin_from_name(self.inference_state, 'None')])
 
@@ -36,13 +39,12 @@ class GeneratorBase(LazyAttributeOverwrite, IterableMixin):
     array_type = None
 
     def _get_wrapped_value(self):
-        generator, = self.inference_state.typing_module \
+        instance, = self._get_cls().execute_annotation()
-            .py__getattribute__('Generator') \
+        return instance
-            .execute_annotation()
-        return generator
 
-    def is_instance(self):
+    def _get_cls(self):
-        return False
+        generator, = self.inference_state.typing_module.py__getattribute__('Generator')
+        return generator
 
     def py__bool__(self):
         return True
@@ -52,9 +54,8 @@ class GeneratorBase(LazyAttributeOverwrite, IterableMixin):
         return ValueSet([self])
 
     @publish_method('send')
-    @publish_method('next')
     @publish_method('__next__')
-    def py__next__(self, arguments):
+    def _next(self, arguments):
         return ValueSet.from_sets(lazy_value.infer() for lazy_value in self.py__iter__())
 
     def py__stop_iteration_returns(self):
@@ -64,6 +65,12 @@ class GeneratorBase(LazyAttributeOverwrite, IterableMixin):
     def name(self):
         return compiled.CompiledValueName(self, 'Generator')
 
+    def get_annotated_class_object(self):
+        from jedi.inference.gradual.generics import TupleGenericManager
+        gen_values = self.merge_types_of_iterate().py__class__()
+        gm = TupleGenericManager((gen_values, NO_VALUES, NO_VALUES))
+        return self._get_cls().with_generics(gm)
+
 
 class Generator(GeneratorBase):
     """Handling of `yield` functions."""
@@ -72,6 +79,9 @@ class Generator(GeneratorBase):
         self._func_execution_context = func_execution_context
 
     def py__iter__(self, contextualized_node=None):
+        iterators = self._func_execution_context.infer_annotations()
+        if iterators:
+            return iterators.iterate(contextualized_node)
         return self._func_execution_context.get_yield_lazy_values()
 
     def py__stop_iteration_returns(self):

jedi/plugins/stdlib.py

@@ -260,9 +260,8 @@ class ReversedObject(AttributeOverwrite):
     def py__iter__(self, contextualized_node):
         return self._iter_list
 
-    @publish_method('next')
     @publish_method('__next__')
-    def py__next__(self, arguments):
+    def _next(self, arguments):
         return ValueSet.from_sets(
             lazy_value.infer() for lazy_value in self._iter_list
         )

test/completion/generators.py

@@ -290,3 +290,22 @@ def test_in_brackets():
     x = yield from [1]
     #? None
     x
+
+
+# -----------------
+# Annotations
+# -----------------
+
+from typing import Iterator
+
+def annotation1() -> float:
+    yield 1
+
+def annotation2() -> Iterator[float]:
+    yield 1
+
+
+#?
+next(annotation1())
+#? float()
+next(annotation2())

test/completion/stdlib.py

@@ -212,11 +212,33 @@ z.read('name').upper
 # contextlib
 # -----------------
 
+from typing import Iterator
 import contextlib
 with contextlib.closing('asd') as string:
     #? str()
     string
 
+@contextlib.contextmanager
+def cm1() -> Iterator[float]:
+    yield 1
+with cm1() as x:
+    #? float()
+    x
+
+@contextlib.contextmanager
+def cm2() -> float:
+    yield 1
+with cm2() as x:
+    #?
+    x
+
+@contextlib.contextmanager
+def cm3():
+    yield 3
+with cm3() as x:
+    #? int()
+    x
+
 # -----------------
 # operator
 # -----------------

test/test_api/test_interpreter.py

@@ -561,12 +561,18 @@ def test_param_annotation_completion(class_is_findable):
         ('mixed[Non', 9, ['e']),
 
         ('implicit[10', None, ['00']),
+
+        ('inherited["', None, ['blablu"']),
     ]
 )
 def test_dict_completion(code, column, expected):
     strs = {'asdf': 1, """foo""": 2, r'fbar': 3}
     mixed = {1: 2, 1.10: 4, None: 6, r'a\sdf': 8, b'foo': 9}
 
+    class Inherited(dict):
+        pass
+    inherited = Inherited(blablu=3)
+
     namespaces = [locals(), {'implicit': {1000: 3}}]
     comps = jedi.Interpreter(code, namespaces).complete(column=column)
     if Ellipsis in expected:
@@ -646,3 +652,28 @@ def test_string_annotation(annotations, result, code):
     x.__annotations__ = annotations
     defs = jedi.Interpreter(code or 'x()', [locals()]).infer()
     assert [d.name for d in defs] == result
+
+
+def test_name_not_inferred_properly():
+    """
+    In IPython notebook it is typical that some parts of the code that is
+    provided was already executed. In that case if something is not properly
+    inferred, it should still infer from the variables it already knows.
+    """
+    x = 1
+    d, = jedi.Interpreter('x = UNDEFINED; x', [locals()]).infer()
+    assert d.name == 'int'
+
+
+def test_variable_reuse():
+    x = 1
+    d, = jedi.Interpreter('y = x\ny', [locals()]).infer()
+    assert d.name == 'int'
+
+
+def test_negate():
+    code = "x = -y"
+    x, = jedi.Interpreter(code, [{'y': 3}]).infer(1, 0)
+    assert x.name == 'int'
+    value, = x._name.infer()
+    assert value.get_safe_value() == -3