From c4c36d8e2e5798b0c1ce915dce63ae2778f741e2 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jul 2020 19:44:48 +0200 Subject: [PATCH 01/10] Mention in Changelog that 3.9 is now supported --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d7b6f388..b34230e0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,7 @@ Unreleased - Added an option to pass environment variables to ``Environment`` - ``Project(...).path`` exists now +- Support for Python 3.9 0.17.1 (2020-06-20) +++++++++++++++++++ From 0c618a4456c4a9f5eb88d5a153f7f3dc399eea68 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jul 2020 21:21:59 +0200 Subject: [PATCH 02/10] Making sure to note that Python 2 will not be supported after 0.17.2 --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b34230e0..0353eee1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Unreleased - ``Project(...).path`` exists now - Support for Python 3.9 +This will probably the last release that supports Python 2 and Python 3.5. +``0.18.0`` will be Python 3.6+. + 0.17.1 (2020-06-20) +++++++++++++++++++ From 396d7df3142c6c6be460376b095ea2ac538edd43 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jul 2020 22:01:55 +0200 Subject: [PATCH 03/10] Fix an issue with interpreter completion, see also #1628 --- jedi/api/interpreter.py | 15 +++++++++++---- test/test_api/test_interpreter.py | 6 ++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/jedi/api/interpreter.py b/jedi/api/interpreter.py index bb7268a7..101c696d 100644 --- a/jedi/api/interpreter.py +++ b/jedi/api/interpreter.py @@ -3,6 +3,7 @@ TODO Some parts of this module are still not well documented. """ from jedi.inference import compiled +from jedi.inference.filters import ParserTreeFilter, MergedFilter from jedi.inference.compiled import mixed from jedi.inference.compiled.access import create_access_path from jedi.inference.context import ModuleContext @@ -30,12 +31,18 @@ class MixedModuleContext(ModuleContext): tree_value=self._value ) - def get_filters(self, *args, **kwargs): - for filter in self._value.as_context().get_filters(*args, **kwargs): - yield filter + def get_filters(self, until_position=None, origin_scope=None): + yield MergedFilter( + ParserTreeFilter( + parent_context=self, + until_position=until_position, + origin_scope=origin_scope + ), + self.get_global_filter(), + ) for namespace_obj in self._namespace_objects: compiled_value = _create(self.inference_state, namespace_obj) mixed_object = self._get_mixed_object(compiled_value) - for filter in mixed_object.get_filters(*args, **kwargs): + for filter in mixed_object.get_filters(until_position, origin_scope): yield filter diff --git a/test/test_api/test_interpreter.py b/test/test_api/test_interpreter.py index bae881de..e6782e7f 100644 --- a/test/test_api/test_interpreter.py +++ b/test/test_api/test_interpreter.py @@ -681,3 +681,9 @@ def test_string_annotation(annotations, result, code): x.__annotations__ = annotations defs = jedi.Interpreter(code or 'x()', [locals()]).infer() assert [d.name for d in defs] == result + + +def test_variable_reuse(): + x = 1 + d, = jedi.Interpreter('y = x\ny', [locals()]).infer() + assert d.name == 'int' From 3ffe8475b8b4780c60b660afab165dda2ceb9e2f Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jul 2020 22:20:06 +0200 Subject: [PATCH 04/10] Make sure the interpreter completions work better in Jupyter Notebook, fixes #1628 --- jedi/api/interpreter.py | 35 +++++++++++++++++++++++++++---- test/test_api/test_interpreter.py | 11 ++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/jedi/api/interpreter.py b/jedi/api/interpreter.py index 101c696d..0f944ccf 100644 --- a/jedi/api/interpreter.py +++ b/jedi/api/interpreter.py @@ -3,7 +3,9 @@ TODO Some parts of this module are still not well documented. """ from jedi.inference import compiled +from jedi.inference.base_value import ValueSet from jedi.inference.filters import ParserTreeFilter, MergedFilter +from jedi.inference.names import TreeNameDefinition from jedi.inference.compiled import mixed from jedi.inference.compiled.access import create_access_path from jedi.inference.context import ModuleContext @@ -20,10 +22,36 @@ class NamespaceObject(object): self.__dict__ = dct +class MixedTreeName(TreeNameDefinition): + def infer(self): + """ + In IPython notebook it is typical that some parts of the code that is + provided was already executed. In that case if something is not properly + inferred, it should still infer from the variables it already knows. + """ + inferred = super(MixedTreeName, self).infer() + if not inferred: + for compiled_value in self.parent_context.compiled_values: + for f in compiled_value.get_filters(): + values = ValueSet.from_sets( + n.infer() for n in f.get(self.string_name) + ) + if values: + return values + return inferred + + +class MixedParserTreeFilter(ParserTreeFilter): + name_class = MixedTreeName + + class MixedModuleContext(ModuleContext): def __init__(self, tree_module_value, namespaces): super(MixedModuleContext, self).__init__(tree_module_value) - self._namespace_objects = [NamespaceObject(n) for n in namespaces] + self.compiled_values = [ + _create(self.inference_state, NamespaceObject(n)) + for n in namespaces + ] def _get_mixed_object(self, compiled_value): return mixed.MixedObject( @@ -33,7 +61,7 @@ class MixedModuleContext(ModuleContext): def get_filters(self, until_position=None, origin_scope=None): yield MergedFilter( - ParserTreeFilter( + MixedParserTreeFilter( parent_context=self, until_position=until_position, origin_scope=origin_scope @@ -41,8 +69,7 @@ class MixedModuleContext(ModuleContext): self.get_global_filter(), ) - for namespace_obj in self._namespace_objects: - compiled_value = _create(self.inference_state, namespace_obj) + for compiled_value in self.compiled_values: mixed_object = self._get_mixed_object(compiled_value) for filter in mixed_object.get_filters(until_position, origin_scope): yield filter diff --git a/test/test_api/test_interpreter.py b/test/test_api/test_interpreter.py index e6782e7f..1bd06405 100644 --- a/test/test_api/test_interpreter.py +++ b/test/test_api/test_interpreter.py @@ -683,6 +683,17 @@ def test_string_annotation(annotations, result, code): assert [d.name for d in defs] == result +def test_name_not_inferred_properly(): + """ + In IPython notebook it is typical that some parts of the code that is + provided was already executed. In that case if something is not properly + inferred, it should still infer from the variables it already knows. + """ + x = 1 + d, = jedi.Interpreter('x = UNDEFINED; x', [locals()]).infer() + assert d.name == 'int' + + def test_variable_reuse(): x = 1 d, = jedi.Interpreter('y = x\ny', [locals()]).infer() From 7f3a7db7e65e545e2b31f7cd820b6aeb60d8a9fd Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 12 Jul 2020 22:26:57 +0200 Subject: [PATCH 05/10] Refactor Interpeter completions a bit --- jedi/api/interpreter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jedi/api/interpreter.py b/jedi/api/interpreter.py index 0f944ccf..541840a1 100644 --- a/jedi/api/interpreter.py +++ b/jedi/api/interpreter.py @@ -31,7 +31,7 @@ class MixedTreeName(TreeNameDefinition): """ inferred = super(MixedTreeName, self).infer() if not inferred: - for compiled_value in self.parent_context.compiled_values: + for compiled_value in self.parent_context.mixed_values: for f in compiled_value.get_filters(): values = ValueSet.from_sets( n.infer() for n in f.get(self.string_name) @@ -48,9 +48,10 @@ class MixedParserTreeFilter(ParserTreeFilter): class MixedModuleContext(ModuleContext): def __init__(self, tree_module_value, namespaces): super(MixedModuleContext, self).__init__(tree_module_value) - self.compiled_values = [ - _create(self.inference_state, NamespaceObject(n)) - for n in namespaces + self.mixed_values = [ + self._get_mixed_object( + _create(self.inference_state, NamespaceObject(n)) + ) for n in namespaces ] def _get_mixed_object(self, compiled_value): @@ -69,7 +70,6 @@ class MixedModuleContext(ModuleContext): self.get_global_filter(), ) - for compiled_value in self.compiled_values: - mixed_object = self._get_mixed_object(compiled_value) + for mixed_object in self.mixed_values: for filter in mixed_object.get_filters(until_position, origin_scope): yield filter From 99573745084ad67f030dbada4051f068a56760ff Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 14 Jul 2020 17:50:12 +0200 Subject: [PATCH 06/10] Fix dict completions for inherited dicts, fixes #1631 --- jedi/inference/compiled/mixed.py | 7 +++++++ test/test_api/test_interpreter.py | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/jedi/inference/compiled/mixed.py b/jedi/inference/compiled/mixed.py index 98959fd8..7d76113a 100644 --- a/jedi/inference/compiled/mixed.py +++ b/jedi/inference/compiled/mixed.py @@ -71,6 +71,13 @@ class MixedObject(ValueWrapper): else: return self.compiled_value.get_safe_value(default) + @property + def array_type(self): + return self.compiled_value.array_type + + def get_key_values(self): + return self.compiled_value.get_key_values() + def py__simple_getitem__(self, index): python_object = self.compiled_value.access_handle.access._obj if type(python_object) in ALLOWED_GETITEM_TYPES: diff --git a/test/test_api/test_interpreter.py b/test/test_api/test_interpreter.py index 1bd06405..ee7617ca 100644 --- a/test/test_api/test_interpreter.py +++ b/test/test_api/test_interpreter.py @@ -594,12 +594,18 @@ def test_param_annotation_completion(class_is_findable): ('mixed[Non', 9, ['e']), ('implicit[10', None, ['00']), + + ('inherited["', None, ['blablu"']), ] ) def test_dict_completion(code, column, expected): strs = {'asdf': 1, u"""foo""": 2, r'fbar': 3} mixed = {1: 2, 1.10: 4, None: 6, r'a\sdf': 8, b'foo': 9} + class Inherited(dict): + pass + inherited = Inherited(blablu=3) + namespaces = [locals(), {'implicit': {1000: 3}}] comps = jedi.Interpreter(code, namespaces).complete(column=column) if Ellipsis in expected: From b0f664ec949f0d65334a4a48c0a39222520b21f4 Mon Sep 17 00:00:00 2001 From: Ryan Clary Date: Wed, 15 Jul 2020 08:30:24 -0700 Subject: [PATCH 07/10] * reflect default Popen behavior by inheriting os.environ * without passing env_vars to create_environment, GeneralizedPopen behavior is same as before fix to issue #1540 (803c3cb271ead297c4fe3ca916b54ed05a623459) * env_vars allows explicit environment variables, per PR #1619 (f9183bbf6436fc4c4a384c7c8dcb45324c8720f6) --- jedi/api/environment.py | 6 +++--- jedi/inference/compiled/subprocess/__init__.py | 13 ++----------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/jedi/api/environment.py b/jedi/api/environment.py index 0b376067..89e47163 100644 --- a/jedi/api/environment.py +++ b/jedi/api/environment.py @@ -61,7 +61,7 @@ class Environment(_BaseEnvironment): """ _subprocess = None - def __init__(self, executable, env_vars={}): + def __init__(self, executable, env_vars=None): self._start_executable = executable self._env_vars = env_vars # Initialize the environment @@ -136,7 +136,7 @@ class _SameEnvironmentMixin(object): self._start_executable = self.executable = sys.executable self.path = sys.prefix self.version_info = _VersionInfo(*sys.version_info[:3]) - self._env_vars = {} + self._env_vars = None class SameEnvironment(_SameEnvironmentMixin, Environment): @@ -379,7 +379,7 @@ def create_environment(path, safe=True, **kwargs): return _create_environment(path, safe, **kwargs) -def _create_environment(path, safe=True, env_vars={}): +def _create_environment(path, safe=True, env_vars=None): if os.path.isfile(path): _assert_safe(path, safe) return Environment(path, env_vars=env_vars) diff --git a/jedi/inference/compiled/subprocess/__init__.py b/jedi/inference/compiled/subprocess/__init__.py index 56c03df9..35724a48 100644 --- a/jedi/inference/compiled/subprocess/__init__.py +++ b/jedi/inference/compiled/subprocess/__init__.py @@ -156,21 +156,12 @@ class CompiledSubprocess(object): # Start with 2, gets set after _get_info. _pickle_protocol = 2 - def __init__(self, executable, env_vars={}): + def __init__(self, executable, env_vars=None): self._executable = executable - self._env_vars = dict(env_vars) + self._env_vars = env_vars self._inference_state_deletion_queue = queue.deque() self._cleanup_callable = lambda: None - # Use explicit envionment to ensure reliable results (#1540) - if os.name == 'nt': - # if SYSTEMROOT (or case variant) exists in environment, - # ensure it goes to subprocess - for k, v in os.environ.items(): - if 'SYSTEMROOT' == k.upper(): - self._env_vars.update({k: os.environ[k]}) - break # don't risk multiple entries - def __repr__(self): pid = os.getpid() return '<%s _executable=%r, _pickle_protocol=%r, is_crashed=%r, pid=%r>' % ( From d1851c369c07cb7b0e54312a7c321e7685ab3be3 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 17 Jul 2020 12:05:59 +0200 Subject: [PATCH 08/10] Introduce py__next__ to have more clear way to use __next__ --- jedi/inference/base_value.py | 3 +++ jedi/inference/value/instance.py | 32 ++++++++++++++++---------------- jedi/inference/value/iterable.py | 5 ++++- jedi/plugins/stdlib.py | 2 +- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/jedi/inference/base_value.py b/jedi/inference/base_value.py index f8a8ff09..c516d463 100644 --- a/jedi/inference/base_value.py +++ b/jedi/inference/base_value.py @@ -175,6 +175,9 @@ class Value(HelperValueMixin): message="TypeError: '%s' object is not iterable" % self) return iter([]) + def py__next__(self, contextualized_node=None): + return self.py__iter__(contextualized_node) + def get_signatures(self): return [] diff --git a/jedi/inference/value/instance.py b/jedi/inference/value/instance.py index 5435c791..dab389d5 100644 --- a/jedi/inference/value/instance.py +++ b/jedi/inference/value/instance.py @@ -256,24 +256,24 @@ class _BaseTreeInstance(AbstractInstanceValue): def iterate(): for generator in self.execute_function_slots(iter_slot_names): - if generator.is_instance() and not generator.is_compiled(): - # `__next__` logic. - if self.inference_state.environment.version_info.major == 2: - name = u'next' - else: - name = u'__next__' - next_slot_names = generator.get_function_slot_names(name) - if next_slot_names: - yield LazyKnownValues( - generator.execute_function_slots(next_slot_names) - ) - else: - debug.warning('Instance has no __next__ function in %s.', generator) - else: - for lazy_value in generator.py__iter__(): - yield lazy_value + for lazy_value in generator.py__next__(contextualized_node): + yield lazy_value return iterate() + def py__next__(self, contextualized_node=None): + # `__next__` logic. + if self.inference_state.environment.version_info.major == 2: + name = u'next' + else: + name = u'__next__' + next_slot_names = self.get_function_slot_names(name) + if next_slot_names: + yield LazyKnownValues( + self.execute_function_slots(next_slot_names) + ) + else: + debug.warning('Instance has no __next__ function in %s.', self) + def py__call__(self, arguments): names = self.get_function_slot_names(u'__call__') if not names: diff --git a/jedi/inference/value/iterable.py b/jedi/inference/value/iterable.py index 43692c57..8a60d31d 100644 --- a/jedi/inference/value/iterable.py +++ b/jedi/inference/value/iterable.py @@ -23,6 +23,9 @@ from jedi.inference.value.dynamic_arrays import check_array_additions class IterableMixin(object): + def py__next__(self, contextualized_node=None): + return self.py__iter__(contextualized_node) + def py__stop_iteration_returns(self): return ValueSet([compiled.builtin_from_name(self.inference_state, u'None')]) @@ -64,7 +67,7 @@ class GeneratorBase(LazyAttributeOverwrite, IterableMixin): @publish_method('send') @publish_method('next', python_version_match=2) @publish_method('__next__', python_version_match=3) - def py__next__(self, arguments): + def _next(self, arguments): return ValueSet.from_sets(lazy_value.infer() for lazy_value in self.py__iter__()) def py__stop_iteration_returns(self): diff --git a/jedi/plugins/stdlib.py b/jedi/plugins/stdlib.py index 5e093e72..f24cadc9 100644 --- a/jedi/plugins/stdlib.py +++ b/jedi/plugins/stdlib.py @@ -267,7 +267,7 @@ class ReversedObject(AttributeOverwrite): @publish_method('next', python_version_match=2) @publish_method('__next__', python_version_match=3) - def py__next__(self, arguments): + def _next(self, arguments): return ValueSet.from_sets( lazy_value.infer() for lazy_value in self._iter_list ) From e4987b3e7adc876d255b9ba0b2ac45c635f5d91e Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 17 Jul 2020 15:35:01 +0200 Subject: [PATCH 09/10] Fix issues with generators, fixes #1624 --- jedi/inference/value/function.py | 8 ++++---- jedi/inference/value/iterable.py | 20 ++++++++++++++------ test/completion/generators.py | 19 +++++++++++++++++++ test/completion/stdlib.py | 24 +++++++++++++++++++++++- 4 files changed, 60 insertions(+), 11 deletions(-) diff --git a/jedi/inference/value/function.py b/jedi/inference/value/function.py index 7513c6a9..6f4da364 100644 --- a/jedi/inference/value/function.py +++ b/jedi/inference/value/function.py @@ -202,7 +202,7 @@ class MethodValue(FunctionValue): class BaseFunctionExecutionContext(ValueContext, TreeContextMixin): - def _infer_annotations(self): + def infer_annotations(self): raise NotImplementedError @inference_state_method_cache(default=NO_VALUES) @@ -216,7 +216,7 @@ class BaseFunctionExecutionContext(ValueContext, TreeContextMixin): value_set = NO_VALUES returns = get_yield_exprs(self.inference_state, funcdef) else: - value_set = self._infer_annotations() + value_set = self.infer_annotations() if value_set: # If there are annotations, prefer them over anything else. # This will make it faster. @@ -373,7 +373,7 @@ class FunctionExecutionContext(BaseFunctionExecutionContext): arguments=self._arguments ) - def _infer_annotations(self): + def infer_annotations(self): from jedi.inference.gradual.annotation import infer_return_types return infer_return_types(self._value, self._arguments) @@ -385,7 +385,7 @@ class FunctionExecutionContext(BaseFunctionExecutionContext): class AnonymousFunctionExecution(BaseFunctionExecutionContext): - def _infer_annotations(self): + def infer_annotations(self): # I don't think inferring anonymous executions is a big thing. # Anonymous contexts are mostly there for the user to work in. ~ dave return NO_VALUES diff --git a/jedi/inference/value/iterable.py b/jedi/inference/value/iterable.py index 8a60d31d..d2411a80 100644 --- a/jedi/inference/value/iterable.py +++ b/jedi/inference/value/iterable.py @@ -49,13 +49,12 @@ class GeneratorBase(LazyAttributeOverwrite, IterableMixin): array_type = None def _get_wrapped_value(self): - generator, = self.inference_state.typing_module \ - .py__getattribute__('Generator') \ - .execute_annotation() - return generator + instance, = self._get_cls().execute_annotation() + return instance - def is_instance(self): - return False + def _get_cls(self): + generator, = self.inference_state.typing_module.py__getattribute__('Generator') + return generator def py__bool__(self): return True @@ -77,6 +76,12 @@ class GeneratorBase(LazyAttributeOverwrite, IterableMixin): def name(self): return compiled.CompiledValueName(self, 'Generator') + def get_annotated_class_object(self): + from jedi.inference.gradual.generics import TupleGenericManager + gen_values = self.merge_types_of_iterate().py__class__() + gm = TupleGenericManager((gen_values, NO_VALUES, NO_VALUES)) + return self._get_cls().with_generics(gm) + class Generator(GeneratorBase): """Handling of `yield` functions.""" @@ -85,6 +90,9 @@ class Generator(GeneratorBase): self._func_execution_context = func_execution_context def py__iter__(self, contextualized_node=None): + iterators = self._func_execution_context.infer_annotations() + if iterators: + return iterators.iterate(contextualized_node) return self._func_execution_context.get_yield_lazy_values() def py__stop_iteration_returns(self): diff --git a/test/completion/generators.py b/test/completion/generators.py index 566f0036..570ce309 100644 --- a/test/completion/generators.py +++ b/test/completion/generators.py @@ -292,3 +292,22 @@ def test_in_brackets(): x = yield from [1] #? None x + + +# ----------------- +# Annotations +# ----------------- + +from typing import Iterator + +def annotation1() -> float: + yield 1 + +def annotation2() -> Iterator[float]: + yield 1 + + +#? +next(annotation1()) +#? float() +next(annotation2()) diff --git a/test/completion/stdlib.py b/test/completion/stdlib.py index 7dcc6c37..48ab1ee4 100644 --- a/test/completion/stdlib.py +++ b/test/completion/stdlib.py @@ -253,12 +253,34 @@ z.read('name').upper # ----------------- # contextlib # ----------------- - +# python > 2.7 +from typing import Iterator import contextlib with contextlib.closing('asd') as string: #? str() string +@contextlib.contextmanager +def cm1() -> Iterator[float]: + yield 1 +with cm1() as x: + #? float() + x + +@contextlib.contextmanager +def cm2() -> float: + yield 1 +with cm2() as x: + #? + x + +@contextlib.contextmanager +def cm3(): + yield 3 +with cm3() as x: + #? int() + x + # ----------------- # operator # ----------------- From 7851dff915cea7bd0d1f386e43740032efd81d8d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 17 Jul 2020 15:57:15 +0200 Subject: [PATCH 10/10] Properly negate with Interpreter, fixes #1636 --- jedi/inference/compiled/mixed.py | 3 +++ test/test_api/test_interpreter.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/jedi/inference/compiled/mixed.py b/jedi/inference/compiled/mixed.py index 7d76113a..ce31d7e0 100644 --- a/jedi/inference/compiled/mixed.py +++ b/jedi/inference/compiled/mixed.py @@ -84,6 +84,9 @@ class MixedObject(ValueWrapper): return self.compiled_value.py__simple_getitem__(index) return self._wrapped_value.py__simple_getitem__(index) + def negate(self): + return self.compiled_value.negate() + def _as_context(self): if self.parent_context is None: return MixedModuleContext(self) diff --git a/test/test_api/test_interpreter.py b/test/test_api/test_interpreter.py index ee7617ca..0155d49f 100644 --- a/test/test_api/test_interpreter.py +++ b/test/test_api/test_interpreter.py @@ -704,3 +704,11 @@ def test_variable_reuse(): x = 1 d, = jedi.Interpreter('y = x\ny', [locals()]).infer() assert d.name == 'int' + + +def test_negate(): + code = "x = -y" + x, = jedi.Interpreter(code, [{'y': 3}]).infer(1, 0) + assert x.name == 'int' + value, = x._name.infer() + assert value.get_safe_value() == -3