From 8eb980db73d1e5576d1ff92e6da079f37f3f644c Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sun, 26 Jan 2020 19:25:23 +0100 Subject: [PATCH 1/7] Create the basics to work with TypedDict in the future --- jedi/inference/base_value.py | 1 + jedi/inference/gradual/typing.py | 62 ++++++++++++++++++++++++++++++- jedi/inference/syntax_tree.py | 3 ++ jedi/inference/value/klass.py | 33 +++++++++++++++- jedi/parser_utils.py | 19 ++++++++++ test/completion/pep0484_typing.py | 41 ++++++++++++++++++++ 6 files changed, 156 insertions(+), 3 deletions(-) diff --git a/jedi/inference/base_value.py b/jedi/inference/base_value.py index 96f43395..3705d7bc 100644 --- a/jedi/inference/base_value.py +++ b/jedi/inference/base_value.py @@ -258,6 +258,7 @@ class Value(HelperValueMixin, BaseValue): def _as_context(self): raise NotImplementedError('Not all values need to be converted to contexts: %s', self) + @property def name(self): raise NotImplementedError diff --git a/jedi/inference/gradual/typing.py b/jedi/inference/gradual/typing.py index 3b3cdf17..64e1e43c 100644 --- a/jedi/inference/gradual/typing.py +++ b/jedi/inference/gradual/typing.py @@ -81,7 +81,8 @@ class TypingModuleName(NameWrapper): elif name == 'TypedDict': # TODO doesn't even exist in typeshed/typing.py, yet. But will be # added soon. - pass + yield TypedDictBase.create_cached( + inference_state, self.parent_context, self.tree_name) elif name in ('no_type_check', 'no_type_check_decorator'): # This is not necessary, as long as we are not doing type checking. for c in self._wrapped_name.infer(): # Fuck my life Python 2 @@ -339,3 +340,62 @@ class CastFunction(BaseTypingValue): @repack_with_argument_clinic('type, object, /') def py__call__(self, type_value_set, object_value_set): return type_value_set.execute_annotation() + + +class TypedDictBase(BaseTypingValue): + """ + This class has no responsibilities and is just here to make sure that typed + dicts can be identified. + """ + + +class TypedDictClass(Value): + """ + This represents a class defined like: + + class Foo(TypedDict): + bar: str + """ + def __init__(self, definition_class): + super().__init__(definition_class.inference_state, definition_class.parent_context) + self.tree_node = definition_class.tree_node + self._definition_class = definition_class + + def get_filters(self, origin_scope=None): + """ + A TypedDict doesn't have attributes. + """ + o, = self.inference_state.builtins_module.py__getattribute__('object') + return o.get_filters() + + @property + def name(self): + return ValueName(self, self.tree_node.name) + + def py__call__(self, arguments): + return ValueSet({TypedDict(self._definition_class)}) + + +class TypedDict(LazyValueWrapper): + """Represents the instance version of ``TypedDictClass``.""" + def __init__(self, definition_class): + self.inference_state = definition_class.inference_state + self.parent_context = definition_class.parent_context + self.tree_node = definition_class.tree_node + self._definition_class = definition_class + + @property + def name(self): + return ValueName(self, self.tree_node.name) + + def py__simple_getitem__(self, index): + return ValueSet({self.inference_state.builtins_module}) + + def get_key_values(self): + from jedi.inference.compiled import create_simple_object + return ValueSet({create_simple_object(self.inference_state, 'baz')}) + + def _get_wrapped_value(self): + d, = self.inference_state.builtins_module.py__getattribute__('dict') + result, = d.execute_with_values() + return result diff --git a/jedi/inference/syntax_tree.py b/jedi/inference/syntax_tree.py index 6512fa76..51bc960a 100644 --- a/jedi/inference/syntax_tree.py +++ b/jedi/inference/syntax_tree.py @@ -26,6 +26,7 @@ from jedi.inference.compiled.access import COMPARISON_OPERATORS from jedi.inference.cache import inference_state_method_cache from jedi.inference.gradual.stub_value import VersionInfo from jedi.inference.gradual import annotation +from jedi.inference.gradual.typing import TypedDictClass from jedi.inference.names import TreeNameDefinition from jedi.inference.context import CompForContext from jedi.inference.value.decorator import Decoratee @@ -748,6 +749,8 @@ def _apply_decorators(context, node): parent_context=context, tree_node=node ) + if decoratee_value.is_typeddict(): + decoratee_value = TypedDictClass(decoratee_value) else: decoratee_value = FunctionValue.from_context(context, node) initial = values = ValueSet([decoratee_value]) diff --git a/jedi/inference/value/klass.py b/jedi/inference/value/klass.py index b7d79646..743c17cf 100644 --- a/jedi/inference/value/klass.py +++ b/jedi/inference/value/klass.py @@ -38,11 +38,11 @@ py__doc__() Returns the docstring for a value. """ from jedi import debug from jedi._compatibility import use_metaclass -from jedi.parser_utils import get_cached_parent_scope +from jedi.parser_utils import get_cached_parent_scope, expr_is_dotted from jedi.inference.cache import inference_state_method_cache, CachedMetaClass, \ inference_state_method_generator_cache from jedi.inference import compiled -from jedi.inference.lazy_value import LazyKnownValues +from jedi.inference.lazy_value import LazyKnownValues, LazyTreeValue from jedi.inference.filters import ParserTreeFilter from jedi.inference.names import TreeNameDefinition, ValueName from jedi.inference.arguments import unpack_arglist, ValuesArguments @@ -265,6 +265,35 @@ class ClassValue(use_metaclass(CachedMetaClass, ClassMixin, FunctionAndClassBase self.inference_state.builtins_module.py__getattribute__('object') )] + def is_typeddict(self): + # TODO Do a proper mro resolution. Currently we are just listing + # classes. However, it's a complicated algorithm. + from jedi.inference.gradual.typing import TypedDictBase + for lazy_cls in self.py__bases__(): + if not isinstance(lazy_cls, LazyTreeValue): + return False + tree_node = lazy_cls.data + # Only resolve simple classes, stuff like Iterable[str] are more + # intensive to resolve and if generics are involved, we know it's + # not a TypedDict. + if not expr_is_dotted(tree_node): + return False + + for cls in lazy_cls.infer(): + if isinstance(cls, TypedDictBase): + return True + try: + method = cls.is_typeddict + except AttributeError: + # We're only dealing with simple classes, so just returning + # here should be fine. This only happens with e.g. compiled + # classes. + return False + else: + if method(): + return True + return False + def py__getitem__(self, index_value_set, contextualized_node): from jedi.inference.gradual.base import GenericClass if not index_value_set: diff --git a/jedi/parser_utils.py b/jedi/parser_utils.py index 0f7ba429..7f69a80c 100644 --- a/jedi/parser_utils.py +++ b/jedi/parser_utils.py @@ -293,6 +293,25 @@ def cut_value_at_position(leaf, position): return ''.join(lines) +def expr_is_dotted(node): + """ + Checks if a path looks like `name` or `name.foo.bar` and not `name()`. + """ + if node.type == 'atom': + if len(node.children) == 3 and node.children[0] == '(': + return expr_is_dotted(node.children[1]) + return False + if node.type == 'atom_expr': + children = node.children + if children[0] == 'await': + return False + if not expr_is_dotted(children[0]): + return False + # Check trailers + return all(c.children[0] == '.' for c in children[1:]) + return node.type == 'name' + + def _function_is_x_method(method_name): def wrapper(function_node): """ diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index 9104ca11..7a48aca9 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -493,3 +493,44 @@ def dynamic_annotation(x: int): #? int() dynamic_annotation('') + +# ------------------------- +# TypeDict +# ------------------------- + +# python >= 3.8 + +class Foo(typing.TypedDict): + foo: str + bar: List[int] + foo + #! ['foo: str'] + foo + #? str() + foo + +#! ['class Foo'] +d: Foo +#? str() +d['foo'] +#? str() +d['bar'][0] +#? +d['baz'] + +#? +d.foo +#? +d.bar +#! [] +d.foo + +#? [] +Foo.set +#? ['setdefault'] +d.setdefaul + +#? 5 ["'foo'"] +d['fo'] +#? 5 ['"bar"'] +d["bar"] From 87161df2f0a5d2fa21d310b65efe538ce6c8d8e7 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 7 Feb 2020 16:45:03 +0100 Subject: [PATCH 2/7] Make sure that typeddict py__getitem__ works --- jedi/inference/gradual/typing.py | 8 +++++++- jedi/inference/value/instance.py | 6 +++--- jedi/inference/value/klass.py | 9 +++++---- test/completion/pep0484_typing.py | 4 ++-- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/jedi/inference/gradual/typing.py b/jedi/inference/gradual/typing.py index 64e1e43c..2d7d18ef 100644 --- a/jedi/inference/gradual/typing.py +++ b/jedi/inference/gradual/typing.py @@ -389,7 +389,13 @@ class TypedDict(LazyValueWrapper): return ValueName(self, self.tree_node.name) def py__simple_getitem__(self, index): - return ValueSet({self.inference_state.builtins_module}) + if isinstance(index, str): + return ValueSet.from_sets( + name.infer() + for filter in self._definition_class.get_filters(is_instance=True) + for name in filter.get(index) + ) + return NO_VALUES def get_key_values(self): from jedi.inference.compiled import create_simple_object diff --git a/jedi/inference/value/instance.py b/jedi/inference/value/instance.py index 2eca9f9d..b884b14d 100644 --- a/jedi/inference/value/instance.py +++ b/jedi/inference/value/instance.py @@ -547,10 +547,10 @@ class InstanceClassFilter(AbstractFilter): self._class_filter = class_filter def get(self, name): - return self._convert(self._class_filter.get(name, from_instance=True)) + return self._convert(self._class_filter.get(name)) def values(self): - return self._convert(self._class_filter.values(from_instance=True)) + return self._convert(self._class_filter.values()) def _convert(self, names): return [ @@ -586,7 +586,7 @@ class SelfAttributeFilter(ClassFilter): if trailer.type == 'trailer' \ and len(trailer.parent.children) == 2 \ and trailer.children[0] == '.': - if name.is_definition() and self._access_possible(name, from_instance=True): + if name.is_definition() and self._access_possible(name): # TODO filter non-self assignments instead of this bad # filter. if self._is_in_right_scope(trailer.parent.children[0], name): diff --git a/jedi/inference/value/klass.py b/jedi/inference/value/klass.py index 764644d6..41c669d7 100644 --- a/jedi/inference/value/klass.py +++ b/jedi/inference/value/klass.py @@ -104,12 +104,12 @@ class ClassFilter(ParserTreeFilter): node = get_cached_parent_scope(self._used_names, node) return False - def _access_possible(self, name, from_instance=False): + def _access_possible(self, name): # Filter for ClassVar variables # TODO this is not properly done, yet. It just checks for the string # ClassVar in the annotation, which can be quite imprecise. If we # wanted to do this correct, we would have to infer the ClassVar. - if not from_instance: + if not self._is_instance: expr_stmt = name.get_definition() if expr_stmt is not None and expr_stmt.type == 'expr_stmt': annassign = expr_stmt.children[1] @@ -122,9 +122,9 @@ class ClassFilter(ParserTreeFilter): return not name.value.startswith('__') or name.value.endswith('__') \ or self._equals_origin_scope() - def _filter(self, names, from_instance=False): + def _filter(self, names): names = super(ClassFilter, self)._filter(names) - return [name for name in names if self._access_possible(name, from_instance)] + return [name for name in names if self._access_possible(name)] class ClassMixin(object): @@ -270,6 +270,7 @@ class ClassValue(use_metaclass(CachedMetaClass, ClassMixin, FunctionAndClassBase self.inference_state.builtins_module.py__getattribute__('object') )] + @inference_state_method_cache(default=False) def is_typeddict(self): # TODO Do a proper mro resolution. Currently we are just listing # classes. However, it's a complicated algorithm. diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index 8dc03d96..eef0bf13 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -508,7 +508,7 @@ dynamic_annotation('') class Foo(typing.TypedDict): foo: str - bar: List[int] + bar: typing.List[float] foo #! ['foo: str'] foo @@ -519,7 +519,7 @@ class Foo(typing.TypedDict): d: Foo #? str() d['foo'] -#? str() +#? float() d['bar'][0] #? d['baz'] From 6a9745b42ba2365f08b4966ee82c9f0a61a48d97 Mon Sep 17 00:00:00 2001 From: Sam Roeca Date: Tue, 28 Jan 2020 14:07:05 -0500 Subject: [PATCH 3/7] Get basic completions working with TypedDict --- jedi/inference/gradual/typing.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/jedi/inference/gradual/typing.py b/jedi/inference/gradual/typing.py index 2d7d18ef..ffb9b99b 100644 --- a/jedi/inference/gradual/typing.py +++ b/jedi/inference/gradual/typing.py @@ -5,8 +5,9 @@ values. This file deals with all the typing.py cases. """ +import itertools from jedi import debug -from jedi.inference.compiled import builtin_from_name +from jedi.inference.compiled import builtin_from_name, create_simple_object from jedi.inference.base_value import ValueSet, NO_VALUES, Value, \ LazyValueWrapper from jedi.inference.lazy_value import LazyKnownValues @@ -398,8 +399,14 @@ class TypedDict(LazyValueWrapper): return NO_VALUES def get_key_values(self): - from jedi.inference.compiled import create_simple_object - return ValueSet({create_simple_object(self.inference_state, 'baz')}) + filtered_values = itertools.chain.from_iterable(( + f.values(from_instance=True) + for f in self._definition_class.get_filters(is_instance=True) + )) + return ValueSet({ + create_simple_object(self.inference_state, v.string_name) + for v in filtered_values + }) def _get_wrapped_value(self): d, = self.inference_state.builtins_module.py__getattribute__('dict') From 9d2083fa08d207c9b9784e8f548cfadc3251d0c6 Mon Sep 17 00:00:00 2001 From: Sam Roeca Date: Fri, 7 Feb 2020 13:38:52 -0500 Subject: [PATCH 4/7] Remove argument to filter.values() Given 87161df2, values(from_instance=False) doesn't produce completions anymore. Therefore, we remove from_instance as an argument. --- jedi/inference/gradual/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/inference/gradual/typing.py b/jedi/inference/gradual/typing.py index ffb9b99b..be353391 100644 --- a/jedi/inference/gradual/typing.py +++ b/jedi/inference/gradual/typing.py @@ -400,7 +400,7 @@ class TypedDict(LazyValueWrapper): def get_key_values(self): filtered_values = itertools.chain.from_iterable(( - f.values(from_instance=True) + f.values() for f in self._definition_class.get_filters(is_instance=True) )) return ValueSet({ From cf954bf0068b3e0b70d0962f87e8e1ac1a751de1 Mon Sep 17 00:00:00 2001 From: Sam Roeca Date: Fri, 7 Feb 2020 14:40:39 -0500 Subject: [PATCH 5/7] Expand on TypedDict tests. Adds a function that takes the TypedDict as an argument. Note: the last two tests are failing, along with lots of other tests throughout the system. --- test/completion/pep0484_typing.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index eef0bf13..e81a443f 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -509,11 +509,33 @@ dynamic_annotation('') class Foo(typing.TypedDict): foo: str bar: typing.List[float] + an_int: int foo #! ['foo: str'] foo #? str() foo + #? int() + an_int + +def typed_dict_test_foo(arg: Foo): + a_string = arg['foo'] + a_list_of_floats = arg['bar'] + an_int = arg['an_int'] + + #? str() + a_string + #? [float()] + a_list_of_strings + #? int() + an_int + + #? ['isupper'] + a_string.isuppe + #? ['pop'] + a_list_of_floats.po + #? ['as_integer_ratio'] + an_int.as_integer_rati #! ['class Foo'] d: Foo From ac47866c4cb8c425b99c4a30918236c90f3f2cce Mon Sep 17 00:00:00 2001 From: Sam Roeca Date: Tue, 11 Feb 2020 18:32:15 -0500 Subject: [PATCH 6/7] TypedDict: fix non-inheritance tests, add inheritance Note: tests currently failing --- test/completion/pep0484_typing.py | 43 ++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index e81a443f..6472880d 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -510,7 +510,6 @@ class Foo(typing.TypedDict): foo: str bar: typing.List[float] an_int: int - foo #! ['foo: str'] foo #? str() @@ -525,8 +524,10 @@ def typed_dict_test_foo(arg: Foo): #? str() a_string - #? [float()] - a_list_of_strings + #? list() + a_list_of_floats + #? float() + a_list_of_floats[0] #? int() an_int @@ -558,7 +559,41 @@ Foo.set #? ['setdefault'] d.setdefaul -#? 5 ["'foo'"] +#? 5 ["'foo"] d['fo'] #? 5 ['"bar"'] d["bar"] + +class Bar(Foo): + another_variable: int + + #? int() + another_variable + #? str() + foo + #? int() + an_int + +def typed_dict_test_foo(arg: Bar): + a_string = arg['foo'] + a_list_of_floats = arg['bar'] + an_int = arg['an_int'] + another_variable = arg['another_variable'] + + #? str() + a_string + #? list() + a_list_of_floats + #? float() + a_list_of_floats[0] + #? int() + an_int + #? int() + another_variable + + #? ['isupper'] + a_string.isuppe + #? ['pop'] + a_list_of_floats.po + #? ['as_integer_ratio'] + an_int.as_integer_rati From d6f6c29a636f631214719494005ed0204846b518 Mon Sep 17 00:00:00 2001 From: Sam Roeca Date: Thu, 13 Feb 2020 10:43:41 -0500 Subject: [PATCH 7/7] TypedDict test: fix Bar inheritance checks Note: foo is defined as a function a the module level so I remove it from consideration here to avoid complicating this test with other tests in the module. --- test/completion/pep0484_typing.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py index 6472880d..73ab32ee 100644 --- a/test/completion/pep0484_typing.py +++ b/test/completion/pep0484_typing.py @@ -569,9 +569,7 @@ class Bar(Foo): #? int() another_variable - #? str() - foo - #? int() + #? an_int def typed_dict_test_foo(arg: Bar):