From a433ee7a7e3b24413634a86cf38050c0faa04a2d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 20 Sep 2017 20:32:26 +0200 Subject: [PATCH 01/35] Move common to evaluate.utils. --- jedi/api/classes.py | 6 +-- jedi/api/keywords.py | 4 +- jedi/evaluate/__init__.py | 2 +- jedi/evaluate/context.py | 2 +- jedi/evaluate/docstrings.py | 3 +- jedi/evaluate/dynamic.py | 2 +- jedi/evaluate/filters.py | 2 +- jedi/evaluate/finder.py | 2 +- jedi/evaluate/imports.py | 2 +- jedi/evaluate/instance.py | 2 +- jedi/evaluate/iterable.py | 6 +-- jedi/evaluate/param.py | 4 +- jedi/evaluate/pep0484.py | 2 +- jedi/evaluate/stdlib.py | 2 +- jedi/evaluate/sys_path.py | 8 ++-- jedi/{common.py => evaluate/utils.py} | 39 +++++++++-------- jedi/refactoring.py | 60 +++++++++++++-------------- 17 files changed, 72 insertions(+), 76 deletions(-) rename jedi/{common.py => evaluate/utils.py} (99%) diff --git a/jedi/api/classes.py b/jedi/api/classes.py index 5b97d743..e4ed84e3 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -11,7 +11,7 @@ from parso.python.tree import search_ancestor from jedi._compatibility import u from jedi import settings -from jedi import common +from jedi.evaluate.utils import ignored, unite from jedi.cache import memoize_method from jedi.evaluate import representation as er from jedi.evaluate import instance @@ -311,7 +311,7 @@ class BaseDefinition(object): if not path: return None # for keywords the path is empty - with common.ignored(KeyError): + with ignored(KeyError): path[0] = self._mapping[path[0]] for key, repl in self._tuple_mapping.items(): if tuple(path[:len(key)]) == key: @@ -588,7 +588,7 @@ class Definition(BaseDefinition): """ defs = self._name.infer() return sorted( - common.unite(defined_names(self._evaluator, d) for d in defs), + unite(defined_names(self._evaluator, d) for d in defs), key=lambda s: s._name.start_pos or (0, 0) ) diff --git a/jedi/api/keywords.py b/jedi/api/keywords.py index 295add6a..a1bc4e7f 100644 --- a/jedi/api/keywords.py +++ b/jedi/api/keywords.py @@ -2,7 +2,7 @@ import pydoc import keyword from jedi._compatibility import is_py3, is_py35 -from jedi import common +from jedi.evaluate.utils import ignored from jedi.evaluate.filters import AbstractNameDefinition from parso.python.tree import Leaf @@ -123,7 +123,7 @@ def imitate_pydoc(string): # with unicode strings) string = str(string) h = pydoc.help - with common.ignored(KeyError): + with ignored(KeyError): # try to access symbols string = h.symbols[string] string, _, related = string.partition(' ') diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index fc074446..346f3528 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -67,7 +67,7 @@ from parso.python import tree import parso from jedi import debug -from jedi.common import unite +from jedi.evaluate.utils import unite from jedi.evaluate import representation as er from jedi.evaluate import imports from jedi.evaluate import recursion diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index d111ff24..8532cdb1 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -1,5 +1,5 @@ from jedi._compatibility import Python3Method -from jedi.common import unite +from jedi.evaluate.utils import unite from parso.python.tree import ExprStmt, CompFor from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index 3a141003..c552d5e3 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -21,10 +21,9 @@ from textwrap import dedent from parso import parse from jedi._compatibility import u -from jedi.common import unite +from jedi.evaluate.utils import unite, indent_block from jedi.evaluate import context from jedi.evaluate.cache import evaluator_method_cache -from jedi.common import indent_block from jedi.evaluate.iterable import SequenceLiteralContext, FakeSequence diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index 00c38f51..fb179b58 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -24,7 +24,7 @@ from jedi.evaluate.cache import evaluator_function_cache from jedi.evaluate import imports from jedi.evaluate.param import TreeArguments, create_default_params from jedi.evaluate.helpers import is_stdlib_path -from jedi.common import to_list, unite +from jedi.evaluate.utils import to_list, unite from jedi.parser_utils import get_parent_scope diff --git a/jedi/evaluate/filters.py b/jedi/evaluate/filters.py index 54eb01f4..004aec2b 100644 --- a/jedi/evaluate/filters.py +++ b/jedi/evaluate/filters.py @@ -6,7 +6,7 @@ from abc import abstractmethod from parso.tree import search_ancestor from jedi.evaluate import flow_analysis -from jedi.common import to_list, unite +from jedi.evaluate.utils import to_list, unite from jedi.parser_utils import get_parent_scope diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index aba190e1..38a1b357 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -18,7 +18,7 @@ check for -> a is a string). There's big potential in these checks. from parso.python import tree from parso.tree import search_ancestor from jedi import debug -from jedi.common import unite +from jedi.evaluate.utils import unite from jedi import settings from jedi.evaluate import representation as er from jedi.evaluate.instance import AbstractInstanceContext diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index d0aec515..12ed630a 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -24,7 +24,7 @@ from parso import python_bytes_to_unicode from jedi._compatibility import find_module, unicode, ImplicitNSInfo from jedi import debug from jedi import settings -from jedi.common import unite +from jedi.evaluate.utils import unite from jedi.evaluate import sys_path from jedi.evaluate import helpers from jedi.evaluate import compiled diff --git a/jedi/evaluate/instance.py b/jedi/evaluate/instance.py index 6ce26f6e..52852d1e 100644 --- a/jedi/evaluate/instance.py +++ b/jedi/evaluate/instance.py @@ -1,7 +1,7 @@ from abc import abstractproperty from jedi._compatibility import is_py3 -from jedi.common import unite +from jedi.evaluate.utils import unite from jedi import debug from jedi.evaluate import compiled from jedi.evaluate import filters diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 593f354a..6e51894d 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -22,9 +22,9 @@ It is important to note that: """ from jedi import debug from jedi import settings -from jedi import common -from jedi.common import unite, safe_property +from jedi.evaluate.utils import unite, safe_property from jedi._compatibility import unicode, zip_longest, is_py3 +from jedi.evaluate.utils import to_list from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate import analysis @@ -262,7 +262,7 @@ class Comprehension(AbstractSequence): yield iterated @evaluator_method_cache(default=[]) - @common.to_list + @to_list def _iterate(self): comp_fors = tuple(get_comp_fors(self._get_comp_for())) for result in self._nested(comp_fors): diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index c6fb55f4..5490df1f 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -2,7 +2,7 @@ from collections import defaultdict from jedi._compatibility import zip_longest from jedi import debug -from jedi import common +from jedi.evaluate.utils import PushBackIterator from parso.python import tree from jedi.evaluate import iterable from jedi.evaluate import analysis @@ -258,7 +258,7 @@ def get_params(execution_context, var_args): for param in funcdef.get_params(): param_dict[param.name.value] = param unpacked_va = list(var_args.unpack(funcdef)) - var_arg_iterator = common.PushBackIterator(iter(unpacked_va)) + var_arg_iterator = PushBackIterator(iter(unpacked_va)) non_matching_keys = defaultdict(lambda: []) keys_used = {} diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index bd90957d..f9a5c5ed 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -26,7 +26,7 @@ import re from parso import ParserSyntaxError from parso.python import tree -from jedi.common import unite +from jedi.evaluate.utils import unite from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate import compiled from jedi.evaluate.context import LazyTreeContext diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index d6a506f2..f2878e71 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -12,7 +12,7 @@ compiled module that returns the types for C-builtins. import collections import re -from jedi.common import unite +from jedi.evaluate.utils import unite from jedi.evaluate import compiled from jedi.evaluate import representation as er from jedi.evaluate.instance import InstanceFunctionExecution, \ diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index a4990c3c..057479ec 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -10,13 +10,13 @@ from jedi.evaluate.compiled import CompiledObject from jedi.evaluate.context import ContextualizedNode from jedi import settings from jedi import debug -from jedi import common +from jedi.evaluate.utils import ignored def get_venv_path(venv): """Get sys.path for specified virtual environment.""" sys_path = _get_venv_path_dirs(venv) - with common.ignored(ValueError): + with ignored(ValueError): sys_path.remove('') sys_path = _get_sys_path_with_egglinks(sys_path) # As of now, get_venv_path_dirs does not scan built-in pythonpath and @@ -194,7 +194,7 @@ def sys_path_with_modifications(evaluator, module_context): curdir = os.path.abspath(os.curdir) #TODO why do we need a chdir? - with common.ignored(OSError): + with ignored(OSError): os.chdir(os.path.dirname(path)) buildout_script_paths = set() @@ -246,7 +246,7 @@ def _detect_django_path(module_path): result = [] for parent in traverse_parents(module_path): - with common.ignored(IOError): + with ignored(IOError): with open(parent + os.path.sep + 'manage.py'): debug.dbg('Found django path: %s', module_path) result.append(parent) diff --git a/jedi/common.py b/jedi/evaluate/utils.py similarity index 99% rename from jedi/common.py rename to jedi/evaluate/utils.py index 007a1852..7fc1c246 100644 --- a/jedi/common.py +++ b/jedi/evaluate/utils.py @@ -4,7 +4,17 @@ import contextlib import functools from jedi._compatibility import reraise -from jedi import settings + + +def to_list(func): + def wrapper(*args, **kwargs): + return list(func(*args, **kwargs)) + return wrapper + + +def unite(iterable): + """Turns a two dimensional array into a one dimensional.""" + return set(typ for types in iterable for typ in types) class UncaughtAttributeError(Exception): @@ -78,16 +88,6 @@ class PushBackIterator(object): return self.current -def indent_block(text, indention=' '): - """This function indents a text block with a default of four spaces.""" - temp = '' - while text and text[-1] == '\n': - temp += text[-1] - text = text[:-1] - lines = text.split('\n') - return '\n'.join(map(lambda s: indention + s, lines)) + temp - - @contextlib.contextmanager def ignored(*exceptions): """ @@ -100,12 +100,11 @@ def ignored(*exceptions): pass -def unite(iterable): - """Turns a two dimensional array into a one dimensional.""" - return set(typ for types in iterable for typ in types) - - -def to_list(func): - def wrapper(*args, **kwargs): - return list(func(*args, **kwargs)) - return wrapper +def indent_block(text, indention=' '): + """This function indents a text block with a default of four spaces.""" + temp = '' + while text and text[-1] == '\n': + temp += text[-1] + text = text[:-1] + lines = text.split('\n') + return '\n'.join(map(lambda s: indention + s, lines)) + temp diff --git a/jedi/refactoring.py b/jedi/refactoring.py index b04caed6..ee938427 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -14,7 +14,6 @@ following functions (sometimes bug-prone): """ import difflib -from jedi import common from parso import python_bytes_to_unicode, split_lines from jedi.evaluate import helpers @@ -165,38 +164,37 @@ def inline(script): dct = {} definitions = script.goto_assignments() - with common.ignored(AssertionError): - assert len(definitions) == 1 - stmt = definitions[0]._definition - usages = script.usages() - inlines = [r for r in usages - if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos] - inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column), - reverse=True) - expression_list = stmt.expression_list() - # don't allow multiline refactorings for now. - assert stmt.start_pos[0] == stmt.end_pos[0] - index = stmt.start_pos[0] - 1 + assert len(definitions) == 1 + stmt = definitions[0]._definition + usages = script.usages() + inlines = [r for r in usages + if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos] + inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column), + reverse=True) + expression_list = stmt.expression_list() + # don't allow multiline refactorings for now. + assert stmt.start_pos[0] == stmt.end_pos[0] + index = stmt.start_pos[0] - 1 - line = new_lines[index] - replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1] - replace_str = replace_str.strip() - # tuples need parentheses - if expression_list and isinstance(expression_list[0], pr.Array): - arr = expression_list[0] - if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1: - replace_str = '(%s)' % replace_str + line = new_lines[index] + replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1] + replace_str = replace_str.strip() + # tuples need parentheses + if expression_list and isinstance(expression_list[0], pr.Array): + arr = expression_list[0] + if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1: + replace_str = '(%s)' % replace_str - # if it's the only assignment, remove the statement - if len(stmt.get_defined_names()) == 1: - line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:] + # if it's the only assignment, remove the statement + if len(stmt.get_defined_names()) == 1: + line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:] - dct = _rename(inlines, replace_str) - # remove the empty line - new_lines = dct[script.path][2] - if line.strip(): - new_lines[index] = line - else: - new_lines.pop(index) + dct = _rename(inlines, replace_str) + # remove the empty line + new_lines = dct[script.path][2] + if line.strip(): + new_lines[index] = line + else: + new_lines.pop(index) return Refactoring(dct) From 5328d1e70083b48c96bbcf21b37f3310a71b52cd Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 25 Sep 2017 11:04:09 +0200 Subject: [PATCH 02/35] Add a ContextSet. This is not bug free yet, but it's going to be a good abstraction for a lot of small things. --- jedi/common/__init__.py | 2 + jedi/common/context.py | 58 +++++++++++++ jedi/evaluate/__init__.py | 127 +++++++++++++++-------------- jedi/evaluate/compiled/__init__.py | 27 +++--- jedi/evaluate/context.py | 10 +-- jedi/evaluate/docstrings.py | 7 +- jedi/evaluate/finder.py | 59 ++++++-------- jedi/evaluate/imports.py | 21 ++--- jedi/evaluate/instance.py | 7 +- jedi/evaluate/iterable.py | 41 +++++----- jedi/evaluate/param.py | 2 +- jedi/evaluate/pep0484.py | 17 ++-- jedi/evaluate/precedence.py | 37 +++++---- jedi/evaluate/representation.py | 32 ++++---- 14 files changed, 257 insertions(+), 190 deletions(-) create mode 100644 jedi/common/__init__.py create mode 100644 jedi/common/context.py diff --git a/jedi/common/__init__.py b/jedi/common/__init__.py new file mode 100644 index 00000000..ba2e2f44 --- /dev/null +++ b/jedi/common/__init__.py @@ -0,0 +1,2 @@ +from jedi.common.context import ContextSet, NO_CONTEXTS, \ + iterator_to_context_set diff --git a/jedi/common/context.py b/jedi/common/context.py new file mode 100644 index 00000000..9a1f476c --- /dev/null +++ b/jedi/common/context.py @@ -0,0 +1,58 @@ +class ContextSet(object): + def __init__(self, *args): + self._set = set(args) + + @classmethod + def from_iterable(cls, iterable): + return cls.from_set(set(iterable)) + + @classmethod + def from_set(cls, set_): + self = cls() + self._set = set_ + return self + + @classmethod + def from_sets(cls, sets): + """ + Used to work with an iterable of set. + """ + aggregated = set() + for set_ in sets: + print(set_) + if isinstance(set_, ContextSet): + aggregated |= set_._set + else: + aggregated |= set_ + return cls.from_set(aggregated) + + def __or__(self, other): + return ContextSet.from_set(self._set | other._set) + + def __iter__(self): + for element in self._set: + yield element + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, ', '.join(str(s) for s in self._set)) + + def filter(self, filter_func): + return ContextSet(filter(filter_func, self._set)) + + def __getattr__(self, name): + def mapper(*args, **kwargs): + return ContextSet.from_sets( + getattr(context, name)(*args, **kwargs) + for context in self._set + ) + return mapper + + +def iterator_to_context_set(func): + def wrapper(*args, **kwargs): + return ContextSet.from_iterable(func(*args, **kwargs)) + + return wrapper + + +NO_CONTEXTS = ContextSet() diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 346f3528..7cff98d4 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -83,6 +83,7 @@ from jedi.evaluate import pep0484 from jedi.evaluate.filters import TreeNameDefinition, ParamName from jedi.evaluate.instance import AnonymousInstance, BoundMethod from jedi.evaluate.context import ContextualizedName, ContextualizedNode +from jedi.common import ContextSet, NO_CONTEXTS from jedi import parser_utils @@ -101,7 +102,7 @@ def _limit_context_infers(func): evaluator.inferred_element_counts[n] += 1 if evaluator.inferred_element_counts[n] > 300: debug.warning('In context %s there were too many inferences.', n) - return set() + return NO_CONTEXTS except KeyError: evaluator.inferred_element_counts[n] = 1 return func(evaluator, context, *args, **kwargs) @@ -163,7 +164,7 @@ class Evaluator(object): with recursion.execution_allowed(self, stmt) as allowed: if allowed or context.get_root_context() == self.BUILTINS: return self._eval_stmt(context, stmt, seek_name) - return set() + return NO_CONTEXTS #@evaluator_function_cache(default=[]) @debug.increase_indent @@ -178,11 +179,11 @@ class Evaluator(object): """ debug.dbg('eval_statement %s (%s)', stmt, seek_name) rhs = stmt.get_rhs() - types = self.eval_element(context, rhs) + context_set = self.eval_element(context, rhs) if seek_name: c_node = ContextualizedName(context, seek_name) - types = finder.check_tuple_assignments(self, c_node, types) + context_set = finder.check_tuple_assignments(self, c_node, context_set) first_operator = next(stmt.yield_operators(), None) if first_operator not in ('=', None) and first_operator.type == 'operator': @@ -194,7 +195,7 @@ class Evaluator(object): name, position=stmt.start_pos, search_global=True) for_stmt = tree.search_ancestor(stmt, 'for_stmt') - if for_stmt is not None and for_stmt.type == 'for_stmt' and types \ + if for_stmt is not None and for_stmt.type == 'for_stmt' and context_set \ and parser_utils.for_stmt_defines_one_name(for_stmt): # Iterate through result and add the values, that's possible # only in for loops without clutter, because they are @@ -208,11 +209,11 @@ class Evaluator(object): with helpers.predefine_names(context, for_stmt, dct): t = self.eval_element(context, rhs) left = precedence.calculate(self, context, left, operator, t) - types = left + context_set = left else: - types = precedence.calculate(self, context, left, operator, types) - debug.dbg('eval_statement result %s', types) - return types + context_set = precedence.calculate(self, context, left, operator, context_set) + debug.dbg('eval_statement result %s', context_set) + return context_set def eval_element(self, context, element): if isinstance(context, iterable.CompForContext): @@ -261,14 +262,14 @@ class Evaluator(object): new_name_dicts = list(original_name_dicts) for i, name_dict in enumerate(new_name_dicts): new_name_dicts[i] = name_dict.copy() - new_name_dicts[i][if_name.value] = set([definition]) + new_name_dicts[i][if_name.value] = ContextSet(definition) name_dicts += new_name_dicts else: for name_dict in name_dicts: name_dict[if_name.value] = definitions if len(name_dicts) > 1: - result = set() + result = ContextSet() for name_dict in name_dicts: with helpers.predefine_names(context, if_stmt, name_dict): result |= self._eval_element_not_cached(context, element) @@ -293,7 +294,7 @@ class Evaluator(object): return self._eval_element_not_cached(context, element) return self._eval_element_cached(context, element) - @evaluator_function_cache(default=set()) + @evaluator_function_cache(default=NO_CONTEXTS) def _eval_element_cached(self, context, element): return self._eval_element_not_cached(context, element) @@ -301,63 +302,66 @@ class Evaluator(object): @_limit_context_infers def _eval_element_not_cached(self, context, element): debug.dbg('eval_element %s@%s', element, element.start_pos) - types = set() typ = element.type if typ in ('name', 'number', 'string', 'atom'): - types = self.eval_atom(context, element) + return self.eval_atom(context, element) elif typ == 'keyword': # For False/True/None if element.value in ('False', 'True', 'None'): - types.add(compiled.builtin_from_name(self, element.value)) + return ContextSet(compiled.builtin_from_name(self, element.value)) # else: print e.g. could be evaluated like this in Python 2.7 + return NO_CONTEXTS elif typ == 'lambdef': - types = set([er.FunctionContext(self, context, element)]) + return ContextSet(er.FunctionContext(self, context, element)) elif typ == 'expr_stmt': - types = self.eval_statement(context, element) + return self.eval_statement(context, element) elif typ in ('power', 'atom_expr'): first_child = element.children[0] if not (first_child.type == 'keyword' and first_child.value == 'await'): - types = self.eval_atom(context, first_child) + context_set = self.eval_atom(context, first_child) for trailer in element.children[1:]: if trailer == '**': # has a power operation. right = self.eval_element(context, element.children[2]) - types = set(precedence.calculate(self, context, types, trailer, right)) + context_set = precedence.calculate( + self, + context, + context_set, + trailer, + right + ) break - types = self.eval_trailer(context, types, trailer) + context_set = self.eval_trailer(context, context_set, trailer) + return context_set elif typ in ('testlist_star_expr', 'testlist',): # The implicit tuple in statements. - types = set([iterable.SequenceLiteralContext(self, context, element)]) + return ContextSet(iterable.SequenceLiteralContext(self, context, element)) elif typ in ('not_test', 'factor'): - types = self.eval_element(context, element.children[-1]) + context_set = self.eval_element(context, element.children[-1]) for operator in element.children[:-1]: - types = set(precedence.factor_calculate(self, types, operator)) + context_set = precedence.factor_calculate(self, context_set, operator) + return context_set elif typ == 'test': # `x if foo else y` case. - types = (self.eval_element(context, element.children[0]) | - self.eval_element(context, element.children[-1])) + return (self.eval_element(context, element.children[0]) | + self.eval_element(context, element.children[-1])) elif typ == 'operator': # Must be an ellipsis, other operators are not evaluated. # In Python 2 ellipsis is coded as three single dot tokens, not # as one token 3 dot token. assert element.value in ('.', '...') - types = set([compiled.create(self, Ellipsis)]) + return ContextSet(compiled.create(self, Ellipsis)) elif typ == 'dotted_name': - types = self.eval_atom(context, element.children[0]) + context_set = self.eval_atom(context, element.children[0]) for next_name in element.children[2::2]: # TODO add search_global=True? - types = unite( - typ.py__getattribute__(next_name, name_context=context) - for typ in types - ) - types = types + context_set.py__getattribute__(next_name, name_context=context) + return context_set elif typ == 'eval_input': - types = self._eval_element_not_cached(context, element.children[0]) + return self._eval_element_not_cached(context, element.children[0]) elif typ == 'annassign': - types = pep0484._evaluate_for_annotation(context, element.children[1]) + return pep0484._evaluate_for_annotation(context, element.children[1]) else: - types = precedence.calculate_children(self, context, element.children) - debug.dbg('eval_element result %s', types) - return types + return precedence.calculate_children(self, context, element.children) def eval_atom(self, context, atom): """ @@ -377,18 +381,19 @@ class Evaluator(object): position=stmt.start_pos, search_global=True ) + elif isinstance(atom, tree.Literal): string = parser_utils.safe_literal_eval(atom.value) - return set([compiled.create(self, string)]) + return ContextSet(compiled.create(self, string)) else: c = atom.children if c[0].type == 'string': # Will be one string. - types = self.eval_atom(context, c[0]) + context_set = self.eval_atom(context, c[0]) for string in c[1:]: right = self.eval_atom(context, string) - types = precedence.calculate(self, context, types, '+', right) - return types + context_set = precedence.calculate(self, context, context_set, '+', right) + return context_set # Parentheses without commas are not tuples. elif c[0] == '(' and not len(c) == 2 \ and not(c[1].type == 'testlist_comp' and @@ -408,7 +413,7 @@ class Evaluator(object): pass if comp_for.type == 'comp_for': - return set([iterable.Comprehension.from_atom(self, context, atom)]) + return ContextSet(iterable.Comprehension.from_atom(self, context, atom)) # It's a dict/list/tuple literal. array_node = c[1] @@ -420,28 +425,28 @@ class Evaluator(object): context = iterable.DictLiteralContext(self, context, atom) else: context = iterable.SequenceLiteralContext(self, context, atom) - return set([context]) + return ContextSet(context) def eval_trailer(self, context, types, trailer): trailer_op, node = trailer.children[:2] if node == ')': # `arglist` is optional. node = () - new_types = set() if trailer_op == '[': - new_types |= iterable.py__getitem__(self, context, types, trailer) + return ContextSet(iterable.py__getitem__(self, context, types, trailer)) else: + context_set = ContextSet() for typ in types: debug.dbg('eval_trailer: %s in scope %s', trailer, typ) if trailer_op == '.': - new_types |= typ.py__getattribute__( + context_set |= typ.py__getattribute__( name_context=context, name_or_str=node ) elif trailer_op == '(': arguments = param.TreeArguments(self, context, node, trailer) - new_types |= self.execute(typ, arguments) - return new_types + context_set |= self.execute(typ, arguments) + return context_set @debug.increase_indent def execute(self, obj, arguments): @@ -460,11 +465,11 @@ class Evaluator(object): func = obj.py__call__ except AttributeError: debug.warning("no execution possible %s", obj) - return set() + return NO_CONTEXTS else: - types = func(arguments) - debug.dbg('execute result: %s in %s', types, obj) - return types + context_set = func(arguments) + debug.dbg('execute result: %s in %s', context_set, obj) + return context_set def goto_definitions(self, context, name): def_ = name.get_definition(import_name_always=True) @@ -509,25 +514,25 @@ class Evaluator(object): return module_names par = name.parent - typ = par.type - if typ == 'argument' and par.children[1] == '=' and par.children[0] == name: + node_type = par.type + if node_type == 'argument' and par.children[1] == '=' and par.children[0] == name: # Named param goto. trailer = par.parent if trailer.type == 'arglist': trailer = trailer.parent if trailer.type != 'classdef': if trailer.type == 'decorator': - types = self.eval_element(context, trailer.children[1]) + context_set = self.eval_element(context, trailer.children[1]) else: i = trailer.parent.children.index(trailer) to_evaluate = trailer.parent.children[:i] - types = self.eval_element(context, to_evaluate[0]) + context_set = self.eval_element(context, to_evaluate[0]) for trailer in to_evaluate[1:]: - types = self.eval_trailer(context, types, trailer) + context_set = self.eval_trailer(context, context_set, trailer) param_names = [] - for typ in types: + for context in context_set: try: - get_param_names = typ.get_param_names + get_param_names = context.get_param_names except AttributeError: pass else: @@ -535,7 +540,7 @@ class Evaluator(object): if param_name.string_name == name.value: param_names.append(param_name) return param_names - elif typ == 'dotted_name': # Is a decorator. + elif node_type == 'dotted_name': # Is a decorator. index = par.children.index(name) if index > 0: new_dotted = helpers.deep_ast_copy(par) @@ -546,7 +551,7 @@ class Evaluator(object): for value in values ) - if typ == 'trailer' and par.children[0] == '.': + if node_type == 'trailer' and par.children[0] == '.': values = helpers.evaluate_call_of_leaf(context, name, cut_own_trailer=True) return unite( value.py__getattribute__(name, name_context=context, is_goto=True) diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index cd59b6b3..091d07d4 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -15,6 +15,7 @@ from jedi.evaluate.filters import AbstractFilter, AbstractNameDefinition, \ ContextNameMixin from jedi.evaluate.context import Context, LazyKnownContext from jedi.evaluate.compiled.getattr_static import getattr_static +from jedi.common import ContextSet from . import fake @@ -83,9 +84,9 @@ class CompiledObject(Context): def py__call__(self, params): if inspect.isclass(self.obj): from jedi.evaluate.instance import CompiledInstance - return set([CompiledInstance(self.evaluator, self.parent_context, self, params)]) + return ContextSet(CompiledInstance(self.evaluator, self.parent_context, self, params)) else: - return set(self._execute_function(params)) + return ContextSet.from_iterable(self._execute_function(params)) @CheckAttribute def py__class__(self): @@ -221,9 +222,9 @@ class CompiledObject(Context): def py__getitem__(self, index): if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict): # Get rid of side effects, we won't call custom `__getitem__`s. - return set() + return ContextSet() - return set([create(self.evaluator, self.obj[index])]) + return ContextSet(create(self.evaluator, self.obj[index])) @CheckAttribute def py__iter__(self): @@ -278,7 +279,7 @@ class CompiledObject(Context): return [] # Builtins don't have imports def dict_values(self): - return set(create(self.evaluator, v) for v in self.obj.values()) + return ContextSet(create(self.evaluator, v) for v in self.obj.values()) class CompiledName(AbstractNameDefinition): @@ -301,7 +302,9 @@ class CompiledName(AbstractNameDefinition): @underscore_memoization def infer(self): module = self.parent_context.get_root_context() - return [_create_from_name(self._evaluator, module, self.parent_context, self.string_name)] + return ContextSet(_create_from_name( + self._evaluator, module, self.parent_context, self.string_name + )) class SignatureParamName(AbstractNameDefinition): @@ -318,13 +321,13 @@ class SignatureParamName(AbstractNameDefinition): def infer(self): p = self._signature_param evaluator = self.parent_context.evaluator - types = set() + contexts = ContextSet() if p.default is not p.empty: - types.add(create(evaluator, p.default)) + contexts = ContextSet(create(evaluator, p.default)) if p.annotation is not p.empty: annotation = create(evaluator, p.annotation) - types |= annotation.execute_evaluated() - return types + contexts |= annotation.execute_evaluated() + return contexts class UnresolvableParamName(AbstractNameDefinition): @@ -335,7 +338,7 @@ class UnresolvableParamName(AbstractNameDefinition): self.string_name = name def infer(self): - return set() + return ContextSet() class CompiledContextName(ContextNameMixin, AbstractNameDefinition): @@ -356,7 +359,7 @@ class EmptyCompiledName(AbstractNameDefinition): self.string_name = name def infer(self): - return [] + return ContextSet() class CompiledObjectFilter(AbstractFilter): diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index 8532cdb1..df73194f 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -1,7 +1,7 @@ from jedi._compatibility import Python3Method -from jedi.evaluate.utils import unite from parso.python.tree import ExprStmt, CompFor from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature +from jedi.common import ContextSet, NO_CONTEXTS class Context(object): @@ -111,11 +111,11 @@ class AbstractLazyContext(object): class LazyKnownContext(AbstractLazyContext): """data is a context.""" def infer(self): - return set([self.data]) + return ContextSet(self.data) class LazyKnownContexts(AbstractLazyContext): - """data is a set of contexts.""" + """data is a ContextSet.""" def infer(self): return self.data @@ -125,7 +125,7 @@ class LazyUnknownContext(AbstractLazyContext): super(LazyUnknownContext, self).__init__(None) def infer(self): - return set() + return NO_CONTEXTS class LazyTreeContext(AbstractLazyContext): @@ -155,7 +155,7 @@ def get_merged_lazy_context(lazy_contexts): class MergedLazyContexts(AbstractLazyContext): """data is a list of lazy contexts.""" def infer(self): - return unite(l.infer() for l in self.data) + return ContextSet.from_sets(l.infer() for l in self.data) class ContextualizedNode(object): diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index c552d5e3..43c950ed 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -25,6 +25,7 @@ from jedi.evaluate.utils import unite, indent_block from jedi.evaluate import context from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.iterable import SequenceLiteralContext, FakeSequence +from jedi.common import iterator_to_context_set, ContextSet, NO_CONTEXTS DOCSTRING_PARAM_PATTERNS = [ @@ -245,7 +246,7 @@ def infer_param(execution_context, param): from jedi.evaluate.instance import AnonymousInstanceFunctionExecution def eval_docstring(docstring): - return set( + return ContextSet.from_iterable( p for param_str in _search_param_in_docstr(docstring, param.name.value) for p in _evaluate_for_statement_string(module_context, param_str) @@ -253,7 +254,7 @@ def infer_param(execution_context, param): module_context = execution_context.get_root_context() func = param.get_parent_function() if func.type == 'lambdef': - return set() + return NO_CONTEXTS types = eval_docstring(execution_context.py__doc__()) if isinstance(execution_context, AnonymousInstanceFunctionExecution) and \ @@ -265,6 +266,7 @@ def infer_param(execution_context, param): @evaluator_method_cache() +@iterator_to_context_set def infer_return_types(function_context): def search_return_in_docstr(code): for p in DOCSTRING_RETURN_PATTERNS: @@ -278,4 +280,3 @@ def infer_return_types(function_context): for type_str in search_return_in_docstr(function_context.py__doc__()): for type_eval in _evaluate_for_statement_string(function_context.get_root_context(), type_str): yield type_eval - diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 38a1b357..3e5e2ef5 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -32,6 +32,7 @@ from jedi.evaluate import param from jedi.evaluate import helpers from jedi.evaluate.filters import get_global_filters, TreeNameDefinition from jedi.evaluate.context import ContextualizedName, ContextualizedNode +from jedi.common import ContextSet from jedi.parser_utils import is_scope, get_parent_scope @@ -62,7 +63,7 @@ class NameFinder(object): check = flow_analysis.reachability_check( self._context, self._context.tree_node, self._name) if check is flow_analysis.UNREACHABLE: - return set() + return ContextSet() return self._found_predefined_types types = self._names_to_types(names, attribute_lookup) @@ -158,22 +159,20 @@ class NameFinder(object): return inst.execute_function_slots(names, name) def _names_to_types(self, names, attribute_lookup): - types = set() + contexts = ContextSet.from_sets(name.infer() for name in names) - types = unite(name.infer() for name in names) - - debug.dbg('finder._names_to_types: %s -> %s', names, types) + debug.dbg('finder._names_to_types: %s -> %s', names, contexts) if not names and isinstance(self._context, AbstractInstanceContext): # handling __getattr__ / __getattribute__ return self._check_getattr(self._context) # Add isinstance and other if/assert knowledge. - if not types and isinstance(self._name, tree.Name) and \ + if not contexts and isinstance(self._name, tree.Name) and \ not isinstance(self._name_context, AbstractInstanceContext): flow_scope = self._name base_node = self._name_context.tree_node if base_node.type == 'comp_for': - return types + return contexts while True: flow_scope = get_parent_scope(flow_scope, include_flows=True) n = _check_flow_information(self._name_context, flow_scope, @@ -182,7 +181,7 @@ class NameFinder(object): return n if flow_scope == base_node: break - return types + return contexts def _name_to_types(evaluator, context, tree_name): @@ -210,6 +209,7 @@ def _name_to_types(evaluator, context, tree_name): types = pep0484.find_type_from_comment_hint_with(context, node, tree_name) if types: return types + if typ in ('for_stmt', 'comp_for'): try: types = context.predefined_names[node][tree_name.value] @@ -222,11 +222,8 @@ def _name_to_types(evaluator, context, tree_name): types = _remove_statements(evaluator, context, node, tree_name) elif typ == 'with_stmt': context_managers = context.eval_node(node.get_test_node_from_name(tree_name)) - enter_methods = unite( - context_manager.py__getattribute__('__enter__') - for context_manager in context_managers - ) - types = unite(method.execute_evaluated() for method in enter_methods) + enter_methods = context_managers.py__getattribute__('__enter__') + return enter_methods.execute_evaluated() elif typ in ('import_from', 'import_name'): types = imports.infer_import(context, tree_name) elif typ in ('funcdef', 'classdef'): @@ -262,7 +259,7 @@ def _apply_decorators(evaluator, context, node): parent_context=context, funcdef=node ) - initial = values = set([decoratee_context]) + initial = values = ContextSet(decoratee_context) for dec in reversed(node.get_decorators()): debug.dbg('decorator: %s %s', dec, values) dec_values = context.eval_node(dec.children[1]) @@ -294,20 +291,12 @@ def _remove_statements(evaluator, context, stmt, name): Due to lazy evaluation, statements like a = func; b = a; b() have to be evaluated. """ - types = set() - check_instance = None - - pep0484types = \ + pep0484_contexts = \ pep0484.find_type_from_comment_hint_assign(context, stmt, name) - if pep0484types: - return pep0484types - types |= context.eval_stmt(stmt, seek_name=name) + if pep0484_contexts: + return pep0484_contexts - if check_instance is not None: - # class renames - types = set([er.get_instance_el(evaluator, check_instance, a, True) - if isinstance(a, er.Function) else a for a in types]) - return types + return context.eval_stmt(stmt, seek_name=name) def _check_flow_information(context, flow, search_name, pos): @@ -377,26 +366,26 @@ def _check_isinstance_type(context, element, search_name): except AssertionError: return None - result = set() + context_set = ContextSet() for cls_or_tup in lazy_context_cls.infer(): if isinstance(cls_or_tup, iterable.AbstractSequence) and \ cls_or_tup.array_type == 'tuple': for lazy_context in cls_or_tup.py__iter__(): for context in lazy_context.infer(): - result |= context.execute_evaluated() + context_set |= context.execute_evaluated() else: - result |= cls_or_tup.execute_evaluated() - return result + context_set |= cls_or_tup.execute_evaluated() + return context_set -def check_tuple_assignments(evaluator, contextualized_name, types): +def check_tuple_assignments(evaluator, contextualized_name, context_set): """ Checks if tuples are assigned. """ lazy_context = None for index, node in contextualized_name.assignment_indexes(): cn = ContextualizedNode(contextualized_name.context, node) - iterated = iterable.py__iter__(evaluator, types, cn) + iterated = iterable.py__iter__(evaluator, context_set, cn) for _ in range(index + 1): try: lazy_context = next(iterated) @@ -405,6 +394,6 @@ def check_tuple_assignments(evaluator, contextualized_name, types): # would allow this loop to run for a very long time if the # index number is high. Therefore break if the loop is # finished. - return set() - types = lazy_context.infer() - return types + return ContextSet() + context_set = lazy_context.infer() + return context_set diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 12ed630a..75139d15 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -31,11 +31,12 @@ from jedi.evaluate import compiled from jedi.evaluate import analysis from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.filters import AbstractNameDefinition +from jedi.common import ContextSet, NO_CONTEXTS # This memoization is needed, because otherwise we will infinitely loop on # certain imports. -@evaluator_method_cache(default=set()) +@evaluator_method_cache(default=NO_CONTEXTS) def infer_import(context, tree_name, is_goto=False): module_context = context.get_root_context() import_node = search_ancestor(tree_name, 'import_name', 'import_from') @@ -63,7 +64,7 @@ def infer_import(context, tree_name, is_goto=False): # scopes = [NestedImportModule(module, import_node)] if not types: - return set() + return NO_CONTEXTS if from_import_name is not None: types = unite( @@ -270,7 +271,7 @@ class Importer(object): def follow(self): if not self.import_path: - return set() + return NO_CONTEXTS return self._do_import(self.import_path, self.sys_path_with_modifications()) def _do_import(self, import_path, sys_path): @@ -296,7 +297,7 @@ class Importer(object): module_name = '.'.join(import_parts) try: - return set([self._evaluator.modules[module_name]]) + return ContextSet(self._evaluator.modules[module_name]) except KeyError: pass @@ -305,7 +306,7 @@ class Importer(object): # the module cache. bases = self._do_import(import_path[:-1], sys_path) if not bases: - return set() + return NO_CONTEXTS # We can take the first element, because only the os special # case yields multiple modules, which is not important for # further imports. @@ -323,7 +324,7 @@ class Importer(object): except AttributeError: # The module is not a package. _add_error(self.module_context, import_path[-1]) - return set() + return NO_CONTEXTS else: paths = method() debug.dbg('search_module %s in paths %s', module_name, paths) @@ -340,7 +341,7 @@ class Importer(object): module_path = None if module_path is None: _add_error(self.module_context, import_path[-1]) - return set() + return NO_CONTEXTS else: parent_module = None try: @@ -356,7 +357,7 @@ class Importer(object): except ImportError: # The module is not a package. _add_error(self.module_context, import_path[-1]) - return set() + return NO_CONTEXTS code = None if is_pkg: @@ -383,10 +384,10 @@ class Importer(object): if module is None: # The file might raise an ImportError e.g. and therefore not be # importable. - return set() + return NO_CONTEXTS self._evaluator.modules[module_name] = module - return set([module]) + return ContextSet(module) def _generate_name(self, name, in_module=None): # Create a pseudo import to be able to follow them. diff --git a/jedi/evaluate/instance.py b/jedi/evaluate/instance.py index 52852d1e..f8df7a1b 100644 --- a/jedi/evaluate/instance.py +++ b/jedi/evaluate/instance.py @@ -11,6 +11,7 @@ from jedi.evaluate.param import AbstractArguments, AnonymousArguments from jedi.cache import memoize_method from jedi.evaluate import representation as er from jedi.evaluate import iterable +from jedi.common import ContextSet, iterator_to_context_set from jedi.parser_utils import get_parent_scope @@ -250,6 +251,7 @@ class CompiledInstanceName(compiled.CompiledName): super(CompiledInstanceName, self).__init__(evaluator, parent_context, name) self._instance = instance + @iterator_to_context_set def infer(self): for result_context in super(CompiledInstanceName, self).infer(): if isinstance(result_context, er.FunctionContext): @@ -311,9 +313,7 @@ class CompiledBoundMethod(compiled.CompiledObject): class InstanceNameDefinition(filters.TreeNameDefinition): def infer(self): - contexts = super(InstanceNameDefinition, self).infer() - for context in contexts: - yield context + return super(InstanceNameDefinition, self).infer() class LazyInstanceName(filters.TreeNameDefinition): @@ -331,6 +331,7 @@ class LazyInstanceName(filters.TreeNameDefinition): class LazyInstanceClassName(LazyInstanceName): + @iterator_to_context_set def infer(self): for result_context in super(LazyInstanceClassName, self).infer(): if isinstance(result_context, er.FunctionContext): diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 6e51894d..cee0879b 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -35,6 +35,7 @@ from jedi.evaluate import recursion from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.filters import DictFilter, AbstractNameDefinition, \ ParserTreeFilter +from jedi.common import ContextSet, NO_CONTEXTS from jedi.parser_utils import get_comp_fors @@ -87,7 +88,7 @@ class SpecialMethodFilter(DictFilter): # always only going to be one name. The same is true for the # inferred values. builtin_func = next(iter(filter.get(self.string_name)[0].infer())) - return set([BuiltinMethod(self.parent_context, self._callable, builtin_func)]) + return ContextSet(BuiltinMethod(self.parent_context, self._callable, builtin_func)) def __init__(self, context, dct, builtin_context): super(SpecialMethodFilter, self).__init__(dct) @@ -304,7 +305,7 @@ class ListComprehension(ArrayMixin, Comprehension): def py__getitem__(self, index): if isinstance(index, slice): - return set([self]) + return ContextSet(self) all_types = list(self.py__iter__()) return all_types[index].infer() @@ -339,11 +340,11 @@ class DictComprehension(ArrayMixin, Comprehension): @register_builtin_method('values') def _imitate_values(self): lazy_context = context.LazyKnownContexts(self.dict_values()) - return set([FakeSequence(self.evaluator, 'list', [lazy_context])]) + return ContextSet(FakeSequence(self.evaluator, 'list', [lazy_context])) @register_builtin_method('items') def _imitate_items(self): - items = set( + items = ContextSet.from_iterable( FakeSequence( self.evaluator, 'tuple' (context.LazyKnownContexts(keys), context.LazyKnownContexts(values)) @@ -385,7 +386,7 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence): # Can raise an IndexError if isinstance(index, slice): - return set([self]) + return ContextSet(self) else: return self._defining_context.eval_node(self._items()[index]) @@ -396,7 +397,7 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence): """ if self.array_type == 'dict': # Get keys. - types = set() + types = ContextSet() for k, _ in self._items(): types |= self._defining_context.eval_node(k) # We don't know which dict index comes first, therefore always @@ -470,7 +471,7 @@ class DictLiteralContext(SequenceLiteralContext): @register_builtin_method('values') def _imitate_values(self): lazy_context = context.LazyKnownContexts(self.dict_values()) - return set([FakeSequence(self.evaluator, 'list', [lazy_context])]) + return ContextSet(FakeSequence(self.evaluator, 'list', [lazy_context])) @register_builtin_method('items') def _imitate_items(self): @@ -482,7 +483,7 @@ class DictLiteralContext(SequenceLiteralContext): )) for key_node, value_node in self._items() ] - return set([FakeSequence(self.evaluator, 'list', lazy_contexts)]) + return ContextSet(FakeSequence(self.evaluator, 'list', lazy_contexts)) class _FakeArray(SequenceLiteralContext): @@ -506,7 +507,7 @@ class FakeSequence(_FakeArray): return self._context_list def py__getitem__(self, index): - return set(self._lazy_context_list[index].infer()) + return self._lazy_context_list[index].infer() def py__iter__(self): return self._lazy_context_list @@ -641,7 +642,7 @@ def py__iter__types(evaluator, types, contextualized_node=None): def py__getitem__(evaluator, context, types, trailer): from jedi.evaluate.representation import ClassContext from jedi.evaluate.instance import TreeInstance - result = set() + result = ContextSet() trailer_op, node, trailer_cl = trailer.children assert trailer_op == "[" @@ -685,7 +686,7 @@ def py__getitem__(evaluator, context, types, trailer): try: result |= getitem(index) except IndexError: - result |= py__iter__types(evaluator, set([typ])) + result |= py__iter__types(evaluator, ContextSet(typ)) except KeyError: # Must be a dict. Lists don't raise KeyErrors. result |= typ.dict_values() @@ -696,12 +697,12 @@ def check_array_additions(context, sequence): """ Just a mapper function for the internal _check_array_additions """ if sequence.array_type not in ('list', 'set'): # TODO also check for dict updates - return set() + return NO_CONTEXTS return _check_array_additions(context, sequence) -@evaluator_method_cache(default=set()) +@evaluator_method_cache(default=NO_CONTEXTS) @debug.increase_indent def _check_array_additions(context, sequence): """ @@ -716,11 +717,11 @@ def _check_array_additions(context, sequence): module_context = context.get_root_context() if not settings.dynamic_array_additions or isinstance(module_context, compiled.CompiledObject): debug.dbg('Dynamic array search aborted.', color='MAGENTA') - return set() + return ContextSet() def find_additions(context, arglist, add_name): params = list(param.TreeArguments(context.evaluator, context, arglist).unpack()) - result = set() + result = ContextSet() if add_name in ['insert']: params = params[1:] if add_name in ['append', 'add', 'insert']: @@ -728,7 +729,9 @@ def _check_array_additions(context, sequence): result.add(lazy_context) elif add_name in ['extend', 'update']: for key, lazy_context in params: - result |= set(py__iter__(context.evaluator, lazy_context.infer())) + result |= ContextSet.from_iterable( + py__iter__(context.evaluator, lazy_context.infer()) + ) return result temp_param_add, settings.dynamic_params_for_other_modules = \ @@ -737,7 +740,7 @@ def _check_array_additions(context, sequence): is_list = sequence.name.string_name == 'list' search_names = (['append', 'extend', 'insert'] if is_list else ['add', 'update']) - added_types = set() + added_types = NO_CONTEXTS() for add_name in search_names: try: possible_names = module_context.tree_node.get_used_names()[add_name] @@ -870,7 +873,7 @@ def create_index_types(evaluator, context, index): """ if index == ':': # Like array[:] - return set([Slice(context, None, None, None)]) + return ContextSet(Slice(context, None, None, None)) elif index.type == 'subscript' and not index.children[0] == '.': # subscript basically implies a slice operation, except for Python 2's @@ -888,7 +891,7 @@ def create_index_types(evaluator, context, index): result.append(el) result += [None] * (3 - len(result)) - return set([Slice(context, *result)]) + return ContextSet(Slice(context, *result)) # No slices return context.eval_node(index) diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index 5490df1f..04d93d42 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -237,7 +237,7 @@ class ExecutedParam(object): pep0484_hints = pep0484.infer_param(self._execution_context, self._param_node) doc_params = docstrings.infer_param(self._execution_context, self._param_node) if pep0484_hints or doc_params: - return list(set(pep0484_hints) | set(doc_params)) + return pep0484_hints | doc_params return self._lazy_context.infer() diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index f9a5c5ed..8e741fb0 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -19,7 +19,6 @@ x support for type hint comments for functions, `# type: (int, str) -> int`. See comment from Guido https://github.com/davidhalter/jedi/issues/662 """ -import itertools import os import re @@ -30,6 +29,7 @@ from jedi.evaluate.utils import unite from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate import compiled from jedi.evaluate.context import LazyTreeContext +from jedi.common import NO_CONTEXTS from jedi import debug from jedi import _compatibility from jedi import parser_utils @@ -42,16 +42,15 @@ def _evaluate_for_annotation(context, annotation, index=None): and we're interested in that index """ if annotation is not None: - definitions = context.eval_node( - _fix_forward_reference(context, annotation)) + context_set = context.eval_node(_fix_forward_reference(context, annotation)) if index is not None: - definitions = list(itertools.chain.from_iterable( - definition.py__getitem__(index) for definition in definitions - if definition.array_type == 'tuple' and - len(list(definition.py__iter__())) >= index)) - return unite(d.execute_evaluated() for d in definitions) + context_set = context_set.filter( + lambda context: context.array_type == 'tuple' \ + and len(list(context.py__iter__())) >= index + ).py__getitem__(index) + return context_set.execute_evaluated() else: - return set() + return NO_CONTEXTS def _fix_forward_reference(context, node): diff --git a/jedi/evaluate/precedence.py b/jedi/evaluate/precedence.py index 44335217..5d6aef13 100644 --- a/jedi/evaluate/precedence.py +++ b/jedi/evaluate/precedence.py @@ -7,6 +7,7 @@ from jedi._compatibility import unicode from jedi import debug from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name from jedi.evaluate import analysis +from jedi.common import ContextSet # Maps Python syntax to the operator module. COMPARISON_OPERATORS = { @@ -33,7 +34,7 @@ def literals_to_types(evaluator, result): new_result |= cls.execute_evaluated() else: new_result.add(typ) - return new_result + return ContextSet.from_set(new_result) def calculate_children(evaluator, context, children): @@ -49,7 +50,7 @@ def calculate_children(evaluator, context, children): # handle lazy evaluation of and/or here. if operator in ('and', 'or'): - left_bools = set([left.py__bool__() for left in types]) + left_bools = ContextSet(left.py__bool__() for left in types) if left_bools == set([True]): if operator == 'and': types = context.eval_node(right) @@ -65,22 +66,22 @@ def calculate_children(evaluator, context, children): def calculate(evaluator, context, left_result, operator, right_result): - result = set() if not left_result or not right_result: # illegal slices e.g. cause left/right_result to be None result = (left_result or set()) | (right_result or set()) - result = literals_to_types(evaluator, result) + return literals_to_types(evaluator, result) else: # I don't think there's a reasonable chance that a string # operation is still correct, once we pass something like six # objects. if len(left_result) * len(right_result) > 6: - result = literals_to_types(evaluator, left_result | right_result) + return literals_to_types(evaluator, left_result | right_result) else: - for left in left_result: - for right in right_result: - result |= _element_calculate(evaluator, context, left, operator, right) - return result + return ContextSet.from_sets( + _element_calculate(evaluator, context, left, operator, right) + for left in left_result + for right in right_result + ) def factor_calculate(evaluator, types, operator): @@ -131,21 +132,21 @@ def _element_calculate(evaluator, context, left, operator, right): if operator == '*': # for iterables, ignore * operations if isinstance(left, iterable.AbstractSequence) or is_string(left): - return set([left]) + return ContextSet(left) elif isinstance(right, iterable.AbstractSequence) or is_string(right): - return set([right]) + return ContextSet(right) elif operator == '+': if l_is_num and r_is_num or is_string(left) and is_string(right): - return set([create(evaluator, left.obj + right.obj)]) + return ContextSet(create(evaluator, left.obj + right.obj)) elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right): - return set([iterable.MergedArray(evaluator, (left, right))]) + return ContextSet(iterable.MergedArray(evaluator, (left, right))) elif operator == '-': if l_is_num and r_is_num: - return set([create(evaluator, left.obj - right.obj)]) + return ContextSet(create(evaluator, left.obj - right.obj)) elif operator == '%': # With strings and numbers the left type typically remains. Except for # `int() % float()`. - return set([left]) + return ContextSet(left) elif operator in COMPARISON_OPERATORS: operation = COMPARISON_OPERATORS[operator] if isinstance(left, CompiledObject) and isinstance(right, CompiledObject): @@ -157,9 +158,9 @@ def _element_calculate(evaluator, context, left, operator, right): result = operation(left, right) except TypeError: # Could be True or False. - return set([create(evaluator, True), create(evaluator, False)]) + return ContextSet(create(evaluator, True), create(evaluator, False)) else: - return set([create(evaluator, result)]) + return ContextSet(create(evaluator, result)) elif operator == 'in': return set() @@ -175,4 +176,4 @@ def _element_calculate(evaluator, context, left, operator, right): analysis.add(context, 'type-error-operation', operator, message % (left, right)) - return set([left, right]) + return ContextSet(left, right) diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 8a0d616c..8ccfc7b2 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -64,6 +64,7 @@ from jedi.evaluate.filters import ParserTreeFilter, FunctionExecutionFilter, \ ContextNameMixin from jedi.evaluate import context from jedi.evaluate.context import ContextualizedNode +from jedi.common import NO_CONTEXTS, ContextSet, iterator_to_context_set from jedi import parser_utils from jedi.evaluate.parser_cache import get_yield_exprs @@ -83,6 +84,7 @@ class ClassName(TreeNameDefinition): super(ClassName, self).__init__(parent_context, tree_name) self._name_context = name_context + @iterator_to_context_set def infer(self): # TODO this _name_to_types might get refactored and be a part of the # parent class. Once it is, we can probably just overwrite method to @@ -162,7 +164,7 @@ class ClassContext(use_metaclass(CachedMetaClass, context.TreeContext)): def py__call__(self, params): from jedi.evaluate.instance import TreeInstance - return set([TreeInstance(self.evaluator, self.parent_context, self, params)]) + return ContextSet(TreeInstance(self.evaluator, self.parent_context, self, params)) def py__class__(self): return compiled.create(self.evaluator, type) @@ -227,7 +229,7 @@ class LambdaName(AbstractNameDefinition): return self._lambda_context.tree_node.start_pos def infer(self): - return set([self._lambda_context]) + return ContextSet(self._lambda_context) class FunctionContext(use_metaclass(CachedMetaClass, context.TreeContext)): @@ -260,7 +262,7 @@ class FunctionContext(use_metaclass(CachedMetaClass, context.TreeContext)): """ yield_exprs = get_yield_exprs(self.evaluator, self.tree_node) if yield_exprs: - return set([iterable.Generator(self.evaluator, function_execution)]) + return ContextSet(iterable.Generator(self.evaluator, function_execution)) else: return function_execution.get_return_values() @@ -312,7 +314,7 @@ class FunctionExecutionContext(context.TreeContext): self.tree_node = function_context.tree_node self.var_args = var_args - @evaluator_method_cache(default=set()) + @evaluator_method_cache(default=NO_CONTEXTS) @recursion.execution_recursion_decorator() def get_return_values(self, check_yields=False): funcdef = self.tree_node @@ -320,12 +322,12 @@ class FunctionExecutionContext(context.TreeContext): return self.evaluator.eval_element(self, funcdef.children[-1]) if check_yields: - types = set() + context_set = NO_CONTEXTS returns = get_yield_exprs(self.evaluator, funcdef) else: returns = funcdef.iter_return_stmts() - types = set(docstrings.infer_return_types(self.function_context)) - types |= set(pep0484.infer_return_types(self.function_context)) + context_set = docstrings.infer_return_types(self.function_context) + context_set |= pep0484.infer_return_types(self.function_context) for r in returns: check = flow_analysis.reachability_check(self, funcdef, r) @@ -333,18 +335,18 @@ class FunctionExecutionContext(context.TreeContext): debug.dbg('Return unreachable: %s', r) else: if check_yields: - types |= set(self._eval_yield(r)) + context_set |= ContextSet(self._eval_yield(r)) else: try: children = r.children except AttributeError: - types.add(compiled.create(self.evaluator, None)) + context_set |= ContextSet(compiled.create(self.evaluator, None)) else: - types |= self.eval_node(children[1]) + context_set |= self.eval_node(children[1]) if check is flow_analysis.REACHABLE: debug.dbg('Return reachable: %s', r) break - return types + return context_set def _eval_yield(self, yield_expr): if yield_expr.type == 'keyword': @@ -430,8 +432,10 @@ class ModuleAttributeName(AbstractNameDefinition): self.string_name = string_name def infer(self): - return compiled.create(self.parent_context.evaluator, str).execute( - param.ValuesArguments([]) + return ContextSet( + compiled.create(self.parent_context.evaluator, str).execute( + param.ValuesArguments([]) + ) ) @@ -628,7 +632,7 @@ class ImplicitNSName(AbstractNameDefinition): self.string_name = string_name def infer(self): - return [] + return NO_CONTEXTS def get_root_context(self): return self.implicit_ns_context From 921d1008f220a428046798bd285d60c50a71e8ab Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 25 Sep 2017 11:10:09 +0200 Subject: [PATCH 03/35] First tests are now passing. --- jedi/common/context.py | 4 +++- jedi/evaluate/filters.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/jedi/common/context.py b/jedi/common/context.py index 9a1f476c..b29b7c2f 100644 --- a/jedi/common/context.py +++ b/jedi/common/context.py @@ -19,7 +19,6 @@ class ContextSet(object): """ aggregated = set() for set_ in sets: - print(set_) if isinstance(set_, ContextSet): aggregated |= set_._set else: @@ -33,6 +32,9 @@ class ContextSet(object): for element in self._set: yield element + def __bool__(self): + return bool(self._set) + def __repr__(self): return '%s(%s)' % (self.__class__.__name__, ', '.join(str(s) for s in self._set)) diff --git a/jedi/evaluate/filters.py b/jedi/evaluate/filters.py index 004aec2b..c94f9758 100644 --- a/jedi/evaluate/filters.py +++ b/jedi/evaluate/filters.py @@ -7,6 +7,7 @@ from abc import abstractmethod from parso.tree import search_ancestor from jedi.evaluate import flow_analysis from jedi.evaluate.utils import to_list, unite +from jedi.common import ContextSet from jedi.parser_utils import get_parent_scope @@ -64,7 +65,7 @@ class AbstractTreeName(AbstractNameDefinition): class ContextNameMixin(object): def infer(self): - return set([self._context]) + return ContextSet(self._context) def get_root_context(self): if self.parent_context is None: @@ -128,7 +129,7 @@ class AnonymousInstanceParamName(ParamName): if param_node.position_index == 0: # This is a speed optimization, to return the self param (because # it's known). This only affects anonymous instances. - return set([self.parent_context.instance]) + return ContextSet(self.parent_context.instance) else: return self.get_param().infer() From 174eff58758a6ee57add114ce6fc5f735afbc19a Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 25 Sep 2017 23:08:59 +0200 Subject: [PATCH 04/35] Replace a lot more of empty sets and unite calls. --- jedi/common/context.py | 3 +++ jedi/evaluate/__init__.py | 2 +- jedi/evaluate/docstrings.py | 10 ++++++-- jedi/evaluate/dynamic.py | 7 +++--- jedi/evaluate/filters.py | 6 ++--- jedi/evaluate/finder.py | 8 ++----- jedi/evaluate/imports.py | 13 ++++------- jedi/evaluate/instance.py | 11 ++++----- jedi/evaluate/iterable.py | 17 +++++++------- jedi/evaluate/param.py | 3 ++- jedi/evaluate/pep0484.py | 5 ++-- jedi/evaluate/precedence.py | 4 ++-- jedi/evaluate/recursion.py | 3 ++- jedi/evaluate/stdlib.py | 46 +++++++++++++++++++------------------ 14 files changed, 72 insertions(+), 66 deletions(-) diff --git a/jedi/common/context.py b/jedi/common/context.py index b29b7c2f..95c4671b 100644 --- a/jedi/common/context.py +++ b/jedi/common/context.py @@ -35,6 +35,9 @@ class ContextSet(object): def __bool__(self): return bool(self._set) + def __len__(self): + return len(self._set) + def __repr__(self): return '%s(%s)' % (self.__class__.__name__, ', '.join(str(s) for s in self._set)) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 7cff98d4..35c4256e 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -433,7 +433,7 @@ class Evaluator(object): node = () if trailer_op == '[': - return ContextSet(iterable.py__getitem__(self, context, types, trailer)) + return iterable.py__getitem__(self, context, types, trailer) else: context_set = ContextSet() for typ in types: diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index 43c950ed..ea471811 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -223,7 +223,10 @@ def _execute_types_in_stmt(module_context, stmt): contain is executed. (Used as type information). """ definitions = module_context.eval_node(stmt) - return unite(_execute_array_values(module_context.evaluator, d) for d in definitions) + return ContextSet.from_sets( + _execute_array_values(module_context.evaluator, d) + for d in definitions + ) def _execute_array_values(evaluator, array): @@ -234,7 +237,10 @@ def _execute_array_values(evaluator, array): if isinstance(array, SequenceLiteralContext): values = [] for lazy_context in array.py__iter__(): - objects = unite(_execute_array_values(evaluator, typ) for typ in lazy_context.infer()) + objects = ContextSet.from_sets( + _execute_array_values(evaluator, typ) + for typ in lazy_context.infer() + ) values.append(context.LazyKnownContexts(objects)) return set([FakeSequence(evaluator, array.array_type, values)]) else: diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index fb179b58..946d81e2 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -24,7 +24,8 @@ from jedi.evaluate.cache import evaluator_function_cache from jedi.evaluate import imports from jedi.evaluate.param import TreeArguments, create_default_params from jedi.evaluate.helpers import is_stdlib_path -from jedi.evaluate.utils import to_list, unite +from jedi.evaluate.utils import to_list +from jedi.common import ContextSet from jedi.parser_utils import get_parent_scope @@ -50,7 +51,7 @@ class MergedExecutedParams(object): self._executed_params = executed_params def infer(self): - return unite(p.infer() for p in self._executed_params) + return ContextSet.from_sets(p.infer() for p in self._executed_params) @debug.increase_indent @@ -103,7 +104,7 @@ def search_params(evaluator, execution_context, funcdef): evaluator.dynamic_params_depth -= 1 -@evaluator_function_cache(default=[]) +@evaluator_function_cache(default=None) @to_list def _search_function_executions(evaluator, module_context, funcdef): """ diff --git a/jedi/evaluate/filters.py b/jedi/evaluate/filters.py index c94f9758..e3edb1eb 100644 --- a/jedi/evaluate/filters.py +++ b/jedi/evaluate/filters.py @@ -6,9 +6,9 @@ from abc import abstractmethod from parso.tree import search_ancestor from jedi.evaluate import flow_analysis -from jedi.evaluate.utils import to_list, unite from jedi.common import ContextSet from jedi.parser_utils import get_parent_scope +from jedi.evaluate.utils import to_list class AbstractNameDefinition(object): @@ -36,10 +36,10 @@ class AbstractNameDefinition(object): return '<%s: %s@%s>' % (self.__class__.__name__, self.string_name, self.start_pos) def execute(self, arguments): - return unite(context.execute(arguments) for context in self.infer()) + return self.infer().execute(arguments) def execute_evaluated(self, *args, **kwargs): - return unite(context.execute_evaluated(*args, **kwargs) for context in self.infer()) + return self.infer().execute_evaluated(*args, **kwargs) @property def api_type(self): diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 3e5e2ef5..15d45108 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -233,10 +233,7 @@ def _name_to_types(evaluator, context, tree_name): # TODO check for types that are not classes and add it to # the static analysis report. exceptions = context.eval_node(tree_name.get_previous_sibling().get_previous_sibling()) - types = unite( - evaluator.execute(t, param.ValuesArguments([])) - for t in exceptions - ) + types = exceptions.execute_evaluated() else: raise ValueError("Should not happen.") return types @@ -274,8 +271,7 @@ def _apply_decorators(evaluator, context, node): debug.warning('decorator not found: %s on %s', dec, node) return initial - values = unite(dec_value.execute(param.ValuesArguments([values])) - for dec_value in dec_values) + values = dec_values.execute(param.ValuesArguments([values])) if not len(values): debug.warning('not possible to resolve wrappers found %s', node) return initial diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 75139d15..5d69e931 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -24,7 +24,6 @@ from parso import python_bytes_to_unicode from jedi._compatibility import find_module, unicode, ImplicitNSInfo from jedi import debug from jedi import settings -from jedi.evaluate.utils import unite from jedi.evaluate import sys_path from jedi.evaluate import helpers from jedi.evaluate import compiled @@ -67,13 +66,11 @@ def infer_import(context, tree_name, is_goto=False): return NO_CONTEXTS if from_import_name is not None: - types = unite( - t.py__getattribute__( - from_import_name, - name_context=context, - is_goto=is_goto, - analysis_errors=False - ) for t in types + types = types.py__getattribute__( + from_import_name, + name_context=context, + is_goto=is_goto, + analysis_errors=False ) if not types: diff --git a/jedi/evaluate/instance.py b/jedi/evaluate/instance.py index f8df7a1b..f1116707 100644 --- a/jedi/evaluate/instance.py +++ b/jedi/evaluate/instance.py @@ -1,7 +1,6 @@ from abc import abstractproperty from jedi._compatibility import is_py3 -from jedi.evaluate.utils import unite from jedi import debug from jedi.evaluate import compiled from jedi.evaluate import filters @@ -11,7 +10,7 @@ from jedi.evaluate.param import AbstractArguments, AnonymousArguments from jedi.cache import memoize_method from jedi.evaluate import representation as er from jedi.evaluate import iterable -from jedi.common import ContextSet, iterator_to_context_set +from jedi.common import ContextSet, iterator_to_context_set, NO_CONTEXTS from jedi.parser_utils import get_parent_scope @@ -59,7 +58,7 @@ class AbstractInstanceContext(Context): raise AttributeError def execute(arguments): - return unite(name.execute(arguments) for name in names) + return ContextSet.from_sets(name.execute(arguments) for name in names) return execute @@ -81,7 +80,7 @@ class AbstractInstanceContext(Context): return [] def execute_function_slots(self, names, *evaluated_args): - return unite( + return ContextSet.from_sets( name.execute_evaluated(*evaluated_args) for name in names ) @@ -97,7 +96,7 @@ class AbstractInstanceContext(Context): none_obj = compiled.create(self.evaluator, None) return self.execute_function_slots(names, none_obj, obj) else: - return set([self]) + return ContextSet(self) def get_filters(self, search_global=None, until_position=None, origin_scope=None, include_self_names=True): @@ -123,7 +122,7 @@ class AbstractInstanceContext(Context): names = self.get_function_slot_names('__getitem__') except KeyError: debug.warning('No __getitem__, cannot access the array.') - return set() + return NO_CONTEXTS else: index_obj = compiled.create(self.evaluator, index) return self.execute_function_slots(names, index_obj) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index cee0879b..dcc2b939 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -144,7 +144,7 @@ class GeneratorMixin(object): @register_builtin_method('__next__', python_version_match=3) def py__next__(self): # TODO add TypeError if params are given. - return unite(lazy_context.infer() for lazy_context in self.py__iter__()) + return ContextSet.from_sets(lazy_context.infer() for lazy_context in self.py__iter__()) def get_filters(self, search_global, until_position=None, origin_scope=None): gen_obj = compiled.get_special_object(self.evaluator, 'GENERATOR_OBJECT') @@ -297,7 +297,8 @@ class ArrayMixin(object): return self.evaluator.BUILTINS def dict_values(self): - return unite(self._defining_context.eval_node(v) for k, v in self._items()) + return ContextSet.from_sets(self._defining_context.eval_node(v) + for k, v in self._items()) class ListComprehension(ArrayMixin, Comprehension): @@ -335,7 +336,7 @@ class DictComprehension(ArrayMixin, Comprehension): return self.dict_values() def dict_values(self): - return unite(values for keys, values in self._iterate()) + return ContextSet.from_sets(values for keys, values in self._iterate()) @register_builtin_method('values') def _imitate_values(self): @@ -414,7 +415,7 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence): def _values(self): """Returns a list of a list of node.""" if self.array_type == 'dict': - return unite(v for k, v in self._items()) + return ContextSet.from_sets(v for k, v in self._items()) else: return self._items() @@ -532,7 +533,7 @@ class FakeDict(_FakeArray): return self._dct[index].infer() def dict_values(self): - return unite(lazy_context.infer() for lazy_context in self._dct.values()) + return ContextSet.from_sets(lazy_context.infer() for lazy_context in self._dct.values()) def _items(self): raise DeprecationWarning @@ -555,7 +556,7 @@ class MergedArray(_FakeArray): yield lazy_context def py__getitem__(self, index): - return unite(lazy_context.infer() for lazy_context in self.py__iter__()) + return ContextSet.from_sets(lazy_context.infer() for lazy_context in self.py__iter__()) def _items(self): for array in self._arrays: @@ -633,7 +634,7 @@ def py__iter__types(evaluator, types, contextualized_node=None): Calls `py__iter__`, but ignores the ordering in the end and just returns all types that it contains. """ - return unite( + return ContextSet.from_sets( lazy_context.infer() for lazy_context in py__iter__(evaluator, types, contextualized_node) ) @@ -740,7 +741,7 @@ def _check_array_additions(context, sequence): is_list = sequence.name.string_name == 'list' search_names = (['append', 'extend', 'insert'] if is_list else ['add', 'update']) - added_types = NO_CONTEXTS() + added_types = NO_CONTEXTS for add_name in search_names: try: possible_names = module_context.tree_node.get_used_names()[add_name] diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index 04d93d42..a914d81a 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -10,6 +10,7 @@ from jedi.evaluate import context from jedi.evaluate import docstrings from jedi.evaluate import pep0484 from jedi.evaluate.filters import ParamName +from jedi.common import NO_CONTEXTS def add_argument_issue(parent_context, error_name, lazy_context, message): @@ -51,7 +52,7 @@ class AbstractArguments(): debug.warning('TypeError: %s expected at least %s arguments, got %s', name, len(parameters), i) raise ValueError - values = set() if argument is None else argument.infer() + values = NO_CONTEXTS if argument is None else argument.infer() if not values and not optional: # For the stdlib we always want values. If we don't get them, diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 8e741fb0..0fa7ca41 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -25,11 +25,10 @@ import re from parso import ParserSyntaxError from parso.python import tree -from jedi.evaluate.utils import unite from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate import compiled from jedi.evaluate.context import LazyTreeContext -from jedi.common import NO_CONTEXTS +from jedi.common import NO_CONTEXTS, ContextSet from jedi import debug from jedi import _compatibility from jedi import parser_utils @@ -146,7 +145,7 @@ def py__getitem__(context, typ, node): if type_name in ("Union", '_Union'): # In Python 3.6 it's still called typing.Union but it's an instance # called _Union. - return unite(context.eval_node(node) for node in nodes) + return ContextSet.from_sets(context.eval_node(node) for node in nodes) if type_name in ("Optional", '_Optional'): # Here we have the same issue like in Union. Therefore we also need to # check for the instance typing._Optional (Python 3.6). diff --git a/jedi/evaluate/precedence.py b/jedi/evaluate/precedence.py index 5d6aef13..72b7a432 100644 --- a/jedi/evaluate/precedence.py +++ b/jedi/evaluate/precedence.py @@ -7,7 +7,7 @@ from jedi._compatibility import unicode from jedi import debug from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name from jedi.evaluate import analysis -from jedi.common import ContextSet +from jedi.common import ContextSet, NO_CONTEXTS # Maps Python syntax to the operator module. COMPARISON_OPERATORS = { @@ -162,7 +162,7 @@ def _element_calculate(evaluator, context, left, operator, right): else: return ContextSet(create(evaluator, result)) elif operator == 'in': - return set() + return NO_CONTEXTS def check(obj): """Checks if a Jedi object is either a float or an int.""" diff --git a/jedi/evaluate/recursion.py b/jedi/evaluate/recursion.py index 85499496..ebc71f09 100644 --- a/jedi/evaluate/recursion.py +++ b/jedi/evaluate/recursion.py @@ -29,6 +29,7 @@ therefore the quality might not always be maximal. from contextlib import contextmanager from jedi import debug +from jedi.common import NO_CONTEXTS recursion_limit = 15 @@ -71,7 +72,7 @@ def execution_allowed(evaluator, node): pushed_nodes.pop() -def execution_recursion_decorator(default=set()): +def execution_recursion_decorator(default=NO_CONTEXTS): def decorator(func): def wrapper(execution, **kwargs): detector = execution.evaluator.execution_recursion_detector diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index f2878e71..053f2b9c 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -24,6 +24,7 @@ from jedi.evaluate import precedence from jedi.evaluate import param from jedi.evaluate import analysis from jedi.evaluate.context import LazyTreeContext, ContextualizedNode +from jedi.common import NO_CONTEXTS, ContextSet # Now this is all part of fake tuples in Jedi. However super doesn't work on # __init__ and __new__ doesn't work at all. So adding this to nametuples is @@ -77,7 +78,7 @@ def _follow_param(evaluator, arguments, index): try: key, lazy_context = list(arguments.unpack())[index] except IndexError: - return set() + return NO_CONTEXTS else: return lazy_context.infer() @@ -109,7 +110,7 @@ def argument_clinic(string, want_obj=False, want_context=False, want_arguments=F try: lst = list(arguments.eval_argument_clinic(clinic_args)) except ValueError: - return set() + return NO_CONTEXTS else: kwargs = {} if want_context: @@ -137,15 +138,16 @@ def builtins_next(evaluator, iterators, defaults): else: name = '__next__' - types = set() + context_set = NO_CONTEXTS for iterator in iterators: if isinstance(iterator, AbstractInstanceContext): - for filter in iterator.get_filters(include_self_names=True): - for n in filter.get(name): - for context in n.infer(): - types |= context.execute_evaluated() - if types: - return types + context_set = ContextSet.from_sets( + n.infer() + for filter in iterator.get_filters(include_self_names=True) + for n in filter.get(name) + ).execute_evaluated() + if context_set: + return context_set return defaults @@ -159,16 +161,16 @@ def builtins_getattr(evaluator, objects, names, defaults=None): else: debug.warning('getattr called without str') continue - return set() + return NO_CONTEXTS @argument_clinic('object[, bases, dict], /') def builtins_type(evaluator, objects, bases, dicts): if bases or dicts: # It's a type creation... maybe someday... - return set() + return NO_CONTEXTS else: - return set([o.py__class__() for o in objects]) + return ContextSet.from_iterable(o.py__class__() for o in objects) class SuperInstance(AbstractInstanceContext): @@ -185,7 +187,7 @@ def builtins_super(evaluator, types, objects, context): AnonymousInstanceFunctionExecution)): su = context.instance.py__class__().py__bases__() return unite(context.execute_evaluated() for context in su[0].infer()) - return set() + return NO_CONTEXTS @argument_clinic('sequence, /', want_obj=True, want_arguments=True) @@ -207,12 +209,12 @@ def builtins_reversed(evaluator, sequences, obj, arguments): # just returned the result directly. seq = iterable.FakeSequence(evaluator, 'list', rev) arguments = param.ValuesArguments([[seq]]) - return set([CompiledInstance(evaluator, evaluator.BUILTINS, obj, arguments)]) + return ContextSet(CompiledInstance(evaluator, evaluator.BUILTINS, obj, arguments)) @argument_clinic('obj, type, /', want_arguments=True) def builtins_isinstance(evaluator, objects, types, arguments): - bool_results = set([]) + bool_results = set() for o in objects: try: mro_func = o.py__class__().py__mro__ @@ -220,7 +222,7 @@ def builtins_isinstance(evaluator, objects, types, arguments): # This is temporary. Everything should have a class attribute in # Python?! Maybe we'll leave it here, because some numpy objects or # whatever might not. - return set([compiled.create(True), compiled.create(False)]) + return ContextSet(compiled.create(True), compiled.create(False)) mro = mro_func() @@ -244,7 +246,7 @@ def builtins_isinstance(evaluator, objects, types, arguments): 'not %s.' % cls_or_tup analysis.add(lazy_context._context, 'type-error-isinstance', node, message) - return set(compiled.create(evaluator, x) for x in bool_results) + return ContextSet.from_iterable(compiled.create(evaluator, x) for x in bool_results) def collections_namedtuple(evaluator, obj, arguments): @@ -259,7 +261,7 @@ def collections_namedtuple(evaluator, obj, arguments): """ # Namedtuples are not supported on Python 2.6 if not hasattr(collections, '_class_template'): - return set() + return NO_CONTEXTS # Process arguments # TODO here we only use one of the types, we should use all. @@ -274,7 +276,7 @@ def collections_namedtuple(evaluator, obj, arguments): for v in lazy_context.infer() if hasattr(v, 'obj') ] else: - return set() + return NO_CONTEXTS base = collections._class_template base += _NAMEDTUPLE_INIT @@ -293,7 +295,7 @@ def collections_namedtuple(evaluator, obj, arguments): module = evaluator.grammar.parse(source) generated_class = next(module.iter_classdefs()) parent_context = er.ModuleContext(evaluator, module, '') - return set([er.ClassContext(evaluator, generated_class, parent_context)]) + return ContextSet(er.ClassContext(evaluator, generated_class, parent_context)) @argument_clinic('first, /') @@ -314,8 +316,8 @@ _implemented = { 'deepcopy': _return_first_param, }, 'json': { - 'load': lambda *args: set(), - 'loads': lambda *args: set(), + 'load': lambda *args: NO_CONTEXTS, + 'loads': lambda *args: NO_CONTEXTS, }, 'collections': { 'namedtuple': collections_namedtuple, From 592f2dac9576cfa5b78656443d4733b91e40bd25 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 26 Sep 2017 16:29:07 +0200 Subject: [PATCH 05/35] A lot more fixes for tests. --- jedi/common/context.py | 2 +- jedi/evaluate/__init__.py | 3 ++- jedi/evaluate/context.py | 2 +- jedi/evaluate/iterable.py | 10 +++++++--- jedi/evaluate/precedence.py | 8 ++++---- jedi/evaluate/representation.py | 13 ++++++------- jedi/evaluate/stdlib.py | 4 ++-- 7 files changed, 23 insertions(+), 19 deletions(-) diff --git a/jedi/common/context.py b/jedi/common/context.py index 95c4671b..514a6ea3 100644 --- a/jedi/common/context.py +++ b/jedi/common/context.py @@ -42,7 +42,7 @@ class ContextSet(object): return '%s(%s)' % (self.__class__.__name__, ', '.join(str(s) for s in self._set)) def filter(self, filter_func): - return ContextSet(filter(filter_func, self._set)) + return ContextSet.from_iterable(filter(filter_func, self._set)) def __getattr__(self, name): def mapper(*args, **kwargs): diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 35c4256e..02b30eab 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -331,7 +331,8 @@ class Evaluator(object): ) break context_set = self.eval_trailer(context, context_set, trailer) - return context_set + return context_set + return NO_CONTEXTS elif typ in ('testlist_star_expr', 'testlist',): # The implicit tuple in statements. return ContextSet(iterable.SequenceLiteralContext(self, context, element)) diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index df73194f..7ff57d5d 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -40,7 +40,7 @@ class Context(object): Execute a function with already executed arguments. """ from jedi.evaluate.param import ValuesArguments - arguments = ValuesArguments([[value] for value in value_list]) + arguments = ValuesArguments([ContextSet(value) for value in value_list]) return self.execute(arguments) def eval_node(self, node): diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index dcc2b939..cbab52c6 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -297,8 +297,10 @@ class ArrayMixin(object): return self.evaluator.BUILTINS def dict_values(self): - return ContextSet.from_sets(self._defining_context.eval_node(v) - for k, v in self._items()) + return ContextSet.from_sets( + self._defining_context.eval_node(v) + for k, v in self._items() + ) class ListComprehension(ArrayMixin, Comprehension): @@ -649,6 +651,8 @@ def py__getitem__(evaluator, context, types, trailer): assert trailer_op == "[" assert trailer_cl == "]" + # TODO It's kind of stupid to cast this from a context set to a set. + types = set(types) # special case: PEP0484 typing module, see # https://github.com/davidhalter/jedi/issues/663 for typ in list(types): @@ -795,7 +799,7 @@ def get_dynamic_array_instance(instance): ai = _ArrayInstance(instance) from jedi.evaluate import param - return param.ValuesArguments([[ai]]) + return param.ValuesArguments([ContextSet(ai)]) class _ArrayInstance(object): diff --git a/jedi/evaluate/precedence.py b/jedi/evaluate/precedence.py index 72b7a432..abc73589 100644 --- a/jedi/evaluate/precedence.py +++ b/jedi/evaluate/precedence.py @@ -25,7 +25,7 @@ COMPARISON_OPERATORS = { def literals_to_types(evaluator, result): # Changes literals ('a', 1, 1.0, etc) to its type instances (str(), # int(), float(), etc). - new_result = set() + new_result = NO_CONTEXTS for typ in result: if is_literal(typ): # Literals are only valid as long as the operations are @@ -33,8 +33,8 @@ def literals_to_types(evaluator, result): cls = builtin_from_name(evaluator, typ.name.string_name) new_result |= cls.execute_evaluated() else: - new_result.add(typ) - return ContextSet.from_set(new_result) + new_result |= ContextSet(typ) + return new_result def calculate_children(evaluator, context, children): @@ -68,7 +68,7 @@ def calculate_children(evaluator, context, children): def calculate(evaluator, context, left_result, operator, right_result): if not left_result or not right_result: # illegal slices e.g. cause left/right_result to be None - result = (left_result or set()) | (right_result or set()) + result = (left_result or NO_CONTEXTS) | (right_result or NO_CONTEXTS) return literals_to_types(evaluator, result) else: # I don't think there's a reasonable chance that a string diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 8ccfc7b2..c427b786 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -335,7 +335,10 @@ class FunctionExecutionContext(context.TreeContext): debug.dbg('Return unreachable: %s', r) else: if check_yields: - context_set |= ContextSet(self._eval_yield(r)) + context_set |= ContextSet.from_sets( + lazy_context.infer() + for lazy_context in self._eval_yield(r) + ) else: try: children = r.children @@ -388,7 +391,7 @@ class FunctionExecutionContext(context.TreeContext): else: types = self.get_return_values(check_yields=True) if types: - yield context.get_merged_lazy_context(list(types)) + yield context.LazyKnownContexts(types) return last_for_stmt = for_stmt @@ -432,11 +435,7 @@ class ModuleAttributeName(AbstractNameDefinition): self.string_name = string_name def infer(self): - return ContextSet( - compiled.create(self.parent_context.evaluator, str).execute( - param.ValuesArguments([]) - ) - ) + return compiled.create(self.parent_context.evaluator, str).execute_evaluated() class ModuleName(ContextNameMixin, AbstractNameDefinition): diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 053f2b9c..169e709e 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -186,7 +186,7 @@ def builtins_super(evaluator, types, objects, context): if isinstance(context, (InstanceFunctionExecution, AnonymousInstanceFunctionExecution)): su = context.instance.py__class__().py__bases__() - return unite(context.execute_evaluated() for context in su[0].infer()) + return su[0].infer().execute_evaluated() return NO_CONTEXTS @@ -208,7 +208,7 @@ def builtins_reversed(evaluator, sequences, obj, arguments): # would fail in certain cases like `reversed(x).__iter__` if we # just returned the result directly. seq = iterable.FakeSequence(evaluator, 'list', rev) - arguments = param.ValuesArguments([[seq]]) + arguments = param.ValuesArguments([ContextSet(seq)]) return ContextSet(CompiledInstance(evaluator, evaluator.BUILTINS, obj, arguments)) From ee52cc75011f3628d577646c8b73323160ed20e6 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 26 Sep 2017 17:26:33 +0200 Subject: [PATCH 06/35] Fix most dynamic array issues. --- jedi/common/context.py | 1 + jedi/evaluate/compiled/__init__.py | 4 +++- jedi/evaluate/compiled/mixed.py | 5 ++++- jedi/evaluate/imports.py | 16 +++++++++++----- jedi/evaluate/iterable.py | 12 +++++------- test/test_evaluate/test_compiled.py | 2 +- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/jedi/common/context.py b/jedi/common/context.py index 514a6ea3..ecf12a5a 100644 --- a/jedi/common/context.py +++ b/jedi/common/context.py @@ -18,6 +18,7 @@ class ContextSet(object): Used to work with an iterable of set. """ aggregated = set() + sets = list(sets) for set_ in sets: if isinstance(set_, ContextSet): aggregated |= set_._set diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 091d07d4..c4d3ae32 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -279,7 +279,9 @@ class CompiledObject(Context): return [] # Builtins don't have imports def dict_values(self): - return ContextSet(create(self.evaluator, v) for v in self.obj.values()) + return ContextSet.from_iterable( + create(self.evaluator, v) for v in self.obj.values() + ) class CompiledName(AbstractNameDefinition): diff --git a/jedi/evaluate/compiled/mixed.py b/jedi/evaluate/compiled/mixed.py index 0be5007b..2fac34ff 100644 --- a/jedi/evaluate/compiled/mixed.py +++ b/jedi/evaluate/compiled/mixed.py @@ -12,6 +12,7 @@ from jedi.evaluate import imports from jedi.evaluate.context import Context from jedi.evaluate.cache import evaluator_function_cache from jedi.evaluate.compiled.getattr_static import getattr_static +from jedi.common import ContextSet class MixedObject(object): @@ -85,7 +86,9 @@ class MixedName(compiled.CompiledName): # PyQt4.QtGui.QStyleOptionComboBox.currentText # -> just set it to None obj = None - return [_create(self._evaluator, obj, parent_context=self.parent_context)] + return ContextSet( + _create(self._evaluator, obj, parent_context=self.parent_context) + ) @property def api_type(self): diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 5d69e931..716fa35f 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -28,6 +28,7 @@ from jedi.evaluate import sys_path from jedi.evaluate import helpers from jedi.evaluate import compiled from jedi.evaluate import analysis +from jedi.evaluate.utils import unite from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.filters import AbstractNameDefinition from jedi.common import ContextSet, NO_CONTEXTS @@ -66,12 +67,17 @@ def infer_import(context, tree_name, is_goto=False): return NO_CONTEXTS if from_import_name is not None: - types = types.py__getattribute__( - from_import_name, - name_context=context, - is_goto=is_goto, - analysis_errors=False + types = unite( + t.py__getattribute__( + from_import_name, + name_context=context, + is_goto=is_goto, + analysis_errors=False + ) + for t in types ) + if not is_goto: + types = ContextSet.from_set(types) if not types: path = import_path + [from_import_name] diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index cbab52c6..2b8425ab 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -726,17 +726,15 @@ def _check_array_additions(context, sequence): def find_additions(context, arglist, add_name): params = list(param.TreeArguments(context.evaluator, context, arglist).unpack()) - result = ContextSet() + result = set() if add_name in ['insert']: params = params[1:] if add_name in ['append', 'add', 'insert']: - for key, lazy_context in params: - result.add(lazy_context) + for key, whatever in params: + result.add(whatever) elif add_name in ['extend', 'update']: for key, lazy_context in params: - result |= ContextSet.from_iterable( - py__iter__(context.evaluator, lazy_context.infer()) - ) + result |= set(py__iter__(context.evaluator, lazy_context.infer())) return result temp_param_add, settings.dynamic_params_for_other_modules = \ @@ -745,7 +743,7 @@ def _check_array_additions(context, sequence): is_list = sequence.name.string_name == 'list' search_names = (['append', 'extend', 'insert'] if is_list else ['add', 'update']) - added_types = NO_CONTEXTS + added_types = set() for add_name in search_names: try: possible_names = module_context.tree_node.get_used_names()[add_name] diff --git a/test/test_evaluate/test_compiled.py b/test/test_evaluate/test_compiled.py index 0ff4693a..69dcdbe7 100644 --- a/test/test_evaluate/test_compiled.py +++ b/test/test_evaluate/test_compiled.py @@ -95,4 +95,4 @@ def test_time_docstring(): def test_dict_values(): - assert Script('import sys/sys.modules["alshdb;lasdhf"]').goto_definitions() + assert Script('import sys\nsys.modules["alshdb;lasdhf"]').goto_definitions() From 00f2f9a90cae10a08368c9181f07a7e0cdfccb29 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 26 Sep 2017 18:17:19 +0200 Subject: [PATCH 07/35] Fix the final issues with the ContextSet refactoring. --- jedi/evaluate/__init__.py | 2 +- jedi/evaluate/precedence.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 02b30eab..96651196 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -355,7 +355,7 @@ class Evaluator(object): context_set = self.eval_atom(context, element.children[0]) for next_name in element.children[2::2]: # TODO add search_global=True? - context_set.py__getattribute__(next_name, name_context=context) + context_set = context_set.py__getattribute__(next_name, name_context=context) return context_set elif typ == 'eval_input': return self._eval_element_not_cached(context, element.children[0]) diff --git a/jedi/evaluate/precedence.py b/jedi/evaluate/precedence.py index abc73589..46ce5507 100644 --- a/jedi/evaluate/precedence.py +++ b/jedi/evaluate/precedence.py @@ -50,7 +50,7 @@ def calculate_children(evaluator, context, children): # handle lazy evaluation of and/or here. if operator in ('and', 'or'): - left_bools = ContextSet(left.py__bool__() for left in types) + left_bools = set(left.py__bool__() for left in types) if left_bools == set([True]): if operator == 'and': types = context.eval_node(right) @@ -65,22 +65,22 @@ def calculate_children(evaluator, context, children): return types -def calculate(evaluator, context, left_result, operator, right_result): - if not left_result or not right_result: +def calculate(evaluator, context, left_contexts, operator, right_contexts): + if not left_contexts or not right_contexts: # illegal slices e.g. cause left/right_result to be None - result = (left_result or NO_CONTEXTS) | (right_result or NO_CONTEXTS) + result = (left_contexts or NO_CONTEXTS) | (right_contexts or NO_CONTEXTS) return literals_to_types(evaluator, result) else: # I don't think there's a reasonable chance that a string # operation is still correct, once we pass something like six # objects. - if len(left_result) * len(right_result) > 6: - return literals_to_types(evaluator, left_result | right_result) + if len(left_contexts) * len(right_contexts) > 6: + return literals_to_types(evaluator, left_contexts | right_contexts) else: return ContextSet.from_sets( _element_calculate(evaluator, context, left, operator, right) - for left in left_result - for right in right_result + for left in left_contexts + for right in right_contexts ) From e4090910f68ff0f99b1be5e7e629cc348e388e83 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 26 Sep 2017 18:24:42 +0200 Subject: [PATCH 08/35] Remove the ParamListener, it was not used anymore. --- jedi/evaluate/dynamic.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index 946d81e2..9857f312 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -14,7 +14,7 @@ It works as follows: - |Jedi| sees a param - search for function calls named ``foo`` -- execute these calls and check the input. This work with a ``ParamListener``. +- execute these calls and check the input. """ from parso.python import tree @@ -32,17 +32,6 @@ from jedi.parser_utils import get_parent_scope MAX_PARAM_SEARCHES = 20 -class ParamListener(object): - """ - This listener is used to get the params for a function. - """ - def __init__(self): - self.param_possibilities = [] - - def execute(self, params): - self.param_possibilities += params - - class MergedExecutedParams(object): """ Simulates being a parameter while actually just being multiple params. From a0a438fe6fb7cedbd3a01fb5af0aa67939955f05 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 26 Sep 2017 18:32:42 +0200 Subject: [PATCH 09/35] Forgot an iterator in context sets. --- jedi/evaluate/precedence.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jedi/evaluate/precedence.py b/jedi/evaluate/precedence.py index 46ce5507..8206dac5 100644 --- a/jedi/evaluate/precedence.py +++ b/jedi/evaluate/precedence.py @@ -7,7 +7,7 @@ from jedi._compatibility import unicode from jedi import debug from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name from jedi.evaluate import analysis -from jedi.common import ContextSet, NO_CONTEXTS +from jedi.common import ContextSet, NO_CONTEXTS, iterator_to_context_set # Maps Python syntax to the operator module. COMPARISON_OPERATORS = { @@ -84,6 +84,7 @@ def calculate(evaluator, context, left_contexts, operator, right_contexts): ) +@iterator_to_context_set def factor_calculate(evaluator, types, operator): """ Calculates `+`, `-`, `~` and `not` prefixes. From faa2d015934b4a6a66e3db75b309b63c12ffe348 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 26 Sep 2017 18:36:10 +0200 Subject: [PATCH 10/35] The memoize decorator doesn't need to magically cache generators as lists. This makes no sense at all. Explicit is better than implicit. --- jedi/evaluate/cache.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jedi/evaluate/cache.py b/jedi/evaluate/cache.py index 37bd0e8a..b7c7cd7e 100644 --- a/jedi/evaluate/cache.py +++ b/jedi/evaluate/cache.py @@ -4,8 +4,6 @@ - ``CachedMetaClass`` uses ``_memoize_default`` to do the same with classes. """ -import inspect - _NO_DEFAULT = object() @@ -40,8 +38,6 @@ def _memoize_default(default=_NO_DEFAULT, evaluator_is_first_arg=False, second_a if default is not _NO_DEFAULT: memo[key] = default rv = function(obj, *args, **kwargs) - if inspect.isgenerator(rv): - rv = list(rv) memo[key] = rv return rv return wrapper From b74c8cb03373d376b9ec50be7bcce58921e9bb4c Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 09:20:58 +0200 Subject: [PATCH 11/35] To be able to customize ContextSet, move a subclass to evaluate.context --- jedi/common/__init__.py | 3 +-- jedi/common/context.py | 20 +++++--------------- jedi/evaluate/__init__.py | 4 ++-- jedi/evaluate/compiled/__init__.py | 3 +-- jedi/evaluate/compiled/mixed.py | 3 +-- jedi/evaluate/context.py | 16 +++++++++++++++- jedi/evaluate/docstrings.py | 8 ++++---- jedi/evaluate/dynamic.py | 2 +- jedi/evaluate/filters.py | 2 +- jedi/evaluate/finder.py | 4 +--- jedi/evaluate/imports.py | 2 +- jedi/evaluate/instance.py | 4 ++-- jedi/evaluate/iterable.py | 6 +++--- jedi/evaluate/param.py | 2 +- jedi/evaluate/pep0484.py | 3 +-- jedi/evaluate/precedence.py | 2 +- jedi/evaluate/recursion.py | 2 +- jedi/evaluate/representation.py | 4 ++-- jedi/evaluate/stdlib.py | 4 ++-- 19 files changed, 46 insertions(+), 48 deletions(-) diff --git a/jedi/common/__init__.py b/jedi/common/__init__.py index ba2e2f44..b190d863 100644 --- a/jedi/common/__init__.py +++ b/jedi/common/__init__.py @@ -1,2 +1 @@ -from jedi.common.context import ContextSet, NO_CONTEXTS, \ - iterator_to_context_set +from jedi.common.context import BaseContextSet diff --git a/jedi/common/context.py b/jedi/common/context.py index ecf12a5a..95839ea9 100644 --- a/jedi/common/context.py +++ b/jedi/common/context.py @@ -1,4 +1,4 @@ -class ContextSet(object): +class BaseContextSet(object): def __init__(self, *args): self._set = set(args) @@ -20,14 +20,14 @@ class ContextSet(object): aggregated = set() sets = list(sets) for set_ in sets: - if isinstance(set_, ContextSet): + if isinstance(set_, BaseContextSet): aggregated |= set_._set else: aggregated |= set_ return cls.from_set(aggregated) def __or__(self, other): - return ContextSet.from_set(self._set | other._set) + return type(self).from_set(self._set | other._set) def __iter__(self): for element in self._set: @@ -43,22 +43,12 @@ class ContextSet(object): return '%s(%s)' % (self.__class__.__name__, ', '.join(str(s) for s in self._set)) def filter(self, filter_func): - return ContextSet.from_iterable(filter(filter_func, self._set)) + return type(self).from_iterable(filter(filter_func, self._set)) def __getattr__(self, name): def mapper(*args, **kwargs): - return ContextSet.from_sets( + return type(self).from_sets( getattr(context, name)(*args, **kwargs) for context in self._set ) return mapper - - -def iterator_to_context_set(func): - def wrapper(*args, **kwargs): - return ContextSet.from_iterable(func(*args, **kwargs)) - - return wrapper - - -NO_CONTEXTS = ContextSet() diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 96651196..7bd17c55 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -82,8 +82,8 @@ from jedi.evaluate import helpers from jedi.evaluate import pep0484 from jedi.evaluate.filters import TreeNameDefinition, ParamName from jedi.evaluate.instance import AnonymousInstance, BoundMethod -from jedi.evaluate.context import ContextualizedName, ContextualizedNode -from jedi.common import ContextSet, NO_CONTEXTS +from jedi.evaluate.context import ContextualizedName, ContextualizedNode, \ + ContextSet, NO_CONTEXTS from jedi import parser_utils diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index c4d3ae32..ab9ea8f5 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -13,9 +13,8 @@ from jedi import debug from jedi.cache import underscore_memoization, memoize_method from jedi.evaluate.filters import AbstractFilter, AbstractNameDefinition, \ ContextNameMixin -from jedi.evaluate.context import Context, LazyKnownContext +from jedi.evaluate.context import Context, LazyKnownContext, ContextSet from jedi.evaluate.compiled.getattr_static import getattr_static -from jedi.common import ContextSet from . import fake diff --git a/jedi/evaluate/compiled/mixed.py b/jedi/evaluate/compiled/mixed.py index 2fac34ff..3b397a31 100644 --- a/jedi/evaluate/compiled/mixed.py +++ b/jedi/evaluate/compiled/mixed.py @@ -9,10 +9,9 @@ from jedi import settings from jedi.evaluate import compiled from jedi.cache import underscore_memoization from jedi.evaluate import imports -from jedi.evaluate.context import Context +from jedi.evaluate.context import Context, ContextSet from jedi.evaluate.cache import evaluator_function_cache from jedi.evaluate.compiled.getattr_static import getattr_static -from jedi.common import ContextSet class MixedObject(object): diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index 7ff57d5d..6202e99f 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -1,7 +1,7 @@ from jedi._compatibility import Python3Method from parso.python.tree import ExprStmt, CompFor from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature -from jedi.common import ContextSet, NO_CONTEXTS +from jedi.common import BaseContextSet class Context(object): @@ -204,3 +204,17 @@ class ContextualizedName(ContextualizedNode): compare = node node = node.parent return indexes + + +class ContextSet(BaseContextSet): + pass + + +NO_CONTEXTS = ContextSet() + + +def iterator_to_context_set(func): + def wrapper(*args, **kwargs): + return ContextSet.from_iterable(func(*args, **kwargs)) + + return wrapper diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index ea471811..f80036bd 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -21,11 +21,11 @@ from textwrap import dedent from parso import parse from jedi._compatibility import u -from jedi.evaluate.utils import unite, indent_block -from jedi.evaluate import context +from jedi.evaluate.utils import indent_block from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.iterable import SequenceLiteralContext, FakeSequence -from jedi.common import iterator_to_context_set, ContextSet, NO_CONTEXTS +from jedi.evaluate.context import iterator_to_context_set, ContextSet, \ + NO_CONTEXTS, LazyKnownContexts DOCSTRING_PARAM_PATTERNS = [ @@ -241,7 +241,7 @@ def _execute_array_values(evaluator, array): _execute_array_values(evaluator, typ) for typ in lazy_context.infer() ) - values.append(context.LazyKnownContexts(objects)) + values.append(LazyKnownContexts(objects)) return set([FakeSequence(evaluator, array.array_type, values)]) else: return array.execute_evaluated() diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index 9857f312..f5b4cd0d 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -25,7 +25,7 @@ from jedi.evaluate import imports from jedi.evaluate.param import TreeArguments, create_default_params from jedi.evaluate.helpers import is_stdlib_path from jedi.evaluate.utils import to_list -from jedi.common import ContextSet +from jedi.evaluate.context import ContextSet from jedi.parser_utils import get_parent_scope diff --git a/jedi/evaluate/filters.py b/jedi/evaluate/filters.py index e3edb1eb..f4c6463b 100644 --- a/jedi/evaluate/filters.py +++ b/jedi/evaluate/filters.py @@ -6,7 +6,7 @@ from abc import abstractmethod from parso.tree import search_ancestor from jedi.evaluate import flow_analysis -from jedi.common import ContextSet +from jedi.evaluate.context import ContextSet from jedi.parser_utils import get_parent_scope from jedi.evaluate.utils import to_list diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 15d45108..fd37adac 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -18,7 +18,6 @@ check for -> a is a string). There's big potential in these checks. from parso.python import tree from parso.tree import search_ancestor from jedi import debug -from jedi.evaluate.utils import unite from jedi import settings from jedi.evaluate import representation as er from jedi.evaluate.instance import AbstractInstanceContext @@ -31,8 +30,7 @@ from jedi.evaluate import flow_analysis from jedi.evaluate import param from jedi.evaluate import helpers from jedi.evaluate.filters import get_global_filters, TreeNameDefinition -from jedi.evaluate.context import ContextualizedName, ContextualizedNode -from jedi.common import ContextSet +from jedi.evaluate.context import ContextualizedName, ContextualizedNode, ContextSet from jedi.parser_utils import is_scope, get_parent_scope diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 716fa35f..f06d3b66 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -31,7 +31,7 @@ from jedi.evaluate import analysis from jedi.evaluate.utils import unite from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.filters import AbstractNameDefinition -from jedi.common import ContextSet, NO_CONTEXTS +from jedi.evaluate.context import ContextSet, NO_CONTEXTS # This memoization is needed, because otherwise we will infinitely loop on diff --git a/jedi/evaluate/instance.py b/jedi/evaluate/instance.py index f1116707..747f5802 100644 --- a/jedi/evaluate/instance.py +++ b/jedi/evaluate/instance.py @@ -4,13 +4,13 @@ from jedi._compatibility import is_py3 from jedi import debug from jedi.evaluate import compiled from jedi.evaluate import filters -from jedi.evaluate.context import Context, LazyKnownContext, LazyKnownContexts +from jedi.evaluate.context import Context, LazyKnownContext, LazyKnownContexts, \ + ContextSet, iterator_to_context_set, NO_CONTEXTS from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.param import AbstractArguments, AnonymousArguments from jedi.cache import memoize_method from jedi.evaluate import representation as er from jedi.evaluate import iterable -from jedi.common import ContextSet, iterator_to_context_set, NO_CONTEXTS from jedi.parser_utils import get_parent_scope diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 2b8425ab..be62ecb0 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -22,20 +22,20 @@ It is important to note that: """ from jedi import debug from jedi import settings -from jedi.evaluate.utils import unite, safe_property +from jedi.evaluate.utils import safe_property from jedi._compatibility import unicode, zip_longest, is_py3 from jedi.evaluate.utils import to_list from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate import analysis from jedi.evaluate import pep0484 -from jedi.evaluate import context from jedi.evaluate import precedence from jedi.evaluate import recursion from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.filters import DictFilter, AbstractNameDefinition, \ ParserTreeFilter -from jedi.common import ContextSet, NO_CONTEXTS +from jedi.evaluate import context +from jedi.evaluate.context import ContextSet, NO_CONTEXTS from jedi.parser_utils import get_comp_fors diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index a914d81a..f5a183a1 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -10,7 +10,7 @@ from jedi.evaluate import context from jedi.evaluate import docstrings from jedi.evaluate import pep0484 from jedi.evaluate.filters import ParamName -from jedi.common import NO_CONTEXTS +from jedi.evaluate.context import NO_CONTEXTS def add_argument_issue(parent_context, error_name, lazy_context, message): diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py index 0fa7ca41..869cf8a8 100644 --- a/jedi/evaluate/pep0484.py +++ b/jedi/evaluate/pep0484.py @@ -27,8 +27,7 @@ from parso.python import tree from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate import compiled -from jedi.evaluate.context import LazyTreeContext -from jedi.common import NO_CONTEXTS, ContextSet +from jedi.evaluate.context import LazyTreeContext, NO_CONTEXTS, ContextSet from jedi import debug from jedi import _compatibility from jedi import parser_utils diff --git a/jedi/evaluate/precedence.py b/jedi/evaluate/precedence.py index 8206dac5..7e054c05 100644 --- a/jedi/evaluate/precedence.py +++ b/jedi/evaluate/precedence.py @@ -7,7 +7,7 @@ from jedi._compatibility import unicode from jedi import debug from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name from jedi.evaluate import analysis -from jedi.common import ContextSet, NO_CONTEXTS, iterator_to_context_set +from jedi.evaluate.context import ContextSet, NO_CONTEXTS, iterator_to_context_set # Maps Python syntax to the operator module. COMPARISON_OPERATORS = { diff --git a/jedi/evaluate/recursion.py b/jedi/evaluate/recursion.py index ebc71f09..02f13db9 100644 --- a/jedi/evaluate/recursion.py +++ b/jedi/evaluate/recursion.py @@ -29,7 +29,7 @@ therefore the quality might not always be maximal. from contextlib import contextmanager from jedi import debug -from jedi.common import NO_CONTEXTS +from jedi.evaluate.context import NO_CONTEXTS recursion_limit = 15 diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index c427b786..fb7e81d5 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -63,8 +63,8 @@ from jedi.evaluate.filters import ParserTreeFilter, FunctionExecutionFilter, \ ParamName, AnonymousInstanceParamName, TreeNameDefinition, \ ContextNameMixin from jedi.evaluate import context -from jedi.evaluate.context import ContextualizedNode -from jedi.common import NO_CONTEXTS, ContextSet, iterator_to_context_set +from jedi.evaluate.context import ContextualizedNode, NO_CONTEXTS, \ + ContextSet, iterator_to_context_set from jedi import parser_utils from jedi.evaluate.parser_cache import get_yield_exprs diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 169e709e..0f85b62d 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -23,8 +23,8 @@ from jedi import debug from jedi.evaluate import precedence from jedi.evaluate import param from jedi.evaluate import analysis -from jedi.evaluate.context import LazyTreeContext, ContextualizedNode -from jedi.common import NO_CONTEXTS, ContextSet +from jedi.evaluate.context import LazyTreeContext, ContextualizedNode, \ + NO_CONTEXTS, ContextSet # Now this is all part of fake tuples in Jedi. However super doesn't work on # __init__ and __new__ doesn't work at all. So adding this to nametuples is From b1ed0c7d22ba0d43118f7a12751d1809cf853e04 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 14:09:09 +0200 Subject: [PATCH 12/35] Add py__class__ to ContextSet. --- jedi/evaluate/context.py | 3 ++- jedi/evaluate/stdlib.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index 6202e99f..98960c64 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -207,7 +207,8 @@ class ContextualizedName(ContextualizedNode): class ContextSet(BaseContextSet): - pass + def py__class__(self): + return ContextSet.from_iterable(c.py__class__() for c in self._set) NO_CONTEXTS = ContextSet() diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 0f85b62d..447079c1 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -170,7 +170,7 @@ def builtins_type(evaluator, objects, bases, dicts): # It's a type creation... maybe someday... return NO_CONTEXTS else: - return ContextSet.from_iterable(o.py__class__() for o in objects) + return objects.py__class__() class SuperInstance(AbstractInstanceContext): From 025951089a5d46865ccf529b00e9bfd11a66f884 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 15:17:11 +0200 Subject: [PATCH 13/35] Some conversions of eval_element -> eval_node. --- jedi/evaluate/__init__.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 7bd17c55..c16a56cb 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -30,11 +30,11 @@ return the ``date`` class. To *visualize* this (simplified): - ``Evaluator.eval_statement`` doesn't do much, because there's no assignment. -- ``Evaluator.eval_element`` cares for resolving the dotted path +- ``Context.eval_node`` cares for resolving the dotted path - ``Evaluator.find_types`` searches for global definitions of datetime, which it finds in the definition of an import, by scanning the syntax tree. - Using the import logic, the datetime module is found. -- Now ``find_types`` is called again by ``eval_element`` to find ``date`` +- Now ``find_types`` is called again by ``eval_node`` to find ``date`` inside the datetime module. Now what would happen if we wanted ``datetime.date.foo.bar``? Two more @@ -179,7 +179,7 @@ class Evaluator(object): """ debug.dbg('eval_statement %s (%s)', stmt, seek_name) rhs = stmt.get_rhs() - context_set = self.eval_element(context, rhs) + context_set = context.eval_node(rhs) if seek_name: c_node = ContextualizedName(context, seek_name) @@ -207,7 +207,7 @@ class Evaluator(object): for lazy_context in ordered: dct = {for_stmt.children[1].value: lazy_context.infer()} with helpers.predefine_names(context, for_stmt, dct): - t = self.eval_element(context, rhs) + t = context.eval_node(rhs) left = precedence.calculate(self, context, left, operator, t) context_set = left else: @@ -337,14 +337,14 @@ class Evaluator(object): # The implicit tuple in statements. return ContextSet(iterable.SequenceLiteralContext(self, context, element)) elif typ in ('not_test', 'factor'): - context_set = self.eval_element(context, element.children[-1]) + context_set = context.eval_node(element.children[-1]) for operator in element.children[:-1]: context_set = precedence.factor_calculate(self, context_set, operator) return context_set elif typ == 'test': # `x if foo else y` case. - return (self.eval_element(context, element.children[0]) | - self.eval_element(context, element.children[-1])) + return (context.eval_node(element.children[0]) | + context.eval_node(element.children[-1])) elif typ == 'operator': # Must be an ellipsis, other operators are not evaluated. # In Python 2 ellipsis is coded as three single dot tokens, not @@ -399,7 +399,7 @@ class Evaluator(object): elif c[0] == '(' and not len(c) == 2 \ and not(c[1].type == 'testlist_comp' and len(c[1].children) > 1): - return self.eval_element(context, c[1]) + return context.eval_node(c[1]) try: comp_for = c[1].children[1] @@ -486,7 +486,7 @@ class Evaluator(object): if is_simple_name: return self.eval_statement(context, def_, name) if type_ == 'for_stmt': - container_types = self.eval_element(context, def_.children[3]) + container_types = context.eval_node(def_.children[3]) cn = ContextualizedNode(context, def_.children[3]) for_types = iterable.py__iter__types(self, container_types, cn) c_node = ContextualizedName(context, name) @@ -523,11 +523,11 @@ class Evaluator(object): trailer = trailer.parent if trailer.type != 'classdef': if trailer.type == 'decorator': - context_set = self.eval_element(context, trailer.children[1]) + context_set = context.eval_node(trailer.children[1]) else: i = trailer.parent.children.index(trailer) to_evaluate = trailer.parent.children[:i] - context_set = self.eval_element(context, to_evaluate[0]) + context_set = context.eval_node(to_evaluate[0]) for trailer in to_evaluate[1:]: context_set = self.eval_trailer(context, context_set, trailer) param_names = [] @@ -546,7 +546,7 @@ class Evaluator(object): if index > 0: new_dotted = helpers.deep_ast_copy(par) new_dotted.children[index - 1:] = [] - values = self.eval_element(context, new_dotted) + values = context.eval_node(new_dotted) return unite( value.py__getattribute__(name, name_context=context, is_goto=True) for value in values From 313e1b38758ec062bbe99a432d909efeb5429f71 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 16:07:24 +0200 Subject: [PATCH 14/35] Use a different way of executing functions. --- jedi/evaluate/__init__.py | 26 ++++++++++++-------------- jedi/evaluate/iterable.py | 7 +++++++ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index c16a56cb..cc93f163 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -428,26 +428,24 @@ class Evaluator(object): context = iterable.SequenceLiteralContext(self, context, atom) return ContextSet(context) - def eval_trailer(self, context, types, trailer): + def eval_trailer(self, context, base_contexts, trailer): trailer_op, node = trailer.children[:2] if node == ')': # `arglist` is optional. node = () if trailer_op == '[': - return iterable.py__getitem__(self, context, types, trailer) + return iterable.py__getitem__(self, context, base_contexts, trailer) else: - context_set = ContextSet() - for typ in types: - debug.dbg('eval_trailer: %s in scope %s', trailer, typ) - if trailer_op == '.': - context_set |= typ.py__getattribute__( - name_context=context, - name_or_str=node - ) - elif trailer_op == '(': - arguments = param.TreeArguments(self, context, node, trailer) - context_set |= self.execute(typ, arguments) - return context_set + debug.dbg('eval_trailer: %s in %s', trailer, base_contexts) + if trailer_op == '.': + return base_contexts.py__getattribute__( + name_context=context, + name_or_str=node + ) + else: + assert trailer_op == '(' + arguments = param.TreeArguments(self, context, node, trailer) + return base_contexts.execute(arguments) @debug.increase_indent def execute(self, obj, arguments): diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index be62ecb0..524d843f 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -61,9 +61,16 @@ class BuiltinMethod(object): self._method = method self._builtin_func = builtin_func + # TODO it seems kind of stupid that we have to overwrite 3 methods here. def py__call__(self, params): return self._method(self._builtin_context) + def execute(self, *args, **kwargs): + return self._builtin_context.evaluator.execute(self, *args, **kwargs) + + def execute_evaluated(self, *args, **kwargs): + return self._builtin_context.evaluator.execute_evaluated(self, *args, **kwargs) + def __getattr__(self, name): return getattr(self._builtin_func, name) From 5415a6164f5f28dd03afe6d399c32e78787672c8 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 16:21:02 +0200 Subject: [PATCH 15/35] Starting to try to move some functions away from Evaluator. This time eval_trailer. --- jedi/evaluate/__init__.py | 25 +++---------------------- jedi/evaluate/compiled/mixed.py | 3 --- jedi/evaluate/context.py | 3 --- jedi/evaluate/finder.py | 3 ++- jedi/evaluate/helpers.py | 4 +++- jedi/evaluate/syntax_tree.py | 27 +++++++++++++++++++++++++++ 6 files changed, 35 insertions(+), 30 deletions(-) create mode 100644 jedi/evaluate/syntax_tree.py diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index cc93f163..03cb9d6a 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -77,13 +77,13 @@ from jedi.evaluate import stdlib from jedi.evaluate import finder from jedi.evaluate import compiled from jedi.evaluate import precedence -from jedi.evaluate import param from jedi.evaluate import helpers from jedi.evaluate import pep0484 from jedi.evaluate.filters import TreeNameDefinition, ParamName from jedi.evaluate.instance import AnonymousInstance, BoundMethod from jedi.evaluate.context import ContextualizedName, ContextualizedNode, \ ContextSet, NO_CONTEXTS +from jedi.evaluate.syntax_tree import eval_trailer from jedi import parser_utils @@ -330,7 +330,7 @@ class Evaluator(object): right ) break - context_set = self.eval_trailer(context, context_set, trailer) + context_set = eval_trailer(context, context_set, trailer) return context_set return NO_CONTEXTS elif typ in ('testlist_star_expr', 'testlist',): @@ -428,25 +428,6 @@ class Evaluator(object): context = iterable.SequenceLiteralContext(self, context, atom) return ContextSet(context) - def eval_trailer(self, context, base_contexts, trailer): - trailer_op, node = trailer.children[:2] - if node == ')': # `arglist` is optional. - node = () - - if trailer_op == '[': - return iterable.py__getitem__(self, context, base_contexts, trailer) - else: - debug.dbg('eval_trailer: %s in %s', trailer, base_contexts) - if trailer_op == '.': - return base_contexts.py__getattribute__( - name_context=context, - name_or_str=node - ) - else: - assert trailer_op == '(' - arguments = param.TreeArguments(self, context, node, trailer) - return base_contexts.execute(arguments) - @debug.increase_indent def execute(self, obj, arguments): if self.is_analysis: @@ -527,7 +508,7 @@ class Evaluator(object): to_evaluate = trailer.parent.children[:i] context_set = context.eval_node(to_evaluate[0]) for trailer in to_evaluate[1:]: - context_set = self.eval_trailer(context, context_set, trailer) + context_set = eval_trailer(context, context_set, trailer) param_names = [] for context in context_set: try: diff --git a/jedi/evaluate/compiled/mixed.py b/jedi/evaluate/compiled/mixed.py index 3b397a31..be7fa34c 100644 --- a/jedi/evaluate/compiled/mixed.py +++ b/jedi/evaluate/compiled/mixed.py @@ -41,9 +41,6 @@ class MixedObject(object): # We have to overwrite everything that has to do with trailers, name # lookups and filters to make it possible to route name lookups towards # compiled objects and the rest towards tree node contexts. - def eval_trailer(*args, **kwags): - return Context.eval_trailer(*args, **kwags) - def py__getattribute__(*args, **kwargs): return Context.py__getattribute__(*args, **kwargs) diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index 98960c64..d9386ea7 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -49,9 +49,6 @@ class Context(object): def eval_stmt(self, stmt, seek_name=None): return self.evaluator.eval_statement(self, stmt, seek_name) - def eval_trailer(self, types, trailer): - return self.evaluator.eval_trailer(self, types, trailer) - @Python3Method def py__getattribute__(self, name_or_str, name_context=None, position=None, search_global=False, is_goto=False, diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index fd37adac..79d24f5d 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -32,6 +32,7 @@ from jedi.evaluate import helpers from jedi.evaluate.filters import get_global_filters, TreeNameDefinition from jedi.evaluate.context import ContextualizedName, ContextualizedNode, ContextSet from jedi.parser_utils import is_scope, get_parent_scope +from jedi.evaluate.syntax_tree import eval_trailer class NameFinder(object): @@ -263,7 +264,7 @@ def _apply_decorators(evaluator, context, node): # Create a trailer and evaluate it. trailer = tree.PythonNode('trailer', trailer_nodes) trailer.parent = dec - dec_values = evaluator.eval_trailer(context, dec_values, trailer) + dec_values = eval_trailer(context, dec_values, trailer) if not len(dec_values): debug.warning('decorator not found: %s on %s', dec, node) diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index b2de3676..d581245e 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -6,7 +6,9 @@ from itertools import chain from contextlib import contextmanager from parso.python import tree + from jedi.parser_utils import get_parent_scope +from jedi.evaluate.syntax_tree import eval_trailer def is_stdlib_path(path): @@ -88,7 +90,7 @@ def evaluate_call_of_leaf(context, leaf, cut_own_trailer=False): values = context.eval_node(base) for trailer in trailers: - values = context.eval_trailer(values, trailer) + values = eval_trailer(context, values, trailer) return values diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py new file mode 100644 index 00000000..953496f4 --- /dev/null +++ b/jedi/evaluate/syntax_tree.py @@ -0,0 +1,27 @@ +""" +Functions evaluating the syntax tree. +""" + +from jedi import debug + + +def eval_trailer(context, base_contexts, trailer): + trailer_op, node = trailer.children[:2] + if node == ')': # `arglist` is optional. + node = () + + if trailer_op == '[': + from jedi.evaluate import iterable + return iterable.py__getitem__(context.evaluator, context, base_contexts, trailer) + else: + debug.dbg('eval_trailer: %s in %s', trailer, base_contexts) + if trailer_op == '.': + return base_contexts.py__getattribute__( + name_context=context, + name_or_str=node + ) + else: + assert trailer_op == '(' + from jedi.evaluate import param + arguments = param.TreeArguments(context.evaluator, context, node, trailer) + return base_contexts.execute(arguments) From b997b538a721ad4b47f1dba9384898a89093289e Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 16:27:37 +0200 Subject: [PATCH 16/35] Move eval_atom to the syntax tree module. --- jedi/api/helpers.py | 3 +- jedi/evaluate/__init__.py | 72 ++---------------------------------- jedi/evaluate/syntax_tree.py | 72 ++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 69 deletions(-) diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index b467aa7c..2c4d8e0d 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -10,6 +10,7 @@ from parso.python import tree from parso import split_lines from jedi._compatibility import u +from jedi.evaluate.syntax_tree import eval_atom from jedi.evaluate.helpers import evaluate_call_of_leaf from jedi.cache import time_cache @@ -206,7 +207,7 @@ def evaluate_goto_definition(evaluator, context, leaf): elif parent.type == 'trailer': return evaluate_call_of_leaf(context, leaf) elif isinstance(leaf, tree.Literal): - return context.evaluator.eval_atom(context, leaf) + return eval_atom(context, leaf) return [] diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 03cb9d6a..61480e6e 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -83,7 +83,7 @@ from jedi.evaluate.filters import TreeNameDefinition, ParamName from jedi.evaluate.instance import AnonymousInstance, BoundMethod from jedi.evaluate.context import ContextualizedName, ContextualizedNode, \ ContextSet, NO_CONTEXTS -from jedi.evaluate.syntax_tree import eval_trailer +from jedi.evaluate.syntax_tree import eval_trailer, eval_atom from jedi import parser_utils @@ -304,7 +304,7 @@ class Evaluator(object): debug.dbg('eval_element %s@%s', element, element.start_pos) typ = element.type if typ in ('name', 'number', 'string', 'atom'): - return self.eval_atom(context, element) + return eval_atom(context, element) elif typ == 'keyword': # For False/True/None if element.value in ('False', 'True', 'None'): @@ -318,7 +318,7 @@ class Evaluator(object): elif typ in ('power', 'atom_expr'): first_child = element.children[0] if not (first_child.type == 'keyword' and first_child.value == 'await'): - context_set = self.eval_atom(context, first_child) + context_set = eval_atom(context, first_child) for trailer in element.children[1:]: if trailer == '**': # has a power operation. right = self.eval_element(context, element.children[2]) @@ -352,7 +352,7 @@ class Evaluator(object): assert element.value in ('.', '...') return ContextSet(compiled.create(self, Ellipsis)) elif typ == 'dotted_name': - context_set = self.eval_atom(context, element.children[0]) + context_set = eval_atom(context, element.children[0]) for next_name in element.children[2::2]: # TODO add search_global=True? context_set = context_set.py__getattribute__(next_name, name_context=context) @@ -364,70 +364,6 @@ class Evaluator(object): else: return precedence.calculate_children(self, context, element.children) - def eval_atom(self, context, atom): - """ - Basically to process ``atom`` nodes. The parser sometimes doesn't - generate the node (because it has just one child). In that case an atom - might be a name or a literal as well. - """ - if atom.type == 'name': - # This is the first global lookup. - stmt = tree.search_ancestor( - atom, 'expr_stmt', 'lambdef' - ) or atom - if stmt.type == 'lambdef': - stmt = atom - return context.py__getattribute__( - name_or_str=atom, - position=stmt.start_pos, - search_global=True - ) - - elif isinstance(atom, tree.Literal): - string = parser_utils.safe_literal_eval(atom.value) - return ContextSet(compiled.create(self, string)) - else: - c = atom.children - if c[0].type == 'string': - # Will be one string. - context_set = self.eval_atom(context, c[0]) - for string in c[1:]: - right = self.eval_atom(context, string) - context_set = precedence.calculate(self, context, context_set, '+', right) - return context_set - # Parentheses without commas are not tuples. - elif c[0] == '(' and not len(c) == 2 \ - and not(c[1].type == 'testlist_comp' and - len(c[1].children) > 1): - return context.eval_node(c[1]) - - try: - comp_for = c[1].children[1] - except (IndexError, AttributeError): - pass - else: - if comp_for == ':': - # Dict comprehensions have a colon at the 3rd index. - try: - comp_for = c[1].children[3] - except IndexError: - pass - - if comp_for.type == 'comp_for': - return ContextSet(iterable.Comprehension.from_atom(self, context, atom)) - - # It's a dict/list/tuple literal. - array_node = c[1] - try: - array_node_c = array_node.children - except AttributeError: - array_node_c = [] - if c[0] == '{' and (array_node == '}' or ':' in array_node_c): - context = iterable.DictLiteralContext(self, context, atom) - else: - context = iterable.SequenceLiteralContext(self, context, atom) - return ContextSet(context) - @debug.increase_indent def execute(self, obj, arguments): if self.is_analysis: diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index 953496f4..9fd18ea0 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -2,7 +2,13 @@ Functions evaluating the syntax tree. """ +from parso.python import tree + from jedi import debug +from jedi import parser_utils +from jedi.evaluate.context import ContextSet +from jedi.evaluate import compiled +from jedi.evaluate import precedence def eval_trailer(context, base_contexts, trailer): @@ -25,3 +31,69 @@ def eval_trailer(context, base_contexts, trailer): from jedi.evaluate import param arguments = param.TreeArguments(context.evaluator, context, node, trailer) return base_contexts.execute(arguments) + + +def eval_atom(context, atom): + """ + Basically to process ``atom`` nodes. The parser sometimes doesn't + generate the node (because it has just one child). In that case an atom + might be a name or a literal as well. + """ + from jedi.evaluate import iterable + if atom.type == 'name': + # This is the first global lookup. + stmt = tree.search_ancestor( + atom, 'expr_stmt', 'lambdef' + ) or atom + if stmt.type == 'lambdef': + stmt = atom + return context.py__getattribute__( + name_or_str=atom, + position=stmt.start_pos, + search_global=True + ) + + elif isinstance(atom, tree.Literal): + string = parser_utils.safe_literal_eval(atom.value) + return ContextSet(compiled.create(context.evaluator, string)) + else: + c = atom.children + if c[0].type == 'string': + # Will be one string. + context_set = eval_atom(context, c[0]) + for string in c[1:]: + right = eval_atom(context, string) + context_set = precedence.calculate(context.evaluator, context, context_set, '+', right) + return context_set + # Parentheses without commas are not tuples. + elif c[0] == '(' and not len(c) == 2 \ + and not(c[1].type == 'testlist_comp' and + len(c[1].children) > 1): + return context.eval_node(c[1]) + + try: + comp_for = c[1].children[1] + except (IndexError, AttributeError): + pass + else: + if comp_for == ':': + # Dict comprehensions have a colon at the 3rd index. + try: + comp_for = c[1].children[3] + except IndexError: + pass + + if comp_for.type == 'comp_for': + return ContextSet(iterable.Comprehension.from_atom(context.evaluator, context, atom)) + + # It's a dict/list/tuple literal. + array_node = c[1] + try: + array_node_c = array_node.children + except AttributeError: + array_node_c = [] + if c[0] == '{' and (array_node == '}' or ':' in array_node_c): + context = iterable.DictLiteralContext(context.evaluator, context, atom) + else: + context = iterable.SequenceLiteralContext(context.evaluator, context, atom) + return ContextSet(context) From d584b698b74bcd83957322418c3060a94cc0183d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 18:14:04 +0200 Subject: [PATCH 17/35] Move eval_element and eval_stmt to the syntax tree module. --- jedi/evaluate/__init__.py | 171 +++-------------------------------- jedi/evaluate/context.py | 3 - jedi/evaluate/finder.py | 4 +- jedi/evaluate/helpers.py | 2 +- jedi/evaluate/syntax_tree.py | 160 +++++++++++++++++++++++++++++++- 5 files changed, 174 insertions(+), 166 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 61480e6e..470f439f 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -14,22 +14,22 @@ Evaluation of Python code in |jedi| is based on three assumptions: The actual algorithm is based on a principle called lazy evaluation. If you don't know about it, google it. That said, the typical entry point for static -analysis is calling ``eval_statement``. There's separate logic for +analysis is calling ``eval_expr_stmt``. There's separate logic for autocompletion in the API, the evaluator is all about evaluating an expression. -Now you need to understand what follows after ``eval_statement``. Let's +Now you need to understand what follows after ``eval_expr_stmt``. Let's make an example:: import datetime datetime.date.toda# <-- cursor here First of all, this module doesn't care about completion. It really just cares -about ``datetime.date``. At the end of the procedure ``eval_statement`` will +about ``datetime.date``. At the end of the procedure ``eval_expr_stmt`` will return the ``date`` class. To *visualize* this (simplified): -- ``Evaluator.eval_statement`` doesn't do much, because there's no assignment. +- ``Evaluator.eval_expr_stmt`` doesn't do much, because there's no assignment. - ``Context.eval_node`` cares for resolving the dotted path - ``Evaluator.find_types`` searches for global definitions of datetime, which it finds in the definition of an import, by scanning the syntax tree. @@ -46,7 +46,7 @@ What if the import would contain another ``ExprStmt`` like this:: from foo import bar Date = bar.baz -Well... You get it. Just another ``eval_statement`` recursion. It's really +Well... You get it. Just another ``eval_expr_stmt`` recursion. It's really easy. Python can obviously get way more complicated then this. To understand tuple assignments, list comprehensions and everything else, a lot more code had to be written. @@ -76,40 +76,15 @@ from jedi.evaluate.cache import evaluator_function_cache from jedi.evaluate import stdlib from jedi.evaluate import finder from jedi.evaluate import compiled -from jedi.evaluate import precedence from jedi.evaluate import helpers -from jedi.evaluate import pep0484 from jedi.evaluate.filters import TreeNameDefinition, ParamName from jedi.evaluate.instance import AnonymousInstance, BoundMethod from jedi.evaluate.context import ContextualizedName, ContextualizedNode, \ ContextSet, NO_CONTEXTS -from jedi.evaluate.syntax_tree import eval_trailer, eval_atom +from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt, eval_node from jedi import parser_utils -def _limit_context_infers(func): - """ - This is for now the way how we limit type inference going wild. There are - other ways to ensure recursion limits as well. This is mostly necessary - because of instance (self) access that can be quite tricky to limit. - - I'm still not sure this is the way to go, but it looks okay for now and we - can still go anther way in the future. Tests are there. ~ dave - """ - def wrapper(evaluator, context, *args, **kwargs): - n = context.tree_node - try: - evaluator.inferred_element_counts[n] += 1 - if evaluator.inferred_element_counts[n] > 300: - debug.warning('In context %s there were too many inferences.', n) - return NO_CONTEXTS - except KeyError: - evaluator.inferred_element_counts[n] = 1 - return func(evaluator, context, *args, **kwargs) - - return wrapper - - class Evaluator(object): def __init__(self, grammar, sys_path=None): self.grammar = grammar @@ -159,65 +134,9 @@ class Evaluator(object): return f.filter_name(filters) return f.find(filters, attribute_lookup=not search_global) - @_limit_context_infers - def eval_statement(self, context, stmt, seek_name=None): - with recursion.execution_allowed(self, stmt) as allowed: - if allowed or context.get_root_context() == self.BUILTINS: - return self._eval_stmt(context, stmt, seek_name) - return NO_CONTEXTS - - #@evaluator_function_cache(default=[]) - @debug.increase_indent - def _eval_stmt(self, context, stmt, seek_name=None): - """ - The starting point of the completion. A statement always owns a call - list, which are the calls, that a statement does. In case multiple - names are defined in the statement, `seek_name` returns the result for - this name. - - :param stmt: A `tree.ExprStmt`. - """ - debug.dbg('eval_statement %s (%s)', stmt, seek_name) - rhs = stmt.get_rhs() - context_set = context.eval_node(rhs) - - if seek_name: - c_node = ContextualizedName(context, seek_name) - context_set = finder.check_tuple_assignments(self, c_node, context_set) - - first_operator = next(stmt.yield_operators(), None) - if first_operator not in ('=', None) and first_operator.type == 'operator': - # `=` is always the last character in aug assignments -> -1 - operator = copy.copy(first_operator) - operator.value = operator.value[:-1] - name = stmt.get_defined_names()[0].value - left = context.py__getattribute__( - name, position=stmt.start_pos, search_global=True) - - for_stmt = tree.search_ancestor(stmt, 'for_stmt') - if for_stmt is not None and for_stmt.type == 'for_stmt' and context_set \ - and parser_utils.for_stmt_defines_one_name(for_stmt): - # Iterate through result and add the values, that's possible - # only in for loops without clutter, because they are - # predictable. Also only do it, if the variable is not a tuple. - node = for_stmt.get_testlist() - cn = ContextualizedNode(context, node) - ordered = list(iterable.py__iter__(self, cn.infer(), cn)) - - for lazy_context in ordered: - dct = {for_stmt.children[1].value: lazy_context.infer()} - with helpers.predefine_names(context, for_stmt, dct): - t = context.eval_node(rhs) - left = precedence.calculate(self, context, left, operator, t) - context_set = left - else: - context_set = precedence.calculate(self, context, left, operator, context_set) - debug.dbg('eval_statement result %s', context_set) - return context_set - def eval_element(self, context, element): if isinstance(context, iterable.CompForContext): - return self._eval_element_not_cached(context, element) + return eval_node(context, element) if_stmt = element while if_stmt is not None: @@ -272,13 +191,13 @@ class Evaluator(object): result = ContextSet() for name_dict in name_dicts: with helpers.predefine_names(context, if_stmt, name_dict): - result |= self._eval_element_not_cached(context, element) + result |= eval_node(context, element) return result else: return self._eval_element_if_evaluated(context, element) else: if predefined_if_name_dict: - return self._eval_element_not_cached(context, element) + return eval_node(context, element) else: return self._eval_element_if_evaluated(context, element) @@ -291,78 +210,12 @@ class Evaluator(object): parent = parent.parent predefined_if_name_dict = context.predefined_names.get(parent) if predefined_if_name_dict is not None: - return self._eval_element_not_cached(context, element) + return eval_node(context, element) return self._eval_element_cached(context, element) @evaluator_function_cache(default=NO_CONTEXTS) def _eval_element_cached(self, context, element): - return self._eval_element_not_cached(context, element) - - @debug.increase_indent - @_limit_context_infers - def _eval_element_not_cached(self, context, element): - debug.dbg('eval_element %s@%s', element, element.start_pos) - typ = element.type - if typ in ('name', 'number', 'string', 'atom'): - return eval_atom(context, element) - elif typ == 'keyword': - # For False/True/None - if element.value in ('False', 'True', 'None'): - return ContextSet(compiled.builtin_from_name(self, element.value)) - # else: print e.g. could be evaluated like this in Python 2.7 - return NO_CONTEXTS - elif typ == 'lambdef': - return ContextSet(er.FunctionContext(self, context, element)) - elif typ == 'expr_stmt': - return self.eval_statement(context, element) - elif typ in ('power', 'atom_expr'): - first_child = element.children[0] - if not (first_child.type == 'keyword' and first_child.value == 'await'): - context_set = eval_atom(context, first_child) - for trailer in element.children[1:]: - if trailer == '**': # has a power operation. - right = self.eval_element(context, element.children[2]) - context_set = precedence.calculate( - self, - context, - context_set, - trailer, - right - ) - break - context_set = eval_trailer(context, context_set, trailer) - return context_set - return NO_CONTEXTS - elif typ in ('testlist_star_expr', 'testlist',): - # The implicit tuple in statements. - return ContextSet(iterable.SequenceLiteralContext(self, context, element)) - elif typ in ('not_test', 'factor'): - context_set = context.eval_node(element.children[-1]) - for operator in element.children[:-1]: - context_set = precedence.factor_calculate(self, context_set, operator) - return context_set - elif typ == 'test': - # `x if foo else y` case. - return (context.eval_node(element.children[0]) | - context.eval_node(element.children[-1])) - elif typ == 'operator': - # Must be an ellipsis, other operators are not evaluated. - # In Python 2 ellipsis is coded as three single dot tokens, not - # as one token 3 dot token. - assert element.value in ('.', '...') - return ContextSet(compiled.create(self, Ellipsis)) - elif typ == 'dotted_name': - context_set = eval_atom(context, element.children[0]) - for next_name in element.children[2::2]: - # TODO add search_global=True? - context_set = context_set.py__getattribute__(next_name, name_context=context) - return context_set - elif typ == 'eval_input': - return self._eval_element_not_cached(context, element.children[0]) - elif typ == 'annassign': - return pep0484._evaluate_for_annotation(context, element.children[1]) - else: - return precedence.calculate_children(self, context, element.children) + return eval_node(context, element) @debug.increase_indent def execute(self, obj, arguments): @@ -399,7 +252,7 @@ class Evaluator(object): if type_ == 'expr_stmt': is_simple_name = name.parent.type not in ('power', 'trailer') if is_simple_name: - return self.eval_statement(context, def_, name) + return eval_expr_stmt(context, def_, name) if type_ == 'for_stmt': container_types = context.eval_node(def_.children[3]) cn = ContextualizedNode(context, def_.children[3]) diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index d9386ea7..414fa5e1 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -46,9 +46,6 @@ class Context(object): def eval_node(self, node): return self.evaluator.eval_element(self, node) - def eval_stmt(self, stmt, seek_name=None): - return self.evaluator.eval_statement(self, stmt, seek_name) - @Python3Method def py__getattribute__(self, name_or_str, name_context=None, position=None, search_global=False, is_goto=False, diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 79d24f5d..19ac6f25 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -32,7 +32,7 @@ from jedi.evaluate import helpers from jedi.evaluate.filters import get_global_filters, TreeNameDefinition from jedi.evaluate.context import ContextualizedName, ContextualizedNode, ContextSet from jedi.parser_utils import is_scope, get_parent_scope -from jedi.evaluate.syntax_tree import eval_trailer +from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt class NameFinder(object): @@ -291,7 +291,7 @@ def _remove_statements(evaluator, context, stmt, name): if pep0484_contexts: return pep0484_contexts - return context.eval_stmt(stmt, seek_name=name) + return eval_expr_stmt(context, stmt, seek_name=name) def _check_flow_information(context, flow, search_name, pos): diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index d581245e..ff53e11d 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -8,7 +8,6 @@ from contextlib import contextmanager from parso.python import tree from jedi.parser_utils import get_parent_scope -from jedi.evaluate.syntax_tree import eval_trailer def is_stdlib_path(path): @@ -89,6 +88,7 @@ def evaluate_call_of_leaf(context, leaf, cut_own_trailer=False): trailers = power.children[1:cut] values = context.eval_node(base) + from jedi.evaluate.syntax_tree import eval_trailer for trailer in trailers: values = eval_trailer(context, values, trailer) return values diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index 9fd18ea0..ec775564 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -1,14 +1,113 @@ """ Functions evaluating the syntax tree. """ +import copy from parso.python import tree from jedi import debug from jedi import parser_utils -from jedi.evaluate.context import ContextSet +from jedi.evaluate.context import ContextSet, NO_CONTEXTS, ContextualizedNode, \ + ContextualizedName from jedi.evaluate import compiled from jedi.evaluate import precedence +from jedi.evaluate import pep0484 +from jedi.evaluate import recursion +from jedi.evaluate import helpers + + +def _limit_context_infers(func): + """ + This is for now the way how we limit type inference going wild. There are + other ways to ensure recursion limits as well. This is mostly necessary + because of instance (self) access that can be quite tricky to limit. + + I'm still not sure this is the way to go, but it looks okay for now and we + can still go anther way in the future. Tests are there. ~ dave + """ + def wrapper(context, *args, **kwargs): + n = context.tree_node + evaluator = context.evaluator + try: + evaluator.inferred_element_counts[n] += 1 + if evaluator.inferred_element_counts[n] > 300: + debug.warning('In context %s there were too many inferences.', n) + return NO_CONTEXTS + except KeyError: + evaluator.inferred_element_counts[n] = 1 + return func(context, *args, **kwargs) + + return wrapper + + +@debug.increase_indent +@_limit_context_infers +def eval_node(context, element): + debug.dbg('eval_element %s@%s', element, element.start_pos) + evaluator = context.evaluator + typ = element.type + if typ in ('name', 'number', 'string', 'atom'): + return eval_atom(context, element) + elif typ == 'keyword': + # For False/True/None + if element.value in ('False', 'True', 'None'): + return ContextSet(compiled.builtin_from_name(evaluator, element.value)) + # else: print e.g. could be evaluated like this in Python 2.7 + return NO_CONTEXTS + elif typ == 'lambdef': + from jedi.evaluate import representation as er + return ContextSet(er.FunctionContext(evaluator, context, element)) + elif typ == 'expr_stmt': + return eval_expr_stmt(context, element) + elif typ in ('power', 'atom_expr'): + first_child = element.children[0] + if not (first_child.type == 'keyword' and first_child.value == 'await'): + context_set = eval_atom(context, first_child) + for trailer in element.children[1:]: + if trailer == '**': # has a power operation. + right = evaluator.eval_element(context, element.children[2]) + context_set = precedence.calculate( + evaluator, + context, + context_set, + trailer, + right + ) + break + context_set = eval_trailer(context, context_set, trailer) + return context_set + return NO_CONTEXTS + elif typ in ('testlist_star_expr', 'testlist',): + # The implicit tuple in statements. + from jedi.evaluate import iterable + return ContextSet(iterable.SequenceLiteralContext(evaluator, context, element)) + elif typ in ('not_test', 'factor'): + context_set = context.eval_node(element.children[-1]) + for operator in element.children[:-1]: + context_set = precedence.factor_calculate(evaluator, context_set, operator) + return context_set + elif typ == 'test': + # `x if foo else y` case. + return (context.eval_node(element.children[0]) | + context.eval_node(element.children[-1])) + elif typ == 'operator': + # Must be an ellipsis, other operators are not evaluated. + # In Python 2 ellipsis is coded as three single dot tokens, not + # as one token 3 dot token. + assert element.value in ('.', '...') + return ContextSet(compiled.create(evaluator, Ellipsis)) + elif typ == 'dotted_name': + context_set = eval_atom(context, element.children[0]) + for next_name in element.children[2::2]: + # TODO add search_global=True? + context_set = context_set.py__getattribute__(next_name, name_context=context) + return context_set + elif typ == 'eval_input': + return eval_node(context, element.children[0]) + elif typ == 'annassign': + return pep0484._evaluate_for_annotation(context, element.children[1]) + else: + return precedence.calculate_children(evaluator, context, element.children) def eval_trailer(context, base_contexts, trailer): @@ -97,3 +196,62 @@ def eval_atom(context, atom): else: context = iterable.SequenceLiteralContext(context.evaluator, context, atom) return ContextSet(context) + + +@_limit_context_infers +def eval_expr_stmt(context, stmt, seek_name=None): + with recursion.execution_allowed(context.evaluator, stmt) as allowed: + if allowed or context.get_root_context() == context.evaluator.BUILTINS: + return _eval_stmt(context, stmt, seek_name) + return NO_CONTEXTS + + +@debug.increase_indent +def _eval_stmt(context, stmt, seek_name=None): + """ + The starting point of the completion. A statement always owns a call + list, which are the calls, that a statement does. In case multiple + names are defined in the statement, `seek_name` returns the result for + this name. + + :param stmt: A `tree.ExprStmt`. + """ + debug.dbg('eval_statement %s (%s)', stmt, seek_name) + rhs = stmt.get_rhs() + context_set = context.eval_node(rhs) + + if seek_name: + c_node = ContextualizedName(context, seek_name) + from jedi.evaluate import finder + context_set = finder.check_tuple_assignments(context.evaluator, c_node, context_set) + + first_operator = next(stmt.yield_operators(), None) + if first_operator not in ('=', None) and first_operator.type == 'operator': + # `=` is always the last character in aug assignments -> -1 + operator = copy.copy(first_operator) + operator.value = operator.value[:-1] + name = stmt.get_defined_names()[0].value + left = context.py__getattribute__( + name, position=stmt.start_pos, search_global=True) + + for_stmt = tree.search_ancestor(stmt, 'for_stmt') + if for_stmt is not None and for_stmt.type == 'for_stmt' and context_set \ + and parser_utils.for_stmt_defines_one_name(for_stmt): + # Iterate through result and add the values, that's possible + # only in for loops without clutter, because they are + # predictable. Also only do it, if the variable is not a tuple. + node = for_stmt.get_testlist() + cn = ContextualizedNode(context, node) + from jedi.evaluate import iterable + ordered = list(iterable.py__iter__(context.evaluator, cn.infer(), cn)) + + for lazy_context in ordered: + dct = {for_stmt.children[1].value: lazy_context.infer()} + with helpers.predefine_names(context, for_stmt, dct): + t = context.eval_node(rhs) + left = precedence.calculate(context.evaluator, context, left, operator, t) + context_set = left + else: + context_set = precedence.calculate(context.evaluator, context, left, operator, context_set) + debug.dbg('eval_statement result %s', context_set) + return context_set From 08a48672bc23c0b71f2d8cea9ce9d35cb40750f9 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 18:15:12 +0200 Subject: [PATCH 18/35] A minor rename. --- jedi/evaluate/syntax_tree.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index ec775564..240a1b14 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -202,12 +202,12 @@ def eval_atom(context, atom): def eval_expr_stmt(context, stmt, seek_name=None): with recursion.execution_allowed(context.evaluator, stmt) as allowed: if allowed or context.get_root_context() == context.evaluator.BUILTINS: - return _eval_stmt(context, stmt, seek_name) + return _eval_expr_stmt(context, stmt, seek_name) return NO_CONTEXTS @debug.increase_indent -def _eval_stmt(context, stmt, seek_name=None): +def _eval_expr_stmt(context, stmt, seek_name=None): """ The starting point of the completion. A statement always owns a call list, which are the calls, that a statement does. In case multiple @@ -216,7 +216,7 @@ def _eval_stmt(context, stmt, seek_name=None): :param stmt: A `tree.ExprStmt`. """ - debug.dbg('eval_statement %s (%s)', stmt, seek_name) + debug.dbg('eval_expr_stmt %s (%s)', stmt, seek_name) rhs = stmt.get_rhs() context_set = context.eval_node(rhs) @@ -253,5 +253,5 @@ def _eval_stmt(context, stmt, seek_name=None): context_set = left else: context_set = precedence.calculate(context.evaluator, context, left, operator, context_set) - debug.dbg('eval_statement result %s', context_set) + debug.dbg('eval_expr_stmt result %s', context_set) return context_set From d0939f0449d56251d9a92043458155ef1c7a4220 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 18:51:53 +0200 Subject: [PATCH 19/35] Move eval_or_test away from precedence module. --- jedi/evaluate/precedence.py | 29 ----------------------------- jedi/evaluate/syntax_tree.py | 27 ++++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/jedi/evaluate/precedence.py b/jedi/evaluate/precedence.py index 7e054c05..54973777 100644 --- a/jedi/evaluate/precedence.py +++ b/jedi/evaluate/precedence.py @@ -4,7 +4,6 @@ Handles operator precedence. import operator as op from jedi._compatibility import unicode -from jedi import debug from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name from jedi.evaluate import analysis from jedi.evaluate.context import ContextSet, NO_CONTEXTS, iterator_to_context_set @@ -37,34 +36,6 @@ def literals_to_types(evaluator, result): return new_result -def calculate_children(evaluator, context, children): - """ - Calculate a list of children with operators. - """ - iterator = iter(children) - types = context.eval_node(next(iterator)) - for operator in iterator: - right = next(iterator) - if operator.type == 'comp_op': # not in / is not - operator = ' '.join(c.value for c in operator.children) - - # handle lazy evaluation of and/or here. - if operator in ('and', 'or'): - left_bools = set(left.py__bool__() for left in types) - if left_bools == set([True]): - if operator == 'and': - types = context.eval_node(right) - elif left_bools == set([False]): - if operator != 'and': - types = context.eval_node(right) - # Otherwise continue, because of uncertainty. - else: - types = calculate(evaluator, context, types, operator, - context.eval_node(right)) - debug.dbg('calculate_children types %s', types) - return types - - def calculate(evaluator, context, left_contexts, operator, right_contexts): if not left_contexts or not right_contexts: # illegal slices e.g. cause left/right_result to be None diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index 240a1b14..84b6c0bf 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -107,7 +107,7 @@ def eval_node(context, element): elif typ == 'annassign': return pep0484._evaluate_for_annotation(context, element.children[1]) else: - return precedence.calculate_children(evaluator, context, element.children) + return eval_or_test(context, element) def eval_trailer(context, base_contexts, trailer): @@ -255,3 +255,28 @@ def _eval_expr_stmt(context, stmt, seek_name=None): context_set = precedence.calculate(context.evaluator, context, left, operator, context_set) debug.dbg('eval_expr_stmt result %s', context_set) return context_set + + +def eval_or_test(context, or_test): + iterator = iter(or_test.children) + types = context.eval_node(next(iterator)) + for operator in iterator: + right = next(iterator) + if operator.type == 'comp_op': # not in / is not + operator = ' '.join(c.value for c in operator.children) + + # handle lazy evaluation of and/or here. + if operator in ('and', 'or'): + left_bools = set(left.py__bool__() for left in types) + if left_bools == set([True]): + if operator == 'and': + types = context.eval_node(right) + elif left_bools == set([False]): + if operator != 'and': + types = context.eval_node(right) + # Otherwise continue, because of uncertainty. + else: + types = precedence.calculate(context.evaluator, context, types, operator, + context.eval_node(right)) + debug.dbg('calculate_children types %s', types) + return types From ed43a68c0330d72e22bb90ad486056eb2c8b37ce Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 19:08:52 +0200 Subject: [PATCH 20/35] Remove the precedence module in favor of the syntax tree module. --- jedi/evaluate/iterable.py | 4 +- jedi/evaluate/precedence.py | 151 -------------------------------- jedi/evaluate/stdlib.py | 4 +- jedi/evaluate/syntax_tree.py | 162 +++++++++++++++++++++++++++++++++-- jedi/evaluate/sys_path.py | 2 +- 5 files changed, 159 insertions(+), 164 deletions(-) delete mode 100644 jedi/evaluate/precedence.py diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 524d843f..9e7fec4c 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -29,7 +29,7 @@ from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate import analysis from jedi.evaluate import pep0484 -from jedi.evaluate import precedence +from jedi.evaluate.syntax_tree import is_string from jedi.evaluate import recursion from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.filters import DictFilter, AbstractNameDefinition, \ @@ -462,7 +462,7 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence): """ for key_node, value in self._items(): for key in self._defining_context.eval_node(key_node): - if precedence.is_string(key): + if is_string(key): yield key.obj, context.LazyTreeContext(self._defining_context, value) def __repr__(self): diff --git a/jedi/evaluate/precedence.py b/jedi/evaluate/precedence.py deleted file mode 100644 index 54973777..00000000 --- a/jedi/evaluate/precedence.py +++ /dev/null @@ -1,151 +0,0 @@ -""" -Handles operator precedence. -""" -import operator as op - -from jedi._compatibility import unicode -from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name -from jedi.evaluate import analysis -from jedi.evaluate.context import ContextSet, NO_CONTEXTS, iterator_to_context_set - -# Maps Python syntax to the operator module. -COMPARISON_OPERATORS = { - '==': op.eq, - '!=': op.ne, - 'is': op.is_, - 'is not': op.is_not, - '<': op.lt, - '<=': op.le, - '>': op.gt, - '>=': op.ge, -} - - -def literals_to_types(evaluator, result): - # Changes literals ('a', 1, 1.0, etc) to its type instances (str(), - # int(), float(), etc). - new_result = NO_CONTEXTS - for typ in result: - if is_literal(typ): - # Literals are only valid as long as the operations are - # correct. Otherwise add a value-free instance. - cls = builtin_from_name(evaluator, typ.name.string_name) - new_result |= cls.execute_evaluated() - else: - new_result |= ContextSet(typ) - return new_result - - -def calculate(evaluator, context, left_contexts, operator, right_contexts): - if not left_contexts or not right_contexts: - # illegal slices e.g. cause left/right_result to be None - result = (left_contexts or NO_CONTEXTS) | (right_contexts or NO_CONTEXTS) - return literals_to_types(evaluator, result) - else: - # I don't think there's a reasonable chance that a string - # operation is still correct, once we pass something like six - # objects. - if len(left_contexts) * len(right_contexts) > 6: - return literals_to_types(evaluator, left_contexts | right_contexts) - else: - return ContextSet.from_sets( - _element_calculate(evaluator, context, left, operator, right) - for left in left_contexts - for right in right_contexts - ) - - -@iterator_to_context_set -def factor_calculate(evaluator, types, operator): - """ - Calculates `+`, `-`, `~` and `not` prefixes. - """ - for typ in types: - if operator == '-': - if _is_number(typ): - yield create(evaluator, -typ.obj) - elif operator == 'not': - value = typ.py__bool__() - if value is None: # Uncertainty. - return - yield create(evaluator, not value) - else: - yield typ - - -def _is_number(obj): - return isinstance(obj, CompiledObject) \ - and isinstance(obj.obj, (int, float)) - - -def is_string(obj): - return isinstance(obj, CompiledObject) \ - and isinstance(obj.obj, (str, unicode)) - - -def is_literal(obj): - return _is_number(obj) or is_string(obj) - - -def _is_tuple(obj): - from jedi.evaluate import iterable - return isinstance(obj, iterable.AbstractSequence) and obj.array_type == 'tuple' - - -def _is_list(obj): - from jedi.evaluate import iterable - return isinstance(obj, iterable.AbstractSequence) and obj.array_type == 'list' - - -def _element_calculate(evaluator, context, left, operator, right): - from jedi.evaluate import iterable, instance - l_is_num = _is_number(left) - r_is_num = _is_number(right) - if operator == '*': - # for iterables, ignore * operations - if isinstance(left, iterable.AbstractSequence) or is_string(left): - return ContextSet(left) - elif isinstance(right, iterable.AbstractSequence) or is_string(right): - return ContextSet(right) - elif operator == '+': - if l_is_num and r_is_num or is_string(left) and is_string(right): - return ContextSet(create(evaluator, left.obj + right.obj)) - elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right): - return ContextSet(iterable.MergedArray(evaluator, (left, right))) - elif operator == '-': - if l_is_num and r_is_num: - return ContextSet(create(evaluator, left.obj - right.obj)) - elif operator == '%': - # With strings and numbers the left type typically remains. Except for - # `int() % float()`. - return ContextSet(left) - elif operator in COMPARISON_OPERATORS: - operation = COMPARISON_OPERATORS[operator] - if isinstance(left, CompiledObject) and isinstance(right, CompiledObject): - # Possible, because the return is not an option. Just compare. - left = left.obj - right = right.obj - - try: - result = operation(left, right) - except TypeError: - # Could be True or False. - return ContextSet(create(evaluator, True), create(evaluator, False)) - else: - return ContextSet(create(evaluator, result)) - elif operator == 'in': - return NO_CONTEXTS - - def check(obj): - """Checks if a Jedi object is either a float or an int.""" - return isinstance(obj, instance.CompiledInstance) and \ - obj.name.string_name in ('int', 'float') - - # Static analysis, one is a number, the other one is not. - if operator in ('+', '-') and l_is_num != r_is_num \ - and not (check(left) or check(right)): - message = "TypeError: unsupported operand type(s) for +: %s and %s" - analysis.add(context, 'type-error-operation', operator, - message % (left, right)) - - return ContextSet(left, right) diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 447079c1..ab758c79 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -20,11 +20,11 @@ from jedi.evaluate.instance import InstanceFunctionExecution, \ AnonymousInstanceFunctionExecution from jedi.evaluate import iterable from jedi import debug -from jedi.evaluate import precedence from jedi.evaluate import param from jedi.evaluate import analysis from jedi.evaluate.context import LazyTreeContext, ContextualizedNode, \ NO_CONTEXTS, ContextSet +from jedi.evaluate.syntax_tree import is_string # Now this is all part of fake tuples in Jedi. However super doesn't work on # __init__ and __new__ doesn't work at all. So adding this to nametuples is @@ -156,7 +156,7 @@ def builtins_getattr(evaluator, objects, names, defaults=None): # follow the first param for obj in objects: for name in names: - if precedence.is_string(name): + if is_string(name): return obj.py__getattribute__(name.obj) else: debug.warning('getattr called without str') diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index 84b6c0bf..0c4ad194 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -2,18 +2,19 @@ Functions evaluating the syntax tree. """ import copy +import operator as op from parso.python import tree from jedi import debug from jedi import parser_utils from jedi.evaluate.context import ContextSet, NO_CONTEXTS, ContextualizedNode, \ - ContextualizedName + ContextualizedName, iterator_to_context_set from jedi.evaluate import compiled -from jedi.evaluate import precedence from jedi.evaluate import pep0484 from jedi.evaluate import recursion from jedi.evaluate import helpers +from jedi.evaluate import analysis def _limit_context_infers(func): @@ -66,7 +67,7 @@ def eval_node(context, element): for trailer in element.children[1:]: if trailer == '**': # has a power operation. right = evaluator.eval_element(context, element.children[2]) - context_set = precedence.calculate( + context_set = calculate( evaluator, context, context_set, @@ -84,7 +85,7 @@ def eval_node(context, element): elif typ in ('not_test', 'factor'): context_set = context.eval_node(element.children[-1]) for operator in element.children[:-1]: - context_set = precedence.factor_calculate(evaluator, context_set, operator) + context_set = eval_factor(context_set, operator) return context_set elif typ == 'test': # `x if foo else y` case. @@ -162,7 +163,7 @@ def eval_atom(context, atom): context_set = eval_atom(context, c[0]) for string in c[1:]: right = eval_atom(context, string) - context_set = precedence.calculate(context.evaluator, context, context_set, '+', right) + context_set = calculate(context.evaluator, context, context_set, '+', right) return context_set # Parentheses without commas are not tuples. elif c[0] == '(' and not len(c) == 2 \ @@ -249,10 +250,10 @@ def _eval_expr_stmt(context, stmt, seek_name=None): dct = {for_stmt.children[1].value: lazy_context.infer()} with helpers.predefine_names(context, for_stmt, dct): t = context.eval_node(rhs) - left = precedence.calculate(context.evaluator, context, left, operator, t) + left = calculate(context.evaluator, context, left, operator, t) context_set = left else: - context_set = precedence.calculate(context.evaluator, context, left, operator, context_set) + context_set = calculate(context.evaluator, context, left, operator, context_set) debug.dbg('eval_expr_stmt result %s', context_set) return context_set @@ -276,7 +277,152 @@ def eval_or_test(context, or_test): types = context.eval_node(right) # Otherwise continue, because of uncertainty. else: - types = precedence.calculate(context.evaluator, context, types, operator, + types = calculate(context.evaluator, context, types, operator, context.eval_node(right)) debug.dbg('calculate_children types %s', types) return types + + +@iterator_to_context_set +def eval_factor(context_set, operator): + """ + Calculates `+`, `-`, `~` and `not` prefixes. + """ + for context in context_set: + if operator == '-': + if _is_number(context): + yield compiled.create(context.evaluator, -context.obj) + elif operator == 'not': + value = context.py__bool__() + if value is None: # Uncertainty. + return + yield compiled.create(context.evaluator, not value) + else: + yield context + + +# Maps Python syntax to the operator module. +COMPARISON_OPERATORS = { + '==': op.eq, + '!=': op.ne, + 'is': op.is_, + 'is not': op.is_not, + '<': op.lt, + '<=': op.le, + '>': op.gt, + '>=': op.ge, +} + + +def literals_to_types(evaluator, result): + # Changes literals ('a', 1, 1.0, etc) to its type instances (str(), + # int(), float(), etc). + new_result = NO_CONTEXTS + for typ in result: + if is_literal(typ): + # Literals are only valid as long as the operations are + # correct. Otherwise add a value-free instance. + cls = compiled.builtin_from_name(evaluator, typ.name.string_name) + new_result |= cls.execute_evaluated() + else: + new_result |= ContextSet(typ) + return new_result + + +def calculate(evaluator, context, left_contexts, operator, right_contexts): + if not left_contexts or not right_contexts: + # illegal slices e.g. cause left/right_result to be None + result = (left_contexts or NO_CONTEXTS) | (right_contexts or NO_CONTEXTS) + return literals_to_types(evaluator, result) + else: + # I don't think there's a reasonable chance that a string + # operation is still correct, once we pass something like six + # objects. + if len(left_contexts) * len(right_contexts) > 6: + return literals_to_types(evaluator, left_contexts | right_contexts) + else: + return ContextSet.from_sets( + _element_calculate(evaluator, context, left, operator, right) + for left in left_contexts + for right in right_contexts + ) + + +def is_compiled(context): + return isinstance(context, compiled.CompiledObject) + + +def _is_number(context): + return is_compiled(context) and isinstance(context.obj, (int, float)) + + +def is_string(context): + return is_compiled(context) and isinstance(context.obj, (str, unicode)) + + +def is_literal(context): + return _is_number(context) or is_string(context) + + +def _is_tuple(context): + from jedi.evaluate import iterable + return isinstance(context, iterable.AbstractSequence) and context.array_type == 'tuple' + + +def _is_list(context): + from jedi.evaluate import iterable + return isinstance(context, iterable.AbstractSequence) and context.array_type == 'list' + + +def _element_calculate(evaluator, context, left, operator, right): + from jedi.evaluate import iterable, instance + l_is_num = _is_number(left) + r_is_num = _is_number(right) + if operator == '*': + # for iterables, ignore * operations + if isinstance(left, iterable.AbstractSequence) or is_string(left): + return ContextSet(left) + elif isinstance(right, iterable.AbstractSequence) or is_string(right): + return ContextSet(right) + elif operator == '+': + if l_is_num and r_is_num or is_string(left) and is_string(right): + return ContextSet(compiled.create(evaluator, left.obj + right.obj)) + elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right): + return ContextSet(iterable.MergedArray(evaluator, (left, right))) + elif operator == '-': + if l_is_num and r_is_num: + return ContextSet(compiled.create(evaluator, left.obj - right.obj)) + elif operator == '%': + # With strings and numbers the left type typically remains. Except for + # `int() % float()`. + return ContextSet(left) + elif operator in COMPARISON_OPERATORS: + operation = COMPARISON_OPERATORS[operator] + if is_compiled(left) and is_compiled(right): + # Possible, because the return is not an option. Just compare. + left = left.obj + right = right.obj + + try: + result = operation(left, right) + except TypeError: + # Could be True or False. + return ContextSet(compiled.create(evaluator, True), compiled.create(evaluator, False)) + else: + return ContextSet(compiled.create(evaluator, result)) + elif operator == 'in': + return NO_CONTEXTS + + def check(obj): + """Checks if a Jedi object is either a float or an int.""" + return isinstance(obj, instance.CompiledInstance) and \ + obj.name.string_name in ('int', 'float') + + # Static analysis, one is a number, the other one is not. + if operator in ('+', '-') and l_is_num != r_is_num \ + and not (check(left) or check(right)): + message = "TypeError: unsupported operand type(s) for +: %s and %s" + analysis.add(context, 'type-error-operation', operator, + message % (left, right)) + + return ContextSet(left, right) diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index 057479ec..e78added 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -121,7 +121,7 @@ def _paths_from_assignment(module_context, expr_stmt): continue from jedi.evaluate.iterable import py__iter__ - from jedi.evaluate.precedence import is_string + from jedi.evaluate.syntax_tree import is_string cn = ContextualizedNode(module_context.create_context(expr_stmt), expr_stmt) for lazy_context in py__iter__(module_context.evaluator, cn.infer(), cn): for context in lazy_context.infer(): From a7a66024d4f85f97fbb6e1ea16c5576446a7808b Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 19:13:19 +0200 Subject: [PATCH 21/35] Make a lot more functions private. --- jedi/evaluate/syntax_tree.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index 0c4ad194..f9ba1847 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -67,7 +67,7 @@ def eval_node(context, element): for trailer in element.children[1:]: if trailer == '**': # has a power operation. right = evaluator.eval_element(context, element.children[2]) - context_set = calculate( + context_set = _eval_comparison( evaluator, context, context_set, @@ -163,7 +163,7 @@ def eval_atom(context, atom): context_set = eval_atom(context, c[0]) for string in c[1:]: right = eval_atom(context, string) - context_set = calculate(context.evaluator, context, context_set, '+', right) + context_set = _eval_comparison(context.evaluator, context, context_set, '+', right) return context_set # Parentheses without commas are not tuples. elif c[0] == '(' and not len(c) == 2 \ @@ -250,10 +250,10 @@ def _eval_expr_stmt(context, stmt, seek_name=None): dct = {for_stmt.children[1].value: lazy_context.infer()} with helpers.predefine_names(context, for_stmt, dct): t = context.eval_node(rhs) - left = calculate(context.evaluator, context, left, operator, t) + left = _eval_comparison(context.evaluator, context, left, operator, t) context_set = left else: - context_set = calculate(context.evaluator, context, left, operator, context_set) + context_set = _eval_comparison(context.evaluator, context, left, operator, context_set) debug.dbg('eval_expr_stmt result %s', context_set) return context_set @@ -277,9 +277,9 @@ def eval_or_test(context, or_test): types = context.eval_node(right) # Otherwise continue, because of uncertainty. else: - types = calculate(context.evaluator, context, types, operator, + types = _eval_comparison(context.evaluator, context, types, operator, context.eval_node(right)) - debug.dbg('calculate_children types %s', types) + debug.dbg('eval_or_test types %s', types) return types @@ -314,12 +314,12 @@ COMPARISON_OPERATORS = { } -def literals_to_types(evaluator, result): +def _literals_to_types(evaluator, result): # Changes literals ('a', 1, 1.0, etc) to its type instances (str(), # int(), float(), etc). new_result = NO_CONTEXTS for typ in result: - if is_literal(typ): + if _is_literal(typ): # Literals are only valid as long as the operations are # correct. Otherwise add a value-free instance. cls = compiled.builtin_from_name(evaluator, typ.name.string_name) @@ -329,38 +329,38 @@ def literals_to_types(evaluator, result): return new_result -def calculate(evaluator, context, left_contexts, operator, right_contexts): +def _eval_comparison(evaluator, context, left_contexts, operator, right_contexts): if not left_contexts or not right_contexts: # illegal slices e.g. cause left/right_result to be None result = (left_contexts or NO_CONTEXTS) | (right_contexts or NO_CONTEXTS) - return literals_to_types(evaluator, result) + return _literals_to_types(evaluator, result) else: # I don't think there's a reasonable chance that a string # operation is still correct, once we pass something like six # objects. if len(left_contexts) * len(right_contexts) > 6: - return literals_to_types(evaluator, left_contexts | right_contexts) + return _literals_to_types(evaluator, left_contexts | right_contexts) else: return ContextSet.from_sets( - _element_calculate(evaluator, context, left, operator, right) + _eval_comparison_part(evaluator, context, left, operator, right) for left in left_contexts for right in right_contexts ) -def is_compiled(context): +def _is_compiled(context): return isinstance(context, compiled.CompiledObject) def _is_number(context): - return is_compiled(context) and isinstance(context.obj, (int, float)) + return _is_compiled(context) and isinstance(context.obj, (int, float)) def is_string(context): - return is_compiled(context) and isinstance(context.obj, (str, unicode)) + return _is_compiled(context) and isinstance(context.obj, (str, unicode)) -def is_literal(context): +def _is_literal(context): return _is_number(context) or is_string(context) @@ -374,7 +374,7 @@ def _is_list(context): return isinstance(context, iterable.AbstractSequence) and context.array_type == 'list' -def _element_calculate(evaluator, context, left, operator, right): +def _eval_comparison_part(evaluator, context, left, operator, right): from jedi.evaluate import iterable, instance l_is_num = _is_number(left) r_is_num = _is_number(right) @@ -398,7 +398,7 @@ def _element_calculate(evaluator, context, left, operator, right): return ContextSet(left) elif operator in COMPARISON_OPERATORS: operation = COMPARISON_OPERATORS[operator] - if is_compiled(left) and is_compiled(right): + if _is_compiled(left) and _is_compiled(right): # Possible, because the return is not an option. Just compare. left = left.obj right = right.obj From 9073f0debc06a26dac7a8755e4a9ccab25fcad88 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 19:16:05 +0200 Subject: [PATCH 22/35] Use the typical ordering of arguments for ClassContext. --- jedi/evaluate/__init__.py | 4 ++-- jedi/evaluate/representation.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 470f439f..6eeebf6e 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -245,7 +245,7 @@ class Evaluator(object): if def_ is not None: type_ = def_.type if type_ == 'classdef': - return [er.ClassContext(self, name.parent, context)] + return [er.ClassContext(self, context, name.parent)] elif type_ == 'funcdef': return [er.FunctionContext(self, context, name.parent)] @@ -378,7 +378,7 @@ class Evaluator(object): return func.get_function_execution() return func elif scope_node.type == 'classdef': - class_context = er.ClassContext(self, scope_node, parent_context) + class_context = er.ClassContext(self, parent_context, scope_node) if child_is_funcdef: # anonymous instance return AnonymousInstance(self, parent_context, class_context) diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index fb7e81d5..cb4525b1 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -113,7 +113,7 @@ class ClassContext(use_metaclass(CachedMetaClass, context.TreeContext)): """ api_type = 'class' - def __init__(self, evaluator, classdef, parent_context): + def __init__(self, evaluator, parent_context, classdef): super(ClassContext, self).__init__(evaluator, parent_context=parent_context) self.tree_node = classdef From 0782a80ceffc8e73ffe3fc810b0e6f3e7aaa0b07 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 27 Sep 2017 19:22:50 +0200 Subject: [PATCH 23/35] Move all the search to py__getattribute__ and remove find_types. --- jedi/evaluate/__init__.py | 27 ++++++--------------------- jedi/evaluate/context.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 6eeebf6e..d9d42c01 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -12,10 +12,12 @@ Evaluation of Python code in |jedi| is based on three assumptions: * The programmer is not a total dick, e.g. like `this `_ :-) -The actual algorithm is based on a principle called lazy evaluation. If you -don't know about it, google it. That said, the typical entry point for static -analysis is calling ``eval_expr_stmt``. There's separate logic for -autocompletion in the API, the evaluator is all about evaluating an expression. +The actual algorithm is based on a principle called lazy evaluation. That +said, the typical entry point for static analysis is calling +``eval_expr_stmt``. There's separate logic for autocompletion in the API, the +evaluator is all about evaluating an expression. + +TODO this paragraph is not what jedi does anymore. Now you need to understand what follows after ``eval_expr_stmt``. Let's make an example:: @@ -117,23 +119,6 @@ class Evaluator(object): self.recursion_detector = recursion.RecursionDetector() self.execution_recursion_detector = recursion.ExecutionRecursionDetector(self) - def find_types(self, context, name_or_str, name_context, position=None, - search_global=False, is_goto=False, analysis_errors=True): - """ - This is the search function. The most important part to debug. - `remove_statements` and `filter_statements` really are the core part of - this completion. - - :param position: Position of the last statement -> tuple of line, column - :return: List of Names. Their parents are the types. - """ - f = finder.NameFinder(self, context, name_context, name_or_str, - position, analysis_errors=analysis_errors) - filters = f.get_filters(search_global) - if is_goto: - return f.filter_name(filters) - return f.find(filters, attribute_lookup=not search_global) - def eval_element(self, context, element): if isinstance(context, iterable.CompForContext): return eval_node(context, element) diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index 414fa5e1..117b189d 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -50,8 +50,21 @@ class Context(object): def py__getattribute__(self, name_or_str, name_context=None, position=None, search_global=False, is_goto=False, analysis_errors=True): + """ + This is the search function. + + :param position: Position of the last statement -> tuple of line, column + """ if name_context is None: name_context = self + from jedi.evaluate import finder + f = finder.NameFinder(self.evaluator, self, name_context, name_or_str, + position, analysis_errors=analysis_errors) + filters = f.get_filters(search_global) + if is_goto: + return f.filter_name(filters) + return f.find(filters, attribute_lookup=not search_global) + return self.evaluator.find_types( self, name_or_str, name_context, position, search_global, is_goto, analysis_errors) From d9d3aeb5bc2e104b7284fe7e03d8c8072e73aa9c Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 09:16:43 +0200 Subject: [PATCH 24/35] Move more functions to the syntax tree module. --- jedi/api/__init__.py | 5 +- jedi/evaluate/__init__.py | 6 +- jedi/evaluate/filters.py | 4 +- jedi/evaluate/finder.py | 138 +------------------------------ jedi/evaluate/representation.py | 4 +- jedi/evaluate/syntax_tree.py | 141 +++++++++++++++++++++++++++++++- 6 files changed, 149 insertions(+), 149 deletions(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index ac1132be..0c21c1d9 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -34,6 +34,7 @@ from jedi.evaluate.helpers import get_module_names, evaluate_call_of_leaf from jedi.evaluate.sys_path import get_venv_path, dotted_path_in_sys_path from jedi.evaluate.iterable import unpack_tuple_to_dict from jedi.evaluate.filters import TreeNameDefinition +from jedi.evaluate.syntax_tree import tree_name_to_contexts # Jedi uses lots and lots of recursion. By setting this a little bit higher, we # can remove some "maximum recursion depth" errors. @@ -348,10 +349,8 @@ class Script(object): for node in get_executable_nodes(module_node): context = self._get_module().create_context(node) if node.type in ('funcdef', 'classdef'): - # TODO This is stupid, should be private - from jedi.evaluate.finder import _name_to_types # Resolve the decorators. - _name_to_types(self._evaluator, context, node.children[1]) + tree_name_to_contexts(self._evaluator, context, node.children[1]) elif isinstance(node, tree.Import): import_names = set(node.get_defined_names()) if node.is_nested(): diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index d9d42c01..39c2fa61 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -76,14 +76,14 @@ from jedi.evaluate import recursion from jedi.evaluate import iterable from jedi.evaluate.cache import evaluator_function_cache from jedi.evaluate import stdlib -from jedi.evaluate import finder from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate.filters import TreeNameDefinition, ParamName from jedi.evaluate.instance import AnonymousInstance, BoundMethod from jedi.evaluate.context import ContextualizedName, ContextualizedNode, \ ContextSet, NO_CONTEXTS -from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt, eval_node +from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt, \ + eval_node, check_tuple_assignments from jedi import parser_utils @@ -243,7 +243,7 @@ class Evaluator(object): cn = ContextualizedNode(context, def_.children[3]) for_types = iterable.py__iter__types(self, container_types, cn) c_node = ContextualizedName(context, name) - return finder.check_tuple_assignments(self, c_node, for_types) + return check_tuple_assignments(self, c_node, for_types) if type_ in ('import_from', 'import_name'): return imports.infer_import(context, name) diff --git a/jedi/evaluate/filters.py b/jedi/evaluate/filters.py index f4c6463b..735b27b7 100644 --- a/jedi/evaluate/filters.py +++ b/jedi/evaluate/filters.py @@ -94,8 +94,8 @@ class TreeNameDefinition(AbstractTreeName): def infer(self): # Refactor this, should probably be here. - from jedi.evaluate.finder import _name_to_types - return _name_to_types(self.parent_context.evaluator, self.parent_context, self.tree_name) + from jedi.evaluate.syntax_tree import tree_name_to_contexts + return tree_name_to_contexts(self.parent_context.evaluator, self.parent_context, self.tree_name) @property def api_type(self): diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index 19ac6f25..9f59c22a 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -19,20 +19,16 @@ from parso.python import tree from parso.tree import search_ancestor from jedi import debug from jedi import settings -from jedi.evaluate import representation as er from jedi.evaluate.instance import AbstractInstanceContext from jedi.evaluate import compiled -from jedi.evaluate import pep0484 from jedi.evaluate import iterable -from jedi.evaluate import imports from jedi.evaluate import analysis from jedi.evaluate import flow_analysis from jedi.evaluate import param from jedi.evaluate import helpers from jedi.evaluate.filters import get_global_filters, TreeNameDefinition -from jedi.evaluate.context import ContextualizedName, ContextualizedNode, ContextSet +from jedi.evaluate.context import ContextSet from jedi.parser_utils import is_scope, get_parent_scope -from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt class NameFinder(object): @@ -183,117 +179,6 @@ class NameFinder(object): return contexts -def _name_to_types(evaluator, context, tree_name): - types = [] - node = tree_name.get_definition(import_name_always=True) - if node is None: - node = tree_name.parent - if node.type == 'global_stmt': - context = evaluator.create_context(context, tree_name) - finder = NameFinder(evaluator, context, context, tree_name.value) - filters = finder.get_filters(search_global=True) - # For global_stmt lookups, we only need the first possible scope, - # which means the function itself. - filters = [next(filters)] - return finder.find(filters, attribute_lookup=False) - elif node.type not in ('import_from', 'import_name'): - raise ValueError("Should not happen.") - - typ = node.type - if typ == 'for_stmt': - types = pep0484.find_type_from_comment_hint_for(context, node, tree_name) - if types: - return types - if typ == 'with_stmt': - types = pep0484.find_type_from_comment_hint_with(context, node, tree_name) - if types: - return types - - if typ in ('for_stmt', 'comp_for'): - try: - types = context.predefined_names[node][tree_name.value] - except KeyError: - cn = ContextualizedNode(context, node.children[3]) - for_types = iterable.py__iter__types(evaluator, cn.infer(), cn) - c_node = ContextualizedName(context, tree_name) - types = check_tuple_assignments(evaluator, c_node, for_types) - elif typ == 'expr_stmt': - types = _remove_statements(evaluator, context, node, tree_name) - elif typ == 'with_stmt': - context_managers = context.eval_node(node.get_test_node_from_name(tree_name)) - enter_methods = context_managers.py__getattribute__('__enter__') - return enter_methods.execute_evaluated() - elif typ in ('import_from', 'import_name'): - types = imports.infer_import(context, tree_name) - elif typ in ('funcdef', 'classdef'): - types = _apply_decorators(evaluator, context, node) - elif typ == 'try_stmt': - # TODO an exception can also be a tuple. Check for those. - # TODO check for types that are not classes and add it to - # the static analysis report. - exceptions = context.eval_node(tree_name.get_previous_sibling().get_previous_sibling()) - types = exceptions.execute_evaluated() - else: - raise ValueError("Should not happen.") - return types - - -def _apply_decorators(evaluator, context, node): - """ - Returns the function, that should to be executed in the end. - This is also the places where the decorators are processed. - """ - if node.type == 'classdef': - decoratee_context = er.ClassContext( - evaluator, - parent_context=context, - classdef=node - ) - else: - decoratee_context = er.FunctionContext( - evaluator, - parent_context=context, - funcdef=node - ) - initial = values = ContextSet(decoratee_context) - for dec in reversed(node.get_decorators()): - debug.dbg('decorator: %s %s', dec, values) - dec_values = context.eval_node(dec.children[1]) - trailer_nodes = dec.children[2:-1] - if trailer_nodes: - # Create a trailer and evaluate it. - trailer = tree.PythonNode('trailer', trailer_nodes) - trailer.parent = dec - dec_values = eval_trailer(context, dec_values, trailer) - - if not len(dec_values): - debug.warning('decorator not found: %s on %s', dec, node) - return initial - - values = dec_values.execute(param.ValuesArguments([values])) - if not len(values): - debug.warning('not possible to resolve wrappers found %s', node) - return initial - - debug.dbg('decorator end %s', values) - return values - - -def _remove_statements(evaluator, context, stmt, name): - """ - This is the part where statements are being stripped. - - Due to lazy evaluation, statements like a = func; b = a; b() have to be - evaluated. - """ - pep0484_contexts = \ - pep0484.find_type_from_comment_hint_assign(context, stmt, name) - if pep0484_contexts: - return pep0484_contexts - - return eval_expr_stmt(context, stmt, seek_name=name) - - def _check_flow_information(context, flow, search_name, pos): """ Try to find out the type of a variable just with the information that is given by the flows: e.g. It is also responsible for assert checks.:: @@ -371,24 +256,3 @@ def _check_isinstance_type(context, element, search_name): else: context_set |= cls_or_tup.execute_evaluated() return context_set - - -def check_tuple_assignments(evaluator, contextualized_name, context_set): - """ - Checks if tuples are assigned. - """ - lazy_context = None - for index, node in contextualized_name.assignment_indexes(): - cn = ContextualizedNode(contextualized_name.context, node) - iterated = iterable.py__iter__(evaluator, context_set, cn) - for _ in range(index + 1): - try: - lazy_context = next(iterated) - except StopIteration: - # We could do this with the default param in next. But this - # would allow this loop to run for a very long time if the - # index number is high. Therefore break if the loop is - # finished. - return ContextSet() - context_set = lazy_context.infer() - return context_set diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index cb4525b1..a7f32855 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -89,8 +89,8 @@ class ClassName(TreeNameDefinition): # TODO this _name_to_types might get refactored and be a part of the # parent class. Once it is, we can probably just overwrite method to # achieve this. - from jedi.evaluate.finder import _name_to_types - inferred = _name_to_types( + from jedi.evaluate.syntax_tree import tree_name_to_contexts + inferred = tree_name_to_contexts( self.parent_context.evaluator, self._name_context, self.tree_name) for result_context in inferred: diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index f9ba1847..4f310a14 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -223,8 +223,7 @@ def _eval_expr_stmt(context, stmt, seek_name=None): if seek_name: c_node = ContextualizedName(context, seek_name) - from jedi.evaluate import finder - context_set = finder.check_tuple_assignments(context.evaluator, c_node, context_set) + context_set = check_tuple_assignments(context.evaluator, c_node, context_set) first_operator = next(stmt.yield_operators(), None) if first_operator not in ('=', None) and first_operator.type == 'operator': @@ -426,3 +425,141 @@ def _eval_comparison_part(evaluator, context, left, operator, right): message % (left, right)) return ContextSet(left, right) + + +def _remove_statements(evaluator, context, stmt, name): + """ + This is the part where statements are being stripped. + + Due to lazy evaluation, statements like a = func; b = a; b() have to be + evaluated. + """ + pep0484_contexts = \ + pep0484.find_type_from_comment_hint_assign(context, stmt, name) + if pep0484_contexts: + return pep0484_contexts + + return eval_expr_stmt(context, stmt, seek_name=name) + + +def tree_name_to_contexts(evaluator, context, tree_name): + types = [] + node = tree_name.get_definition(import_name_always=True) + if node is None: + node = tree_name.parent + if node.type == 'global_stmt': + context = evaluator.create_context(context, tree_name) + from jedi.evaluate.finder import NameFinder + finder = NameFinder(evaluator, context, context, tree_name.value) + filters = finder.get_filters(search_global=True) + # For global_stmt lookups, we only need the first possible scope, + # which means the function itself. + filters = [next(filters)] + return finder.find(filters, attribute_lookup=False) + elif node.type not in ('import_from', 'import_name'): + raise ValueError("Should not happen.") + + typ = node.type + if typ == 'for_stmt': + types = pep0484.find_type_from_comment_hint_for(context, node, tree_name) + if types: + return types + if typ == 'with_stmt': + types = pep0484.find_type_from_comment_hint_with(context, node, tree_name) + if types: + return types + + if typ in ('for_stmt', 'comp_for'): + from jedi.evaluate import iterable + try: + types = context.predefined_names[node][tree_name.value] + except KeyError: + cn = ContextualizedNode(context, node.children[3]) + for_types = iterable.py__iter__types(evaluator, cn.infer(), cn) + c_node = ContextualizedName(context, tree_name) + types = check_tuple_assignments(evaluator, c_node, for_types) + elif typ == 'expr_stmt': + types = _remove_statements(evaluator, context, node, tree_name) + elif typ == 'with_stmt': + context_managers = context.eval_node(node.get_test_node_from_name(tree_name)) + enter_methods = context_managers.py__getattribute__('__enter__') + return enter_methods.execute_evaluated() + elif typ in ('import_from', 'import_name'): + from jedi.evaluate import imports + types = imports.infer_import(context, tree_name) + elif typ in ('funcdef', 'classdef'): + types = _apply_decorators(evaluator, context, node) + elif typ == 'try_stmt': + # TODO an exception can also be a tuple. Check for those. + # TODO check for types that are not classes and add it to + # the static analysis report. + exceptions = context.eval_node(tree_name.get_previous_sibling().get_previous_sibling()) + types = exceptions.execute_evaluated() + else: + raise ValueError("Should not happen.") + return types + + +def _apply_decorators(evaluator, context, node): + """ + Returns the function, that should to be executed in the end. + This is also the places where the decorators are processed. + """ + from jedi.evaluate import representation as er + if node.type == 'classdef': + decoratee_context = er.ClassContext( + evaluator, + parent_context=context, + classdef=node + ) + else: + decoratee_context = er.FunctionContext( + evaluator, + parent_context=context, + funcdef=node + ) + initial = values = ContextSet(decoratee_context) + for dec in reversed(node.get_decorators()): + debug.dbg('decorator: %s %s', dec, values) + dec_values = context.eval_node(dec.children[1]) + trailer_nodes = dec.children[2:-1] + if trailer_nodes: + # Create a trailer and evaluate it. + trailer = tree.PythonNode('trailer', trailer_nodes) + trailer.parent = dec + dec_values = eval_trailer(context, dec_values, trailer) + + if not len(dec_values): + debug.warning('decorator not found: %s on %s', dec, node) + return initial + + from jedi.evaluate import param + values = dec_values.execute(param.ValuesArguments([values])) + if not len(values): + debug.warning('not possible to resolve wrappers found %s', node) + return initial + + debug.dbg('decorator end %s', values) + return values + + +def check_tuple_assignments(evaluator, contextualized_name, context_set): + """ + Checks if tuples are assigned. + """ + lazy_context = None + for index, node in contextualized_name.assignment_indexes(): + cn = ContextualizedNode(contextualized_name.context, node) + from jedi.evaluate import iterable + iterated = iterable.py__iter__(evaluator, context_set, cn) + for _ in range(index + 1): + try: + lazy_context = next(iterated) + except StopIteration: + # We could do this with the default param in next. But this + # would allow this loop to run for a very long time if the + # index number is high. Therefore break if the loop is + # finished. + return ContextSet() + context_set = lazy_context.infer() + return context_set From 1c9060ebc59ce51c042463f10896b51aa59f3fc5 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 09:18:12 +0200 Subject: [PATCH 25/35] Remove evaluator as param from apply_decorators. --- jedi/evaluate/syntax_tree.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index 4f310a14..66ee823c 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -6,6 +6,7 @@ import operator as op from parso.python import tree +from jedi._compatibility import unicode from jedi import debug from jedi import parser_utils from jedi.evaluate.context import ContextSet, NO_CONTEXTS, ContextualizedNode, \ @@ -488,7 +489,7 @@ def tree_name_to_contexts(evaluator, context, tree_name): from jedi.evaluate import imports types = imports.infer_import(context, tree_name) elif typ in ('funcdef', 'classdef'): - types = _apply_decorators(evaluator, context, node) + types = _apply_decorators(context, node) elif typ == 'try_stmt': # TODO an exception can also be a tuple. Check for those. # TODO check for types that are not classes and add it to @@ -500,7 +501,7 @@ def tree_name_to_contexts(evaluator, context, tree_name): return types -def _apply_decorators(evaluator, context, node): +def _apply_decorators(context, node): """ Returns the function, that should to be executed in the end. This is also the places where the decorators are processed. @@ -508,13 +509,13 @@ def _apply_decorators(evaluator, context, node): from jedi.evaluate import representation as er if node.type == 'classdef': decoratee_context = er.ClassContext( - evaluator, + context.evaluator, parent_context=context, classdef=node ) else: decoratee_context = er.FunctionContext( - evaluator, + context.evaluator, parent_context=context, funcdef=node ) From b08300813e24ffeadf86b27381d1728d3d68f22d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 10:39:54 +0200 Subject: [PATCH 26/35] Fix an issue surrounding namedtuples where I didn't see the tests failing. --- jedi/evaluate/stdlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index ab758c79..205a9f36 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -295,7 +295,7 @@ def collections_namedtuple(evaluator, obj, arguments): module = evaluator.grammar.parse(source) generated_class = next(module.iter_classdefs()) parent_context = er.ModuleContext(evaluator, module, '') - return ContextSet(er.ClassContext(evaluator, generated_class, parent_context)) + return ContextSet(er.ClassContext(evaluator, parent_context, generated_class)) @argument_clinic('first, /') From 47c249957d5a7f6c9ad50b73c59ed8d6ddd14dc4 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 12:04:44 +0200 Subject: [PATCH 27/35] Make BuiltinMethod a Context object. --- jedi/evaluate/__init__.py | 24 ---------------------- jedi/evaluate/compiled/__init__.py | 2 +- jedi/evaluate/context.py | 33 +++++++++++++++++++++++++++++- jedi/evaluate/iterable.py | 20 ++++++++---------- 4 files changed, 42 insertions(+), 37 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 39c2fa61..6bde42b7 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -75,7 +75,6 @@ from jedi.evaluate import imports from jedi.evaluate import recursion from jedi.evaluate import iterable from jedi.evaluate.cache import evaluator_function_cache -from jedi.evaluate import stdlib from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate.filters import TreeNameDefinition, ParamName @@ -202,29 +201,6 @@ class Evaluator(object): def _eval_element_cached(self, context, element): return eval_node(context, element) - @debug.increase_indent - def execute(self, obj, arguments): - if self.is_analysis: - arguments.eval_all() - - debug.dbg('execute: %s %s', obj, arguments) - try: - # Some stdlib functions like super(), namedtuple(), etc. have been - # hard-coded in Jedi to support them. - return stdlib.execute(self, obj, arguments) - except stdlib.NotInStdLib: - pass - - try: - func = obj.py__call__ - except AttributeError: - debug.warning("no execution possible %s", obj) - return NO_CONTEXTS - else: - context_set = func(arguments) - debug.dbg('execute result: %s in %s', context_set, obj) - return context_set - def goto_definitions(self, context, name): def_ = name.get_definition(import_name_always=True) if def_ is not None: diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index ab9ea8f5..56faef5b 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -266,7 +266,7 @@ class CompiledObject(Context): # TODO do we? continue bltn_obj = create(self.evaluator, bltn_obj) - for result in self.evaluator.execute(bltn_obj, params): + for result in bltn_obj.execute(params): yield result for type_ in docstrings.infer_return_types(self): yield type_ diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index 117b189d..9d62a51b 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -1,5 +1,7 @@ -from jedi._compatibility import Python3Method from parso.python.tree import ExprStmt, CompFor + +from jedi import debug +from jedi._compatibility import Python3Method from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature from jedi.common import BaseContextSet @@ -32,7 +34,36 @@ class Context(object): return context context = context.parent_context + @debug.increase_indent def execute(self, arguments): + """ + In contrast to py__call__ this function is always available. + + `hasattr(x, py__call__)` can also be checked to see if a context is + executable. + """ + if self.evaluator.is_analysis: + arguments.eval_all() + + debug.dbg('execute: %s %s', self, arguments) + from jedi.evaluate import stdlib + try: + # Some stdlib functions like super(), namedtuple(), etc. have been + # hard-coded in Jedi to support them. + return stdlib.execute(self.evaluator, self, arguments) + except stdlib.NotInStdLib: + pass + + try: + func = self.py__call__ + except AttributeError: + debug.warning("no execution possible %s", self) + return NO_CONTEXTS + else: + context_set = func(arguments) + debug.dbg('execute result: %s in %s', context_set, self) + return context_set + return self.evaluator.execute(self, arguments) def execute_evaluated(self, *value_list): diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 9e7fec4c..bb9a65f1 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -35,7 +35,7 @@ from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.filters import DictFilter, AbstractNameDefinition, \ ParserTreeFilter from jedi.evaluate import context -from jedi.evaluate.context import ContextSet, NO_CONTEXTS +from jedi.evaluate.context import ContextSet, NO_CONTEXTS, Context from jedi.parser_utils import get_comp_fors @@ -54,22 +54,20 @@ class AbstractSequence(context.Context): return compiled.CompiledContextName(self, self.array_type) -class BuiltinMethod(object): +class BuiltinMethod(Context): """``Generator.__next__`` ``dict.values`` methods and so on.""" + api_type = 'function' + def __init__(self, builtin_context, method, builtin_func): - self._builtin_context = builtin_context + super(BuiltinMethod, self).__init__( + builtin_context.evaluator, + parent_context=builtin_context + ) self._method = method self._builtin_func = builtin_func - # TODO it seems kind of stupid that we have to overwrite 3 methods here. def py__call__(self, params): - return self._method(self._builtin_context) - - def execute(self, *args, **kwargs): - return self._builtin_context.evaluator.execute(self, *args, **kwargs) - - def execute_evaluated(self, *args, **kwargs): - return self._builtin_context.evaluator.execute_evaluated(self, *args, **kwargs) + return self._method(self.parent_context) def __getattr__(self, name): return getattr(self._builtin_func, name) From 8c0845cf0ce2f2d2fef3287724b1dd48f899a2f8 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 13:13:09 +0200 Subject: [PATCH 28/35] Move iterate logic to the context. --- jedi/evaluate/context.py | 25 ++++++++++++++++++- jedi/evaluate/iterable.py | 43 +++++++++------------------------ jedi/evaluate/representation.py | 5 ++-- jedi/evaluate/stdlib.py | 7 +++--- jedi/evaluate/syntax_tree.py | 6 ++--- jedi/evaluate/sys_path.py | 3 +-- 6 files changed, 44 insertions(+), 45 deletions(-) diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index 9d62a51b..e18474c7 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -1,7 +1,7 @@ from parso.python.tree import ExprStmt, CompFor from jedi import debug -from jedi._compatibility import Python3Method +from jedi._compatibility import Python3Method, zip_longest from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature from jedi.common import BaseContextSet @@ -66,6 +66,22 @@ class Context(object): return self.evaluator.execute(self, arguments) + def iterate(self, contextualized_node=None): + debug.dbg('iterate') + try: + iter_method = self.py__iter__ + except AttributeError: + if contextualized_node is not None: + from jedi.evaluate import analysis + analysis.add( + contextualized_node.context, + 'type-error-not-iterable', + contextualized_node._node, + message="TypeError: '%s' object is not iterable" % self) + return iter([]) + else: + return iter_method() + def execute_evaluated(self, *value_list): """ Execute a function with already executed arguments. @@ -248,6 +264,13 @@ class ContextSet(BaseContextSet): def py__class__(self): return ContextSet.from_iterable(c.py__class__() for c in self._set) + def iterate(self, contextualized_node=None): + type_iters = [c.iterate(contextualized_node) for c in self._set] + for lazy_contexts in zip_longest(*type_iters): + yield get_merged_lazy_context( + [l for l in lazy_contexts if l is not None] + ) + NO_CONTEXTS = ContextSet() diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index bb9a65f1..465a1226 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -23,7 +23,7 @@ It is important to note that: from jedi import debug from jedi import settings from jedi.evaluate.utils import safe_property -from jedi._compatibility import unicode, zip_longest, is_py3 +from jedi._compatibility import unicode, is_py3 from jedi.evaluate.utils import to_list from jedi.evaluate import compiled from jedi.evaluate import helpers @@ -240,14 +240,13 @@ class Comprehension(AbstractSequence): return CompForContext.from_comp_for(parent_context, comp_for) def _nested(self, comp_fors, parent_context=None): - evaluator = self.evaluator comp_for = comp_fors[0] input_node = comp_for.children[3] parent_context = parent_context or self._defining_context input_types = parent_context.eval_node(input_node) cn = context.ContextualizedNode(parent_context, input_node) - iterated = py__iter__(evaluator, input_types, cn) + iterated = input_types.iterate(cn) exprlist = comp_for.children[1] for i, lazy_context in enumerate(iterated): types = lazy_context.infer() @@ -587,7 +586,7 @@ def unpack_tuple_to_dict(context, types, exprlist): dct = {} parts = iter(exprlist.children[::2]) n = 0 - for lazy_context in py__iter__(context.evaluator, types, exprlist): + for lazy_context in types.iterate(exprlist): n += 1 try: part = next(parts) @@ -614,36 +613,14 @@ def unpack_tuple_to_dict(context, types, exprlist): raise NotImplementedError -def py__iter__(evaluator, types, contextualized_node=None): - debug.dbg('py__iter__') - type_iters = [] - for typ in types: - try: - iter_method = typ.py__iter__ - except AttributeError: - if contextualized_node is not None: - analysis.add( - contextualized_node.context, - 'type-error-not-iterable', - contextualized_node._node, - message="TypeError: '%s' object is not iterable" % typ) - else: - type_iters.append(iter_method()) - - for lazy_contexts in zip_longest(*type_iters): - yield context.get_merged_lazy_context( - [l for l in lazy_contexts if l is not None] - ) - - -def py__iter__types(evaluator, types, contextualized_node=None): +def py__iter__types(evaluator, contexts, contextualized_node=None): """ Calls `py__iter__`, but ignores the ordering in the end and just returns all types that it contains. """ return ContextSet.from_sets( lazy_context.infer() - for lazy_context in py__iter__(evaluator, types, contextualized_node) + for lazy_context in contexts.iterate(contextualized_node) ) @@ -682,7 +659,8 @@ def py__getitem__(evaluator, context, types, trailer): if isinstance(typ, AbstractSequence) and typ.array_type == 'dict': types.remove(typ) result |= typ.dict_values() - return result | py__iter__types(evaluator, types) + cs = ContextSet.from_set(types) + return result | py__iter__types(evaluator, cs) for typ in types: # The actual getitem call. @@ -739,7 +717,7 @@ def _check_array_additions(context, sequence): result.add(whatever) elif add_name in ['extend', 'update']: for key, lazy_context in params: - result |= set(py__iter__(context.evaluator, lazy_context.infer())) + result |= set(lazy_context.infer().iterate()) return result temp_param_add, settings.dynamic_params_for_other_modules = \ @@ -827,7 +805,7 @@ class _ArrayInstance(object): except StopIteration: pass else: - for lazy in py__iter__(self.instance.evaluator, lazy_context.infer()): + for lazy in lazy_context.infer().iterate(): yield lazy from jedi.evaluate import param @@ -836,6 +814,9 @@ class _ArrayInstance(object): for addition in additions: yield addition + def iterate(self, contextualized_node=None): + return self.py__iter__() + class Slice(context.Context): def __init__(self, context, start, stop, step): diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index a7f32855..639c4ca2 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -360,7 +360,7 @@ class FunctionExecutionContext(context.TreeContext): node = yield_expr.children[1] if node.type == 'yield_arg': # It must be a yield from. cn = ContextualizedNode(self, node.children[1]) - for lazy_context in iterable.py__iter__(self.evaluator, cn.infer(), cn): + for lazy_context in cn.infer().iterate(cn): yield lazy_context else: yield context.LazyTreeContext(self, node) @@ -395,7 +395,6 @@ class FunctionExecutionContext(context.TreeContext): return last_for_stmt = for_stmt - evaluator = self.evaluator for for_stmt, yields in yields_order: if for_stmt is None: # No for_stmt, just normal yields. @@ -405,7 +404,7 @@ class FunctionExecutionContext(context.TreeContext): else: input_node = for_stmt.get_testlist() cn = ContextualizedNode(self, input_node) - ordered = iterable.py__iter__(evaluator, cn.infer(), cn) + ordered = cn.infer().iterate(cn) ordered = list(ordered) for lazy_context in ordered: dct = {str(for_stmt.children[1].value): lazy_context.infer()} diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 205a9f36..24cc3342 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -12,7 +12,6 @@ compiled module that returns the types for C-builtins. import collections import re -from jedi.evaluate.utils import unite from jedi.evaluate import compiled from jedi.evaluate import representation as er from jedi.evaluate.instance import InstanceFunctionExecution, \ @@ -200,7 +199,7 @@ def builtins_reversed(evaluator, sequences, obj, arguments): if isinstance(lazy_context, LazyTreeContext): # TODO access private cn = ContextualizedNode(lazy_context._context, lazy_context.data) - ordered = list(iterable.py__iter__(evaluator, sequences, cn)) + ordered = list(sequences.iterate(cn)) rev = list(reversed(ordered)) # Repack iterator values and then run it the normal way. This is @@ -232,9 +231,9 @@ def builtins_isinstance(evaluator, objects, types, arguments): elif cls_or_tup.name.string_name == 'tuple' \ and cls_or_tup.get_root_context() == evaluator.BUILTINS: # Check for tuples. - classes = unite( + classes = ContextSet.from_sets( lazy_context.infer() - for lazy_context in cls_or_tup.py__iter__() + for lazy_context in cls_or_tup.iterate() ) bool_results.add(any(cls in mro for cls in classes)) else: diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index 66ee823c..f0f5af10 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -243,8 +243,7 @@ def _eval_expr_stmt(context, stmt, seek_name=None): # predictable. Also only do it, if the variable is not a tuple. node = for_stmt.get_testlist() cn = ContextualizedNode(context, node) - from jedi.evaluate import iterable - ordered = list(iterable.py__iter__(context.evaluator, cn.infer(), cn)) + ordered = list(cn.infer().iterate(cn)) for lazy_context in ordered: dct = {for_stmt.children[1].value: lazy_context.infer()} @@ -551,8 +550,7 @@ def check_tuple_assignments(evaluator, contextualized_name, context_set): lazy_context = None for index, node in contextualized_name.assignment_indexes(): cn = ContextualizedNode(contextualized_name.context, node) - from jedi.evaluate import iterable - iterated = iterable.py__iter__(evaluator, context_set, cn) + iterated = context_set.iterate(cn) for _ in range(index + 1): try: lazy_context = next(iterated) diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index e78added..23e06e39 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -120,10 +120,9 @@ def _paths_from_assignment(module_context, expr_stmt): except AssertionError: continue - from jedi.evaluate.iterable import py__iter__ from jedi.evaluate.syntax_tree import is_string cn = ContextualizedNode(module_context.create_context(expr_stmt), expr_stmt) - for lazy_context in py__iter__(module_context.evaluator, cn.infer(), cn): + for lazy_context in cn.infer().iterate(cn): for context in lazy_context.infer(): if is_string(context): yield context.obj From 30df79e2349493f2eccc9d42b2922d6bd25fb66a Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 13:19:33 +0200 Subject: [PATCH 29/35] Rename py__iter__types to iterate_contexts. --- jedi/evaluate/__init__.py | 2 +- jedi/evaluate/iterable.py | 10 +++++----- jedi/evaluate/syntax_tree.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 6bde42b7..ecc6d66b 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -217,7 +217,7 @@ class Evaluator(object): if type_ == 'for_stmt': container_types = context.eval_node(def_.children[3]) cn = ContextualizedNode(context, def_.children[3]) - for_types = iterable.py__iter__types(self, container_types, cn) + for_types = iterable.iterate_contexts(self, container_types, cn) c_node = ContextualizedName(context, name) return check_tuple_assignments(self, c_node, for_types) if type_ in ('import_from', 'import_name'): diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 465a1226..be30b6a5 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -613,10 +613,10 @@ def unpack_tuple_to_dict(context, types, exprlist): raise NotImplementedError -def py__iter__types(evaluator, contexts, contextualized_node=None): +def iterate_contexts(evaluator, contexts, contextualized_node=None): """ - Calls `py__iter__`, but ignores the ordering in the end and just returns - all types that it contains. + Calls `iterate`, on all contexts but ignores the ordering and just returns + all contexts that the iterate functions yield. """ return ContextSet.from_sets( lazy_context.infer() @@ -660,7 +660,7 @@ def py__getitem__(evaluator, context, types, trailer): types.remove(typ) result |= typ.dict_values() cs = ContextSet.from_set(types) - return result | py__iter__types(evaluator, cs) + return result | iterate_contexts(evaluator, cs) for typ in types: # The actual getitem call. @@ -674,7 +674,7 @@ def py__getitem__(evaluator, context, types, trailer): try: result |= getitem(index) except IndexError: - result |= py__iter__types(evaluator, ContextSet(typ)) + result |= iterate_contexts(evaluator, ContextSet(typ)) except KeyError: # Must be a dict. Lists don't raise KeyErrors. result |= typ.dict_values() diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index f0f5af10..2b9ad9d0 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -475,7 +475,7 @@ def tree_name_to_contexts(evaluator, context, tree_name): types = context.predefined_names[node][tree_name.value] except KeyError: cn = ContextualizedNode(context, node.children[3]) - for_types = iterable.py__iter__types(evaluator, cn.infer(), cn) + for_types = iterable.iterate_contexts(evaluator, cn.infer(), cn) c_node = ContextualizedName(context, tree_name) types = check_tuple_assignments(evaluator, c_node, for_types) elif typ == 'expr_stmt': From 65ef6a3166c82abf04e7cdb07234aaf7b5a998d3 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 14:10:32 +0200 Subject: [PATCH 30/35] Move py__getitem__ to the context module. --- jedi/evaluate/__init__.py | 4 +- jedi/evaluate/context.py | 79 +++++++++++++++++++++++++++++------- jedi/evaluate/iterable.py | 68 ------------------------------- jedi/evaluate/syntax_tree.py | 26 ++++++++++-- 4 files changed, 90 insertions(+), 87 deletions(-) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index ecc6d66b..3f0b9ec8 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -80,7 +80,7 @@ from jedi.evaluate import helpers from jedi.evaluate.filters import TreeNameDefinition, ParamName from jedi.evaluate.instance import AnonymousInstance, BoundMethod from jedi.evaluate.context import ContextualizedName, ContextualizedNode, \ - ContextSet, NO_CONTEXTS + ContextSet, NO_CONTEXTS, iterate_contexts from jedi.evaluate.syntax_tree import eval_trailer, eval_expr_stmt, \ eval_node, check_tuple_assignments from jedi import parser_utils @@ -217,7 +217,7 @@ class Evaluator(object): if type_ == 'for_stmt': container_types = context.eval_node(def_.children[3]) cn = ContextualizedNode(context, def_.children[3]) - for_types = iterable.iterate_contexts(self, container_types, cn) + for_types = iterate_contexts(container_types, cn) c_node = ContextualizedName(context, name) return check_tuple_assignments(self, c_node, for_types) if type_ in ('import_from', 'import_name'): diff --git a/jedi/evaluate/context.py b/jedi/evaluate/context.py index e18474c7..c4fd3fab 100644 --- a/jedi/evaluate/context.py +++ b/jedi/evaluate/context.py @@ -1,7 +1,7 @@ from parso.python.tree import ExprStmt, CompFor from jedi import debug -from jedi._compatibility import Python3Method, zip_longest +from jedi._compatibility import Python3Method, zip_longest, unicode from jedi.parser_utils import clean_scope_docstring, get_doc_with_call_signature from jedi.common import BaseContextSet @@ -66,6 +66,14 @@ class Context(object): return self.evaluator.execute(self, arguments) + def execute_evaluated(self, *value_list): + """ + Execute a function with already executed arguments. + """ + from jedi.evaluate.param import ValuesArguments + arguments = ValuesArguments([ContextSet(value) for value in value_list]) + return self.execute(arguments) + def iterate(self, contextualized_node=None): debug.dbg('iterate') try: @@ -76,19 +84,51 @@ class Context(object): analysis.add( contextualized_node.context, 'type-error-not-iterable', - contextualized_node._node, + contextualized_node.node, message="TypeError: '%s' object is not iterable" % self) return iter([]) else: return iter_method() - def execute_evaluated(self, *value_list): - """ - Execute a function with already executed arguments. - """ - from jedi.evaluate.param import ValuesArguments - arguments = ValuesArguments([ContextSet(value) for value in value_list]) - return self.execute(arguments) + def get_item(self, index_contexts, contextualized_node): + from jedi.evaluate.compiled import CompiledObject + from jedi.evaluate.iterable import Slice, AbstractSequence + result = ContextSet() + + for index in index_contexts: + if isinstance(index, (CompiledObject, Slice)): + index = index.obj + + if type(index) not in (float, int, str, unicode, slice, type(Ellipsis)): + # If the index is not clearly defined, we have to get all the + # possiblities. + if isinstance(self, AbstractSequence) and self.array_type == 'dict': + result |= self.dict_values() + else: + result |= iterate_contexts(ContextSet(self)) + continue + + # The actual getitem call. + try: + getitem = self.py__getitem__ + except AttributeError: + from jedi.evaluate import analysis + # TODO this context is probably not right. + analysis.add( + contextualized_node.context, + 'type-error-not-subscriptable', + contextualized_node.node, + message="TypeError: '%s' object is not subscriptable" % self + ) + else: + try: + result |= getitem(index) + except IndexError: + result |= iterate_contexts(ContextSet(self)) + except KeyError: + # Must be a dict. Lists don't raise KeyErrors. + result |= self.dict_values() + return result def eval_node(self, node): return self.evaluator.eval_element(self, node) @@ -142,6 +182,17 @@ class Context(object): return None +def iterate_contexts(contexts, contextualized_node=None): + """ + Calls `iterate`, on all contexts but ignores the ordering and just returns + all contexts that the iterate functions yield. + """ + return ContextSet.from_sets( + lazy_context.infer() + for lazy_context in contexts.iterate(contextualized_node) + ) + + class TreeContext(Context): def __init__(self, evaluator, parent_context=None): super(TreeContext, self).__init__(evaluator, parent_context) @@ -215,20 +266,20 @@ class MergedLazyContexts(AbstractLazyContext): class ContextualizedNode(object): def __init__(self, context, node): self.context = context - self._node = node + self.node = node def get_root_context(self): return self.context.get_root_context() def infer(self): - return self.context.eval_node(self._node) + return self.context.eval_node(self.node) class ContextualizedName(ContextualizedNode): # TODO merge with TreeNameDefinition?! @property def name(self): - return self._node + return self.node def assignment_indexes(self): """ @@ -242,8 +293,8 @@ class ContextualizedName(ContextualizedNode): would result in ``[(1, xyz_node), (0, yz_node)]``. """ indexes = [] - node = self._node.parent - compare = self._node + node = self.node.parent + compare = self.node while node is not None: if node.type in ('testlist', 'testlist_comp', 'testlist_star_expr', 'exprlist'): for i, child in enumerate(node.children): diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index be30b6a5..d8730dee 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -613,74 +613,6 @@ def unpack_tuple_to_dict(context, types, exprlist): raise NotImplementedError -def iterate_contexts(evaluator, contexts, contextualized_node=None): - """ - Calls `iterate`, on all contexts but ignores the ordering and just returns - all contexts that the iterate functions yield. - """ - return ContextSet.from_sets( - lazy_context.infer() - for lazy_context in contexts.iterate(contextualized_node) - ) - - -def py__getitem__(evaluator, context, types, trailer): - from jedi.evaluate.representation import ClassContext - from jedi.evaluate.instance import TreeInstance - result = ContextSet() - - trailer_op, node, trailer_cl = trailer.children - assert trailer_op == "[" - assert trailer_cl == "]" - - # TODO It's kind of stupid to cast this from a context set to a set. - types = set(types) - # special case: PEP0484 typing module, see - # https://github.com/davidhalter/jedi/issues/663 - for typ in list(types): - if isinstance(typ, (ClassContext, TreeInstance)): - typing_module_types = pep0484.py__getitem__(context, typ, node) - if typing_module_types is not None: - types.remove(typ) - result |= typing_module_types - - if not types: - # all consumed by special cases - return result - - for index in create_index_types(evaluator, context, node): - if isinstance(index, (compiled.CompiledObject, Slice)): - index = index.obj - - if type(index) not in (float, int, str, unicode, slice, type(Ellipsis)): - # If the index is not clearly defined, we have to get all the - # possiblities. - for typ in list(types): - if isinstance(typ, AbstractSequence) and typ.array_type == 'dict': - types.remove(typ) - result |= typ.dict_values() - cs = ContextSet.from_set(types) - return result | iterate_contexts(evaluator, cs) - - for typ in types: - # The actual getitem call. - try: - getitem = typ.py__getitem__ - except AttributeError: - # TODO this context is probably not right. - analysis.add(context, 'type-error-not-subscriptable', trailer_op, - message="TypeError: '%s' object is not subscriptable" % typ) - else: - try: - result |= getitem(index) - except IndexError: - result |= iterate_contexts(evaluator, ContextSet(typ)) - except KeyError: - # Must be a dict. Lists don't raise KeyErrors. - result |= typ.dict_values() - return result - - def check_array_additions(context, sequence): """ Just a mapper function for the internal _check_array_additions """ if sequence.array_type not in ('list', 'set'): diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index 2b9ad9d0..1ea6455f 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -10,7 +10,7 @@ from jedi._compatibility import unicode from jedi import debug from jedi import parser_utils from jedi.evaluate.context import ContextSet, NO_CONTEXTS, ContextualizedNode, \ - ContextualizedName, iterator_to_context_set + ContextualizedName, iterator_to_context_set, iterate_contexts from jedi.evaluate import compiled from jedi.evaluate import pep0484 from jedi.evaluate import recursion @@ -119,7 +119,27 @@ def eval_trailer(context, base_contexts, trailer): if trailer_op == '[': from jedi.evaluate import iterable - return iterable.py__getitem__(context.evaluator, context, base_contexts, trailer) + from jedi.evaluate.representation import ClassContext + from jedi.evaluate.instance import TreeInstance + + trailer_op, node, _ = trailer.children + + # TODO It's kind of stupid to cast this from a context set to a set. + foo = set(base_contexts) + # special case: PEP0484 typing module, see + # https://github.com/davidhalter/jedi/issues/663 + result = ContextSet() + for typ in list(foo): + if isinstance(typ, (ClassContext, TreeInstance)): + typing_module_types = pep0484.py__getitem__(context, typ, node) + if typing_module_types is not None: + foo.remove(typ) + result |= typing_module_types + + return result | base_contexts.get_item( + iterable.create_index_types(context.evaluator, context, node), + ContextualizedNode(context, trailer) + ) else: debug.dbg('eval_trailer: %s in %s', trailer, base_contexts) if trailer_op == '.': @@ -475,7 +495,7 @@ def tree_name_to_contexts(evaluator, context, tree_name): types = context.predefined_names[node][tree_name.value] except KeyError: cn = ContextualizedNode(context, node.children[3]) - for_types = iterable.iterate_contexts(evaluator, cn.infer(), cn) + for_types = iterate_contexts(cn.infer(), cn) c_node = ContextualizedName(context, tree_name) types = check_tuple_assignments(evaluator, c_node, for_types) elif typ == 'expr_stmt': From 612ad2f49145bfee9b1c8e71ff615740f7824d86 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 14:17:37 +0200 Subject: [PATCH 31/35] Move eval_subscript_list to the syntax_tree module. --- jedi/evaluate/iterable.py | 30 ------------------------------ jedi/evaluate/syntax_tree.py | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index d8730dee..d13970ec 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -786,33 +786,3 @@ class Slice(context.Context): return slice(get(self._start), get(self._stop), get(self._step)) except IndexError: return slice(None, None, None) - - -def create_index_types(evaluator, context, index): - """ - Handles slices in subscript nodes. - """ - if index == ':': - # Like array[:] - return ContextSet(Slice(context, None, None, None)) - - elif index.type == 'subscript' and not index.children[0] == '.': - # subscript basically implies a slice operation, except for Python 2's - # Ellipsis. - # e.g. array[:3] - result = [] - for el in index.children: - if el == ':': - if not result: - result.append(None) - elif el.type == 'sliceop': - if len(el.children) == 2: - result.append(el.children[1]) - else: - result.append(el) - result += [None] * (3 - len(result)) - - return ContextSet(Slice(context, *result)) - - # No slices - return context.eval_node(index) diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index 1ea6455f..e542ce52 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -118,7 +118,6 @@ def eval_trailer(context, base_contexts, trailer): node = () if trailer_op == '[': - from jedi.evaluate import iterable from jedi.evaluate.representation import ClassContext from jedi.evaluate.instance import TreeInstance @@ -137,7 +136,7 @@ def eval_trailer(context, base_contexts, trailer): result |= typing_module_types return result | base_contexts.get_item( - iterable.create_index_types(context.evaluator, context, node), + eval_subscript_list(context.evaluator, context, node), ContextualizedNode(context, trailer) ) else: @@ -582,3 +581,34 @@ def check_tuple_assignments(evaluator, contextualized_name, context_set): return ContextSet() context_set = lazy_context.infer() return context_set + + +def eval_subscript_list(evaluator, context, index): + """ + Handles slices in subscript nodes. + """ + from jedi.evaluate.iterable import Slice + if index == ':': + # Like array[:] + return ContextSet(Slice(context, None, None, None)) + + elif index.type == 'subscript' and not index.children[0] == '.': + # subscript basically implies a slice operation, except for Python 2's + # Ellipsis. + # e.g. array[:3] + result = [] + for el in index.children: + if el == ':': + if not result: + result.append(None) + elif el.type == 'sliceop': + if len(el.children) == 2: + result.append(el.children[1]) + else: + result.append(el) + result += [None] * (3 - len(result)) + + return ContextSet(Slice(context, *result)) + + # No slices + return context.eval_node(index) From 6b76e37673ae2681fe0c9cd2f473cabd03c3de5a Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 14:19:11 +0200 Subject: [PATCH 32/35] Make some functions private in evaluate/iterable. --- jedi/evaluate/iterable.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index d13970ec..520a6e00 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -110,7 +110,7 @@ class SpecialMethodFilter(DictFilter): return self.SpecialMethodName(self.context, name, value, self._builtin_context) -def has_builtin_methods(cls): +def _has_builtin_methods(cls): base_dct = {} # Need to care properly about inheritance. Builtin Methods should not get # lost, just because they are not mentioned in a class. @@ -129,7 +129,7 @@ def has_builtin_methods(cls): return cls -def register_builtin_method(method_name, python_version_match=None): +def _register_builtin_method(method_name, python_version_match=None): def wrapper(func): if python_version_match and python_version_match != 2 + int(is_py3): # Some functions do only apply to certain versions. @@ -140,13 +140,13 @@ def register_builtin_method(method_name, python_version_match=None): return wrapper -@has_builtin_methods +@_has_builtin_methods class GeneratorMixin(object): array_type = None - @register_builtin_method('send') - @register_builtin_method('next', python_version_match=2) - @register_builtin_method('__next__', python_version_match=3) + @_register_builtin_method('send') + @_register_builtin_method('next', python_version_match=2) + @_register_builtin_method('__next__', python_version_match=3) def py__next__(self): # TODO add TypeError if params are given. return ContextSet.from_sets(lazy_context.infer() for lazy_context in self.py__iter__()) @@ -322,7 +322,7 @@ class SetComprehension(ArrayMixin, Comprehension): array_type = 'set' -@has_builtin_methods +@_has_builtin_methods class DictComprehension(ArrayMixin, Comprehension): array_type = 'dict' @@ -344,12 +344,12 @@ class DictComprehension(ArrayMixin, Comprehension): def dict_values(self): return ContextSet.from_sets(values for keys, values in self._iterate()) - @register_builtin_method('values') + @_register_builtin_method('values') def _imitate_values(self): lazy_context = context.LazyKnownContexts(self.dict_values()) return ContextSet(FakeSequence(self.evaluator, 'list', [lazy_context])) - @register_builtin_method('items') + @_register_builtin_method('items') def _imitate_items(self): items = ContextSet.from_iterable( FakeSequence( @@ -466,7 +466,7 @@ class SequenceLiteralContext(ArrayMixin, AbstractSequence): return "<%s of %s>" % (self.__class__.__name__, self.atom) -@has_builtin_methods +@_has_builtin_methods class DictLiteralContext(SequenceLiteralContext): array_type = 'dict' @@ -475,12 +475,12 @@ class DictLiteralContext(SequenceLiteralContext): self._defining_context = defining_context self.atom = atom - @register_builtin_method('values') + @_register_builtin_method('values') def _imitate_values(self): lazy_context = context.LazyKnownContexts(self.dict_values()) return ContextSet(FakeSequence(self.evaluator, 'list', [lazy_context])) - @register_builtin_method('items') + @_register_builtin_method('items') def _imitate_items(self): lazy_contexts = [ context.LazyKnownContext(FakeSequence( From e62d89bb03479012e9a124fa647540897004341d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 14:28:07 +0200 Subject: [PATCH 33/35] Move the is_string etc functions to the helpers module. --- jedi/evaluate/helpers.py | 18 ++++++++++++++++++ jedi/evaluate/iterable.py | 3 +-- jedi/evaluate/syntax_tree.py | 28 ++++++---------------------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index ff53e11d..6652287b 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -7,7 +7,9 @@ from contextlib import contextmanager from parso.python import tree +from jedi._compatibility import unicode from jedi.parser_utils import get_parent_scope +from jedi.evaluate.compiled import CompiledObject def is_stdlib_path(path): @@ -174,3 +176,19 @@ def predefine_names(context, flow_scope, dct): yield finally: del predefined[flow_scope] + + +def is_compiled(context): + return isinstance(context, CompiledObject) + + +def is_string(context): + return is_compiled(context) and isinstance(context.obj, (str, unicode)) + + +def is_literal(context): + return is_number(context) or is_string(context) + + +def is_number(context): + return is_compiled(context) and isinstance(context.obj, (int, float)) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index 520a6e00..efb0cb4f 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -23,12 +23,11 @@ It is important to note that: from jedi import debug from jedi import settings from jedi.evaluate.utils import safe_property -from jedi._compatibility import unicode, is_py3 +from jedi._compatibility import is_py3 from jedi.evaluate.utils import to_list from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate import analysis -from jedi.evaluate import pep0484 from jedi.evaluate.syntax_tree import is_string from jedi.evaluate import recursion from jedi.evaluate.cache import evaluator_method_cache diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index e542ce52..34d2c18f 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -6,7 +6,6 @@ import operator as op from parso.python import tree -from jedi._compatibility import unicode from jedi import debug from jedi import parser_utils from jedi.evaluate.context import ContextSet, NO_CONTEXTS, ContextualizedNode, \ @@ -16,6 +15,7 @@ from jedi.evaluate import pep0484 from jedi.evaluate import recursion from jedi.evaluate import helpers from jedi.evaluate import analysis +from jedi.evaluate.helpers import is_string, is_literal, is_number, is_compiled def _limit_context_infers(func): @@ -308,7 +308,7 @@ def eval_factor(context_set, operator): """ for context in context_set: if operator == '-': - if _is_number(context): + if is_number(context): yield compiled.create(context.evaluator, -context.obj) elif operator == 'not': value = context.py__bool__() @@ -337,7 +337,7 @@ def _literals_to_types(evaluator, result): # int(), float(), etc). new_result = NO_CONTEXTS for typ in result: - if _is_literal(typ): + if is_literal(typ): # Literals are only valid as long as the operations are # correct. Otherwise add a value-free instance. cls = compiled.builtin_from_name(evaluator, typ.name.string_name) @@ -366,22 +366,6 @@ def _eval_comparison(evaluator, context, left_contexts, operator, right_contexts ) -def _is_compiled(context): - return isinstance(context, compiled.CompiledObject) - - -def _is_number(context): - return _is_compiled(context) and isinstance(context.obj, (int, float)) - - -def is_string(context): - return _is_compiled(context) and isinstance(context.obj, (str, unicode)) - - -def _is_literal(context): - return _is_number(context) or is_string(context) - - def _is_tuple(context): from jedi.evaluate import iterable return isinstance(context, iterable.AbstractSequence) and context.array_type == 'tuple' @@ -394,8 +378,8 @@ def _is_list(context): def _eval_comparison_part(evaluator, context, left, operator, right): from jedi.evaluate import iterable, instance - l_is_num = _is_number(left) - r_is_num = _is_number(right) + l_is_num = is_number(left) + r_is_num = is_number(right) if operator == '*': # for iterables, ignore * operations if isinstance(left, iterable.AbstractSequence) or is_string(left): @@ -416,7 +400,7 @@ def _eval_comparison_part(evaluator, context, left, operator, right): return ContextSet(left) elif operator in COMPARISON_OPERATORS: operation = COMPARISON_OPERATORS[operator] - if _is_compiled(left) and _is_compiled(right): + if is_compiled(left) and is_compiled(right): # Possible, because the return is not an option. Just compare. left = left.obj right = right.obj From 18bab194c01379933ea7937207443a4045b97bc1 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 14:35:18 +0200 Subject: [PATCH 34/35] Move a few imports out of functions. --- jedi/evaluate/iterable.py | 2 +- jedi/evaluate/representation.py | 2 +- jedi/evaluate/syntax_tree.py | 21 +++++++-------------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index efb0cb4f..b96d1fc4 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -28,7 +28,7 @@ from jedi.evaluate.utils import to_list from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate import analysis -from jedi.evaluate.syntax_tree import is_string +from jedi.evaluate.helpers import is_string from jedi.evaluate import recursion from jedi.evaluate.cache import evaluator_method_cache from jedi.evaluate.filters import DictFilter, AbstractNameDefinition, \ diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 639c4ca2..37c452bf 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -51,7 +51,6 @@ from jedi import debug from jedi.evaluate.cache import evaluator_method_cache, CachedMetaClass from jedi.evaluate import compiled from jedi.evaluate import recursion -from jedi.evaluate import iterable from jedi.evaluate import docstrings from jedi.evaluate import pep0484 from jedi.evaluate import param @@ -262,6 +261,7 @@ class FunctionContext(use_metaclass(CachedMetaClass, context.TreeContext)): """ yield_exprs = get_yield_exprs(self.evaluator, self.tree_node) if yield_exprs: + from jedi.evaluate import iterable return ContextSet(iterable.Generator(self.evaluator, function_execution)) else: return function_execution.get_return_values() diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index 34d2c18f..b71d5df2 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -15,6 +15,9 @@ from jedi.evaluate import pep0484 from jedi.evaluate import recursion from jedi.evaluate import helpers from jedi.evaluate import analysis +from jedi.evaluate import iterable +from jedi.evaluate import imports +from jedi.evaluate import representation as er from jedi.evaluate.helpers import is_string, is_literal, is_number, is_compiled @@ -57,7 +60,6 @@ def eval_node(context, element): # else: print e.g. could be evaluated like this in Python 2.7 return NO_CONTEXTS elif typ == 'lambdef': - from jedi.evaluate import representation as er return ContextSet(er.FunctionContext(evaluator, context, element)) elif typ == 'expr_stmt': return eval_expr_stmt(context, element) @@ -81,7 +83,6 @@ def eval_node(context, element): return NO_CONTEXTS elif typ in ('testlist_star_expr', 'testlist',): # The implicit tuple in statements. - from jedi.evaluate import iterable return ContextSet(iterable.SequenceLiteralContext(evaluator, context, element)) elif typ in ('not_test', 'factor'): context_set = context.eval_node(element.children[-1]) @@ -118,7 +119,6 @@ def eval_trailer(context, base_contexts, trailer): node = () if trailer_op == '[': - from jedi.evaluate.representation import ClassContext from jedi.evaluate.instance import TreeInstance trailer_op, node, _ = trailer.children @@ -129,7 +129,7 @@ def eval_trailer(context, base_contexts, trailer): # https://github.com/davidhalter/jedi/issues/663 result = ContextSet() for typ in list(foo): - if isinstance(typ, (ClassContext, TreeInstance)): + if isinstance(typ, (er.ClassContext, TreeInstance)): typing_module_types = pep0484.py__getitem__(context, typ, node) if typing_module_types is not None: foo.remove(typ) @@ -159,7 +159,6 @@ def eval_atom(context, atom): generate the node (because it has just one child). In that case an atom might be a name or a literal as well. """ - from jedi.evaluate import iterable if atom.type == 'name': # This is the first global lookup. stmt = tree.search_ancestor( @@ -367,17 +366,15 @@ def _eval_comparison(evaluator, context, left_contexts, operator, right_contexts def _is_tuple(context): - from jedi.evaluate import iterable return isinstance(context, iterable.AbstractSequence) and context.array_type == 'tuple' def _is_list(context): - from jedi.evaluate import iterable return isinstance(context, iterable.AbstractSequence) and context.array_type == 'list' def _eval_comparison_part(evaluator, context, left, operator, right): - from jedi.evaluate import iterable, instance + from jedi.evaluate import instance l_is_num = is_number(left) r_is_num = is_number(right) if operator == '*': @@ -473,7 +470,6 @@ def tree_name_to_contexts(evaluator, context, tree_name): return types if typ in ('for_stmt', 'comp_for'): - from jedi.evaluate import iterable try: types = context.predefined_names[node][tree_name.value] except KeyError: @@ -488,7 +484,6 @@ def tree_name_to_contexts(evaluator, context, tree_name): enter_methods = context_managers.py__getattribute__('__enter__') return enter_methods.execute_evaluated() elif typ in ('import_from', 'import_name'): - from jedi.evaluate import imports types = imports.infer_import(context, tree_name) elif typ in ('funcdef', 'classdef'): types = _apply_decorators(context, node) @@ -508,7 +503,6 @@ def _apply_decorators(context, node): Returns the function, that should to be executed in the end. This is also the places where the decorators are processed. """ - from jedi.evaluate import representation as er if node.type == 'classdef': decoratee_context = er.ClassContext( context.evaluator, @@ -571,10 +565,9 @@ def eval_subscript_list(evaluator, context, index): """ Handles slices in subscript nodes. """ - from jedi.evaluate.iterable import Slice if index == ':': # Like array[:] - return ContextSet(Slice(context, None, None, None)) + return ContextSet(iterable.Slice(context, None, None, None)) elif index.type == 'subscript' and not index.children[0] == '.': # subscript basically implies a slice operation, except for Python 2's @@ -592,7 +585,7 @@ def eval_subscript_list(evaluator, context, index): result.append(el) result += [None] * (3 - len(result)) - return ContextSet(Slice(context, *result)) + return ContextSet(iterable.Slice(context, *result)) # No slices return context.eval_node(index) From 3734d52c8bfdf874eb37e489a947590984921b40 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 28 Sep 2017 14:44:58 +0200 Subject: [PATCH 35/35] Move all the remaining imports out of the syntax tree functions --- jedi/evaluate/representation.py | 2 +- jedi/evaluate/syntax_tree.py | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 37c452bf..a89e4875 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -57,6 +57,7 @@ from jedi.evaluate import param from jedi.evaluate import flow_analysis from jedi.evaluate import imports from jedi.evaluate import helpers +from jedi.evaluate import iterable from jedi.evaluate.filters import ParserTreeFilter, FunctionExecutionFilter, \ GlobalNameFilter, DictFilter, ContextName, AbstractNameDefinition, \ ParamName, AnonymousInstanceParamName, TreeNameDefinition, \ @@ -261,7 +262,6 @@ class FunctionContext(use_metaclass(CachedMetaClass, context.TreeContext)): """ yield_exprs = get_yield_exprs(self.evaluator, self.tree_node) if yield_exprs: - from jedi.evaluate import iterable return ContextSet(iterable.Generator(self.evaluator, function_execution)) else: return function_execution.get_return_values() diff --git a/jedi/evaluate/syntax_tree.py b/jedi/evaluate/syntax_tree.py index b71d5df2..a3c50c95 100644 --- a/jedi/evaluate/syntax_tree.py +++ b/jedi/evaluate/syntax_tree.py @@ -17,7 +17,10 @@ from jedi.evaluate import helpers from jedi.evaluate import analysis from jedi.evaluate import iterable from jedi.evaluate import imports +from jedi.evaluate import param from jedi.evaluate import representation as er +from jedi.evaluate.instance import TreeInstance, CompiledInstance +from jedi.evaluate.finder import NameFinder from jedi.evaluate.helpers import is_string, is_literal, is_number, is_compiled @@ -119,8 +122,6 @@ def eval_trailer(context, base_contexts, trailer): node = () if trailer_op == '[': - from jedi.evaluate.instance import TreeInstance - trailer_op, node, _ = trailer.children # TODO It's kind of stupid to cast this from a context set to a set. @@ -148,7 +149,6 @@ def eval_trailer(context, base_contexts, trailer): ) else: assert trailer_op == '(' - from jedi.evaluate import param arguments = param.TreeArguments(context.evaluator, context, node, trailer) return base_contexts.execute(arguments) @@ -374,7 +374,6 @@ def _is_list(context): def _eval_comparison_part(evaluator, context, left, operator, right): - from jedi.evaluate import instance l_is_num = is_number(left) r_is_num = is_number(right) if operator == '*': @@ -414,7 +413,7 @@ def _eval_comparison_part(evaluator, context, left, operator, right): def check(obj): """Checks if a Jedi object is either a float or an int.""" - return isinstance(obj, instance.CompiledInstance) and \ + return isinstance(obj, CompiledInstance) and \ obj.name.string_name in ('int', 'float') # Static analysis, one is a number, the other one is not. @@ -449,7 +448,6 @@ def tree_name_to_contexts(evaluator, context, tree_name): node = tree_name.parent if node.type == 'global_stmt': context = evaluator.create_context(context, tree_name) - from jedi.evaluate.finder import NameFinder finder = NameFinder(evaluator, context, context, tree_name.value) filters = finder.get_filters(search_global=True) # For global_stmt lookups, we only need the first possible scope, @@ -530,7 +528,6 @@ def _apply_decorators(context, node): debug.warning('decorator not found: %s on %s', dec, node) return initial - from jedi.evaluate import param values = dec_values.execute(param.ValuesArguments([values])) if not len(values): debug.warning('not possible to resolve wrappers found %s', node)