diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 548ba34e..91e2a9d0 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -166,6 +166,13 @@ class Evaluator(object): # The string tokens are just operations (+, -, etc.) elif isinstance(call, compiled.CompiledObject): result.append(call) + elif isinstance(call, pr.Operator): + if call == '*': + if [r for r in result if isinstance(r, iterable.Array) + or isinstance(r, compiled.CompiledObject) + and isinstance(r.obj, (str, unicode))]: + # if it is an iterable, ignore * operations + next(calls_iterator) elif not isinstance(call, (str, unicode)): if isinstance(call, pr.Call) and str(call.name) == 'if': # Ternary operators. @@ -179,12 +186,6 @@ class Evaluator(object): break continue result += self.eval_call(call) - elif call == '*': - if [r for r in result if isinstance(r, iterable.Array) - or isinstance(r, compiled.CompiledObject) - and isinstance(r.obj, (str, unicode))]: - # if it is an iterable, ignore * operations - next(calls_iterator) return set(result) def eval_call(self, call): diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index e1841786..28d6f78b 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -11,7 +11,7 @@ they change classes in Python 3. """ import copy -from jedi._compatibility import use_metaclass, unicode +from jedi._compatibility import use_metaclass from jedi.parser import representation as pr from jedi import debug from jedi import common @@ -218,7 +218,7 @@ class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)): def expression_list(self): # Copy and modify the array. return [InstanceElement(self.instance._evaluator, self.instance, command, self.is_class_var) - if not isinstance(command, unicode) else command + if not isinstance(command, pr.Operator) else command for command in self.var.expression_list()] def __iter__(self): diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 346bd0dc..74819f5e 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -317,7 +317,10 @@ class Parser(object): or tok.string in breaks and level <= 0): try: # print 'parse_stmt', tok, tokenize.tok_name[token_type] - tok_list.append(tok) + if tok.type == tokenize.OP: + tok_list.append(pr.Operator(tok.string, tok.start_pos)) + else: + tok_list.append(tok) if tok.string == 'as': tok = next(self._gen) if tok.type == tokenize.NAME: diff --git a/jedi/parser/representation.py b/jedi/parser/representation.py index 7757310e..71cf7ffb 100644 --- a/jedi/parser/representation.py +++ b/jedi/parser/representation.py @@ -924,8 +924,8 @@ isinstance(c, tokenize.Token) else unicode(c) it and make it nicer, that would be cool :-) """ def is_assignment(tok): - return isinstance(tok, (str, unicode)) and tok.endswith('=') \ - and not tok in ['>=', '<=', '==', '!='] + return isinstance(tok, Operator) and tok.operator.endswith('=') \ + and not tok.operator in ['>=', '<=', '==', '!='] def parse_array(token_iterator, array_type, start_pos, add_el=None, added_breaks=()): @@ -981,6 +981,22 @@ isinstance(c, tokenize.Token) else unicode(c) if isinstance(tok, ListComprehension): # it's not possible to set it earlier tok.parent = self + + if tok in closing_brackets: + level -= 1 + elif tok in brackets.keys(): + level += 1 + + if level == 0 and tok in closing_brackets \ + or tok in added_breaks \ + or level == 1 and ( + tok == ',' + or maybe_dict and tok == ':' + or is_assignment(tok) + and break_on_assignment + ): + end_pos = end_pos[0], end_pos[1] - 1 + break else: tok = tok_temp.string start_tok_pos = tok_temp.start_pos @@ -1003,22 +1019,6 @@ isinstance(c, tokenize.Token) else unicode(c) ) if list_comp is not None: token_list = [list_comp] - - if tok in closing_brackets: - level -= 1 - elif tok in brackets.keys(): - level += 1 - - if level == 0 and tok in closing_brackets \ - or tok in added_breaks \ - or level == 1 and ( - tok == ',' - or maybe_dict and tok == ':' - or is_assignment(tok) - and break_on_assignment - ): - end_pos = end_pos[0], end_pos[1] - 1 - break token_list.append(tok_temp) if not token_list: @@ -1108,26 +1108,26 @@ isinstance(c, tokenize.Token) else unicode(c) closing_brackets = ')', '}', ']' token_iterator = common.PushBackIterator(enumerate(self.token_list)) - for i, tok_temp in token_iterator: - if isinstance(tok_temp, Base): + for i, tok in token_iterator: + if isinstance(tok, Base): # the token is a Name, which has already been parsed - tok = tok_temp token_type = None start_pos = tok.start_pos end_pos = tok.end_pos - else: - token_type = tok_temp.type - tok = tok_temp.string - start_pos = tok_temp.start_pos - end_pos = tok_temp.end_pos + if is_assignment(tok): # This means, there is an assignment here. # Add assignments, which can be more than one - self._assignment_details.append((result, tok_temp.string)) + self._assignment_details.append((result, tok.operator)) result = [] is_chain = False continue - elif tok == 'as': # just ignore as, because it sets values + else: + token_type = tok.type + start_pos = tok.start_pos + end_pos = tok.end_pos + tok = tok.string + if tok == 'as': # just ignore as, because it sets values next(token_iterator, None) continue @@ -1150,7 +1150,7 @@ isinstance(c, tokenize.Token) else unicode(c) is_chain = False elif tok in brackets.keys(): arr, is_ass = parse_array( - token_iterator, brackets[tok], start_pos + token_iterator, brackets[tok.operator], start_pos ) if result and isinstance(result[-1], StatementElement): result[-1].set_execution(arr) @@ -1503,8 +1503,7 @@ class ListComprehension(Base): return self.stmt.end_pos def __repr__(self): - return "<%s: %s>" % \ - (type(self).__name__, self.get_code()) + return "<%s: %s>" % (type(self).__name__, self.get_code()) def get_code(self): statements = self.stmt, self.middle, self.input @@ -1512,7 +1511,7 @@ class ListComprehension(Base): return "%s for %s in %s" % tuple(code) -class Operator(): +class Operator(Base): __slots__ = ('operator', '_line', '_column') def __init__(self, operator, start_pos): @@ -1520,9 +1519,20 @@ class Operator(): self._line = start_pos[0] self._column = start_pos[1] + def __repr__(self): + return "<%s: `%s`>" % (type(self).__name__, self.operator) + @property def start_pos(self): - return self._column, self._line + return self._line, self._column - def get_code(self): - return self.operator + @property + def end_pos(self): + return self._line, self._column + len(self.operator) + + def __eq__(self, other): + """Make comparisons easy. Improves the readability of the parser.""" + return self.operator == other + + def __hash__(self): + return hash(self.operator) diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index c3b4fc7b..3b040f3a 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -6,7 +6,8 @@ if the indentation is not right. The fast parser of jedi however requires "wrong" indentation. Basically this is a stripped down version of the standard library module, so -you can read the documentation there. +you can read the documentation there. Additionally we included some speed and +memory optimizations, here. """ from __future__ import absolute_import