start with the integration of an Operator class to make way for precedences

jedi/evaluate/__init__.py

@@ -166,6 +166,13 @@ class Evaluator(object):
                 # The string tokens are just operations (+, -, etc.)
                 elif isinstance(call, compiled.CompiledObject):
                     result.append(call)
+                elif isinstance(call, pr.Operator):
+                    if call == '*':
+                        if [r for r in result if isinstance(r, iterable.Array)
+                                or isinstance(r, compiled.CompiledObject)
+                                and isinstance(r.obj, (str, unicode))]:
+                            # if it is an iterable, ignore * operations
+                            next(calls_iterator)
                 elif not isinstance(call, (str, unicode)):
                     if isinstance(call, pr.Call) and str(call.name) == 'if':
                         # Ternary operators.
@@ -179,12 +186,6 @@ class Evaluator(object):
                                     break
                         continue
                     result += self.eval_call(call)
-                elif call == '*':
-                    if [r for r in result if isinstance(r, iterable.Array)
-                            or isinstance(r, compiled.CompiledObject)
-                            and isinstance(r.obj, (str, unicode))]:
-                        # if it is an iterable, ignore * operations
-                        next(calls_iterator)
         return set(result)
 
     def eval_call(self, call):

jedi/evaluate/representation.py

@@ -11,7 +11,7 @@ they change classes in Python 3.
 """
 import copy
 
-from jedi._compatibility import use_metaclass, unicode
+from jedi._compatibility import use_metaclass
 from jedi.parser import representation as pr
 from jedi import debug
 from jedi import common
@@ -218,7 +218,7 @@ class InstanceElement(use_metaclass(CachedMetaClass, pr.Base)):
     def expression_list(self):
         # Copy and modify the array.
         return [InstanceElement(self.instance._evaluator, self.instance, command, self.is_class_var)
-                if not isinstance(command, unicode) else command
+                if not isinstance(command, pr.Operator) else command
                 for command in self.var.expression_list()]
 
     def __iter__(self):

jedi/parser/__init__.py

@@ -317,7 +317,10 @@ class Parser(object):
                    or tok.string in breaks and level <= 0):
             try:
                 # print 'parse_stmt', tok, tokenize.tok_name[token_type]
-                tok_list.append(tok)
+                if tok.type == tokenize.OP:
+                    tok_list.append(pr.Operator(tok.string, tok.start_pos))
+                else:
+                    tok_list.append(tok)
                 if tok.string == 'as':
                     tok = next(self._gen)
                     if tok.type == tokenize.NAME:

jedi/parser/representation.py

@@ -924,8 +924,8 @@ isinstance(c, tokenize.Token) else unicode(c)
         it and make it nicer, that would be cool :-)
         """
         def is_assignment(tok):
-            return isinstance(tok, (str, unicode)) and tok.endswith('=') \
-                and not tok in ['>=', '<=', '==', '!=']
+            return isinstance(tok, Operator) and tok.operator.endswith('=') \
+                and not tok.operator in ['>=', '<=', '==', '!=']
 
         def parse_array(token_iterator, array_type, start_pos, add_el=None,
                         added_breaks=()):
@@ -981,6 +981,22 @@ isinstance(c, tokenize.Token) else unicode(c)
                     if isinstance(tok, ListComprehension):
                         # it's not possible to set it earlier
                         tok.parent = self
+
+                    if tok in closing_brackets:
+                        level -= 1
+                    elif tok in brackets.keys():
+                        level += 1
+
+                    if level == 0 and tok in closing_brackets \
+                            or tok in added_breaks \
+                            or level == 1 and (
+                                tok == ','
+                                or maybe_dict and tok == ':'
+                                or is_assignment(tok)
+                                and break_on_assignment
+                            ):
+                        end_pos = end_pos[0], end_pos[1] - 1
+                        break
                 else:
                     tok = tok_temp.string
                     start_tok_pos = tok_temp.start_pos
@@ -1003,22 +1019,6 @@ isinstance(c, tokenize.Token) else unicode(c)
                         )
                         if list_comp is not None:
                             token_list = [list_comp]
-
-                    if tok in closing_brackets:
-                        level -= 1
-                    elif tok in brackets.keys():
-                        level += 1
-
-                    if level == 0 and tok in closing_brackets \
-                            or tok in added_breaks \
-                            or level == 1 and (
-                                tok == ','
-                                or maybe_dict and tok == ':'
-                                or is_assignment(tok)
-                                and break_on_assignment
-                            ):
-                        end_pos = end_pos[0], end_pos[1] - 1
-                        break
                 token_list.append(tok_temp)
 
             if not token_list:
@@ -1108,26 +1108,26 @@ isinstance(c, tokenize.Token) else unicode(c)
         closing_brackets = ')', '}', ']'
 
         token_iterator = common.PushBackIterator(enumerate(self.token_list))
-        for i, tok_temp in token_iterator:
-            if isinstance(tok_temp, Base):
+        for i, tok in token_iterator:
+            if isinstance(tok, Base):
                 # the token is a Name, which has already been parsed
-                tok = tok_temp
                 token_type = None
                 start_pos = tok.start_pos
                 end_pos = tok.end_pos
-            else:
-                token_type = tok_temp.type
-                tok = tok_temp.string
-                start_pos = tok_temp.start_pos
-                end_pos = tok_temp.end_pos
+
                 if is_assignment(tok):
                     # This means, there is an assignment here.
                     # Add assignments, which can be more than one
-                    self._assignment_details.append((result, tok_temp.string))
+                    self._assignment_details.append((result, tok.operator))
                     result = []
                     is_chain = False
                     continue
-                elif tok == 'as':  # just ignore as, because it sets values
+            else:
+                token_type = tok.type
+                start_pos = tok.start_pos
+                end_pos = tok.end_pos
+                tok = tok.string
+                if tok == 'as':  # just ignore as, because it sets values
                     next(token_iterator, None)
                     continue
 
@@ -1150,7 +1150,7 @@ isinstance(c, tokenize.Token) else unicode(c)
                 is_chain = False
             elif tok in brackets.keys():
                 arr, is_ass = parse_array(
-                    token_iterator, brackets[tok], start_pos
+                    token_iterator, brackets[tok.operator], start_pos
                 )
                 if result and isinstance(result[-1], StatementElement):
                     result[-1].set_execution(arr)
@@ -1503,8 +1503,7 @@ class ListComprehension(Base):
         return self.stmt.end_pos
 
     def __repr__(self):
-        return "<%s: %s>" % \
-            (type(self).__name__, self.get_code())
+        return "<%s: %s>" % (type(self).__name__, self.get_code())
 
     def get_code(self):
         statements = self.stmt, self.middle, self.input
@@ -1512,7 +1511,7 @@ class ListComprehension(Base):
         return "%s for %s in %s" % tuple(code)
 
 
-class Operator():
+class Operator(Base):
     __slots__ = ('operator', '_line', '_column')
 
     def __init__(self, operator, start_pos):
@@ -1520,9 +1519,20 @@ class Operator():
         self._line = start_pos[0]
         self._column = start_pos[1]
 
+    def __repr__(self):
+        return "<%s: `%s`>" % (type(self).__name__, self.operator)
+
     @property
     def start_pos(self):
-        return self._column, self._line
+        return self._line, self._column
 
-    def get_code(self):
-        return self.operator
+    @property
+    def end_pos(self):
+        return self._line, self._column + len(self.operator)
+
+    def __eq__(self, other):
+        """Make comparisons easy. Improves the readability of the parser."""
+        return self.operator == other
+
+    def __hash__(self):
+        return hash(self.operator)

jedi/parser/tokenize.py

@@ -6,7 +6,8 @@ if the indentation is not right. The fast parser of jedi however requires
 "wrong" indentation.
 
 Basically this is a stripped down version of the standard library module, so
-you can read the documentation there.
+you can read the documentation there. Additionally we included some speed and
+memory optimizations, here.
 """
 from __future__ import absolute_import