merge of tokenizer/dev branch

jedi/_compatibility.py

@@ -7,11 +7,69 @@ Most of the code here is necessary to support Python 2.5. Once this dependency
 will be dropped, we'll get rid of most code.
 """
 import sys
+import imp
+import os
+try:
+    import importlib
+except:
+    pass
 
 is_py3k = sys.hexversion >= 0x03000000
-
+is_py33 = sys.hexversion >= 0x03030000
 is_py25 = sys.hexversion < 0x02060000
 
+def find_module_py33(string, path=None):
+    mod_info = (None, None, None)
+    loader = None
+    if path is not None:
+        # Check for the module in the specidied path
+        loader = importlib.machinery.PathFinder.find_module(string, path)
+    else:
+        # Check for the module in sys.path
+        loader = importlib.machinery.PathFinder.find_module(string, sys.path)
+        if loader is None:
+            # Fallback to find builtins
+            loader = importlib.find_loader(string)
+
+    if loader is None:
+        raise ImportError
+
+    try:
+        if (loader.is_package(string)):
+            mod_info = (None, os.path.dirname(loader.path), True)
+        else:
+            filename = loader.get_filename(string)
+            if filename and os.path.exists(filename):
+                mod_info = (open(filename, 'U'), filename, False)
+            else:
+                mod_info = (None, filename, False)
+    except AttributeError:
+        mod_info = (None, loader.load_module(string).__name__, False)
+
+    return mod_info
+
+def find_module_pre_py33(string, path=None):
+    mod_info = None
+    if path is None:
+        mod_info = imp.find_module(string)
+    else:
+        mod_info = imp.find_module(string, path)
+
+    return (mod_info[0], mod_info[1], mod_info[2][2] == imp.PKG_DIRECTORY)
+
+def find_module(string, path=None):
+    """Provides information about a module.
+
+    This function isolates the differences in importing libraries introduced with
+    python 3.3 on; it gets a module name and optionally a path. It will return a
+    tuple containin an open file for the module (if not builtin), the filename
+    or the name of the module if it is a builtin one and a boolean indicating
+    if the module is contained in a package."""
+    if is_py33:
+        return find_module_py33(string, path)
+    else:
+        return find_module_pre_py33(string, path)
+
 # next was defined in python 2.6, in python 3 obj.next won't be possible
 # anymore
 try:
@@ -81,6 +139,25 @@ else:
     eval(compile("""def exec_function(source, global_map):
                         exec source in global_map """, 'blub', 'exec'))
 
+# re-raise function
+if is_py3k:
+    def reraise(exception, traceback):
+        raise exception.with_traceback(traceback)
+else:
+    eval(compile("""
+def reraise(exception, traceback):
+    raise exception, None, traceback
+""", 'blub', 'exec'))
+
+reraise.__doc__ = """
+Re-raise `exception` with a `traceback` object.
+
+Usage::
+
+    reraise(Exception, sys.exc_info()[2])
+
+"""
+
 # StringIO (Python 2.5 has no io module), so use io only for py3k
 try:
     from StringIO import StringIO

jedi/api.py

@@ -7,29 +7,27 @@ catch :exc:`NotFoundError` which is being raised if your completion is not
 possible.
 """
 from __future__ import with_statement
-__all__ = ['Script', 'NotFoundError', 'set_debug_function', '_quick_complete']
 
 import re
 import os
 import warnings
 
-import parsing
-import parsing_representation as pr
+from jedi import parsing
+from jedi import parsing_representation as pr
+from jedi import debug
+from jedi import settings
+from jedi import helpers
+from jedi import common
+from jedi import cache
+from jedi import modules
+from jedi._compatibility import next, unicode
+import evaluate
+import keywords
+import api_classes
 import evaluate_representation as er
 import dynamic
 import imports
-import evaluate
-import modules
-import debug
-import settings
-import keywords
-import helpers
-import common
 import builtin
-import api_classes
-import cache
-
-from _compatibility import next, unicode
 
 
 class NotFoundError(Exception):
@@ -76,6 +74,7 @@ class Script(object):
         """ lazy parser."""
         return self._module.parser
 
+    @api_classes._clear_caches_after_call
     def complete(self):
         """
         Return :class:`api_classes.Completion` objects. Those objects contain
@@ -209,6 +208,7 @@ class Script(object):
         warnings.warn("Use line instead.", DeprecationWarning)
         return self.definition()
 
+    @api_classes._clear_caches_after_call
     def definition(self):
         """
         Return the definitions of a the path under the cursor. This is not a
@@ -270,8 +270,9 @@ class Script(object):
 
         d = set([api_classes.Definition(s) for s in scopes
                     if not isinstance(s, imports.ImportPath._GlobalNamespace)])
-        return sorted(d, key=lambda x: (x.module_path, x.start_pos))
+        return self._sorted_defs(d)
 
+    @api_classes._clear_caches_after_call
     def goto(self):
         """
         Return the first definition found by goto. Imports and statements
@@ -282,7 +283,7 @@ class Script(object):
         :rtype: list of :class:`api_classes.Definition`
         """
         d = [api_classes.Definition(d) for d in set(self._goto()[0])]
-        return sorted(d, key=lambda x: (x.module_path, x.start_pos))
+        return self._sorted_defs(d)
 
     def _goto(self, add_import_name=False):
         """
@@ -334,6 +335,7 @@ class Script(object):
                     definitions = [user_stmt]
         return definitions, search_name
 
+    @api_classes._clear_caches_after_call
     def related_names(self, additional_module_paths=()):
         """
         Return :class:`api_classes.RelatedName` objects, which contain all
@@ -367,7 +369,7 @@ class Script(object):
             else:
                 names.append(api_classes.RelatedName(d.names[-1], d))
 
-        return sorted(set(names), key=lambda x: (x.module_path, x.start_pos))
+        return self._sorted_defs(set(names))
 
     def get_in_function_call(self):
         """
@@ -378,6 +380,7 @@ class Script(object):
         warnings.warn("Use line instead.", DeprecationWarning)
         return self.function_definition()
 
+    @api_classes._clear_caches_after_call
     def function_definition(self):
         """
         Return the function object of the call you're currently in.
@@ -487,8 +490,11 @@ class Script(object):
         match = re.match(r'^(.*?)(\.|)(\w?[\w\d]*)$', path, flags=re.S)
         return match.groups()
 
-    def __del__(self):
-        api_classes._clear_caches()
+    @staticmethod
+    def _sorted_defs(d):
+        # Note: `or ''` below is required because `module_path` could be
+        #       None and you can't compare None and str in Python 3.
+        return sorted(d, key=lambda x: (x.module_path or '', x.start_pos))
 
 
 def defined_names(source, source_path=None, source_encoding='utf-8'):
@@ -507,7 +513,7 @@ def defined_names(source, source_path=None, source_encoding='utf-8'):
         modules.source_to_unicode(source, source_encoding),
         module_path=source_path,
     )
-    return api_classes.defined_names(parser.scope)
+    return api_classes._defined_names(parser.scope)
 
 
 def set_debug_function(func_cb=debug.print_to_stdout, warnings=True,

jedi/api_classes.py

@@ -3,21 +3,24 @@ The :mod:`api_classes` module contains the return classes of the API. These
 classes are the much bigger part of the whole API, because they contain the
 interesting information about completion and goto operations.
 """
+from __future__ import with_statement
 
 import re
 import os
 import warnings
+import functools
 
-from _compatibility import unicode, next
-import cache
-import dynamic
+from jedi._compatibility import unicode, next
+from jedi import settings
+from jedi import common
+from jedi import parsing_representation as pr
+from jedi import cache
+import keywords
 import recursion
-import settings
+import dynamic
 import evaluate
 import imports
-import parsing_representation as pr
 import evaluate_representation as er
-import keywords
 
 
 def _clear_caches():
@@ -34,6 +37,18 @@ def _clear_caches():
     imports.imports_processed = 0
 
 
+def _clear_caches_after_call(func):
+    """
+    Clear caches just before returning a value.
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwds):
+        result = func(*args, **kwds)
+        _clear_caches()
+        return result
+    return wrapper
+
+
 class BaseDefinition(object):
     _mapping = {'posixpath': 'os.path',
                'riscospath': 'os.path',
@@ -69,12 +84,60 @@ class BaseDefinition(object):
 
     @property
     def type(self):
-        """The type of the definition."""
+        """
+        The type of the definition.
+
+        Here is an example of the value of this attribute.  Let's consider
+        the following source.  As what is in ``variable`` is unambiguous
+        to Jedi, :meth:`api.Script.definition` should return a list of
+        definition for ``sys``, ``f``, ``C`` and ``x``.
+
+        >>> from jedi import Script
+        >>> source = '''
+        ... import sys
+        ...
+        ... class C:
+        ...     pass
+        ...
+        ... class D:
+        ...     pass
+        ...
+        ... x = D()
+        ...
+        ... def f():
+        ...     pass
+        ...
+        ... variable = sys or f or C or x'''
+        >>> script = Script(source, len(source.splitlines()), 3, 'example.py')
+        >>> defs = script.definition()
+
+        Before showing what is in ``defs``, let's sort it by :attr:`line`
+        so that it is easy to relate the result to the source code.
+
+        >>> defs = sorted(defs, key=lambda d: d.line)
+        >>> defs                           # doctest: +NORMALIZE_WHITESPACE
+        [<Definition module sys>, <Definition class C>,
+         <Definition class D>, <Definition def f>]
+
+        Finally, here is what you can get from :attr:`type`:
+
+        >>> defs[0].type
+        'module'
+        >>> defs[1].type
+        'class'
+        >>> defs[2].type
+        'instance'
+        >>> defs[3].type
+        'function'
+
+        """
         # generate the type
         stripped = self.definition
         if isinstance(self.definition, er.InstanceElement):
             stripped = self.definition.var
-        return type(stripped).__name__
+        if isinstance(stripped, pr.Name):
+            stripped = stripped.parent
+        return type(stripped).__name__.lower()
 
     @property
     def path(self):
@@ -83,19 +146,27 @@ class BaseDefinition(object):
         if not isinstance(self.definition, keywords.Keyword):
             par = self.definition
             while par is not None:
-                try:
+                with common.ignored(AttributeError):
                     path.insert(0, par.name)
-                except AttributeError:
-                    pass
                 par = par.parent
         return path
 
     @property
     def module_name(self):
-        """The module name."""
+        """
+        The module name.
+
+        >>> from jedi import Script
+        >>> source = 'import datetime'
+        >>> script = Script(source, 1, len(source), 'example.py')
+        >>> d = script.definition()[0]
+        >>> print(d.module_name)                       # doctest: +ELLIPSIS
+        datetime
+
+        """
         path = self.module_path
         sep = os.path.sep
-        p = re.sub(r'^.*?([\w\d]+)(%s__init__)?.py$' % sep, r'\1', path)
+        p = re.sub(r'^.*?([\w\d]+)(%s__init__)?.(py|so)$' % sep, r'\1', path)
         return p
 
     def in_builtin_module(self):
@@ -125,7 +196,31 @@ class BaseDefinition(object):
 
     @property
     def doc(self):
-        """Return a document string for this completion object."""
+        r"""
+        Return a document string for this completion object.
+
+        Example:
+
+        >>> from jedi import Script
+        >>> source = '''\
+        ... def f(a, b=1):
+        ...     "Document for function f."
+        ... '''
+        >>> script = Script(source, 1, len('def f'), 'example.py')
+        >>> d = script.definition()[0]
+        >>> print(d.doc)
+        f(a, b = 1)
+        <BLANKLINE>
+        Document for function f.
+
+        Notice that useful extra information is added to the actual
+        docstring.  For function, it is call signature.  If you need
+        actual docstring, use :attr:`raw_doc` instead.
+
+        >>> print(d.raw_doc)
+        Document for function f.
+
+        """
         try:
             return self.definition.doc
         except AttributeError:
@@ -133,7 +228,11 @@ class BaseDefinition(object):
 
     @property
     def raw_doc(self):
-        """The raw docstring ``__doc__`` for any object."""
+        """
+        The raw docstring ``__doc__`` for any object.
+
+        See :attr:`doc` for example.
+        """
         try:
             return unicode(self.definition.docstr)
         except AttributeError:
@@ -141,21 +240,63 @@ class BaseDefinition(object):
 
     @property
     def description(self):
-        """A textual description of the object."""
+        """
+        A textual description of the object.
+
+        Example:
+
+        >>> from jedi import Script
+        >>> source = '''
+        ... def f():
+        ...     pass
+        ...
+        ... class C:
+        ...     pass
+        ...
+        ... variable = f or C'''
+        >>> script = Script(source, len(source.splitlines()), 3, 'example.py')
+        >>> defs = script.definition()                      # doctest: +SKIP
+        >>> defs = sorted(defs, key=lambda d: d.line)       # doctest: +SKIP
+        >>> defs                                            # doctest: +SKIP
+        [<Definition def f>, <Definition class C>]
+        >>> defs[0].description                             # doctest: +SKIP
+        'def f'
+        >>> defs[1].description                             # doctest: +SKIP
+        'class C'
+
+        """
         return unicode(self.definition)
 
     @property
     def full_name(self):
-        """The path to a certain class/function, see #61."""
+        """
+        Dot-separated path of this object.
+
+        It is in the form of ``<module>[.<submodule>[...]][.<object>]``.
+        It is useful when you want to look up Python manual of the
+        object at hand.
+
+        Example:
+
+        >>> from jedi import Script
+        >>> source = '''
+        ... import os
+        ... os.path.join'''
+        >>> script = Script(source, 3, len('os.path.join'), 'example.py')
+        >>> print(script.definition()[0].full_name)
+        os.path.join
+
+        Notice that it correctly returns ``'os.path.join'`` instead of
+        (for example) ``'posixpath.join'``.
+
+        """
         path = [unicode(p) for p in self.path]
         # TODO add further checks, the mapping should only occur on stdlib.
         if not path:
             return None  # for keywords the path is empty
 
-        try:
+        with common.ignored(KeyError):
             path[0] = self._mapping[path[0]]
-        except KeyError:
-            pass
         for key, repl in self._tuple_mapping.items():
             if tuple(path[:len(key)]) == key:
                 path = [repl] + path[len(key):]
@@ -250,7 +391,7 @@ class Completion(BaseDefinition):
         """
         if self._followed_definitions is None:
             if self.definition.isinstance(pr.Statement):
-                defs = er.follow_statement(self.definition)
+                defs = evaluate.follow_statement(self.definition)
             elif self.definition.isinstance(pr.Import):
                 defs = imports.strip_imports([self.definition])
             else:
@@ -365,10 +506,10 @@ class Definition(BaseDefinition):
             d = d.var
         if isinstance(d, pr.Name):
             d = d.parent
-        return defined_names(d)
+        return _defined_names(d)
 
 
-def defined_names(scope):
+def _defined_names(scope):
     """
     List sub-definitions (e.g., methods in class).
 

jedi/builtin.py

@@ -22,7 +22,7 @@ possible to access functions like ``list`` and ``int`` directly, the same way
 """
 
 from __future__ import with_statement
-from _compatibility import exec_function, is_py3k
+from jedi._compatibility import exec_function, is_py3k
 
 import re
 import sys
@@ -32,11 +32,11 @@ if is_py3k:
 import types
 import inspect
 
-import common
-import debug
-import parsing
+from jedi import common
+from jedi import debug
+from jedi import parsing
+from jedi import modules
 import evaluate
-import modules
 
 
 class BuiltinModule(modules.CachedModule):

jedi/cache.py

@@ -21,11 +21,17 @@ from __future__ import with_statement
 import time
 import os
 import sys
-import pickle
+import hashlib
+try:
+    import cPickle as pickle
+except:
+    import pickle
+import shutil
 
-from _compatibility import json
-import settings
-import debug
+from jedi._compatibility import json
+from jedi import settings
+from jedi import common
+from jedi import debug
 
 # memoize caches will be deleted after every action
 memoize_caches = []
@@ -143,12 +149,10 @@ def cache_function_definition(stmt):
 
 def cache_star_import(func):
     def wrapper(scope, *args, **kwargs):
-        try:
+        with common.ignored(KeyError):
             mods = star_import_cache[scope]
             if mods[0] + settings.star_import_cache_validity > time.time():
                 return mods[1]
-        except KeyError:
-            pass
         # cache is too old and therefore invalid or not available
         invalidate_star_import_cache(scope)
         mods = func(scope, *args, **kwargs)
@@ -160,15 +164,13 @@ def cache_star_import(func):
 
 def invalidate_star_import_cache(module, only_main=False):
     """ Important if some new modules are being reparsed """
-    try:
+    with common.ignored(KeyError):
         t, mods = star_import_cache[module]
 
         del star_import_cache[module]
 
         for m in mods:
             invalidate_star_import_cache(m, only_main=True)
-    except KeyError:
-        pass
 
     if not only_main:
         # We need a list here because otherwise the list is being changed
@@ -216,13 +218,36 @@ def save_module(path, name, parser, pickling=True):
 
 
 class _ModulePickling(object):
+
+    version = 2
+    """
+    Version number (integer) for file system cache.
+
+    Increment this number when there are any incompatible changes in
+    parser representation classes.  For example, the following changes
+    are regarded as incompatible.
+
+    - Class name is changed.
+    - Class is moved to another module.
+    - Defined slot of the class is changed.
+    """
+
     def __init__(self):
         self.__index = None
-        self.py_version = '%s.%s' % sys.version_info[:2]
+        self.py_tag = 'cpython-%s%s' % sys.version_info[:2]
+        """
+        Short name for distinguish Python implementations and versions.
+
+        It's like `sys.implementation.cache_tag` but for Python < 3.3
+        we generate something similar.  See:
+        http://docs.python.org/3/library/sys.html#sys.implementation
+
+        .. todo:: Detect interpreter (e.g., PyPy).
+        """
 
     def load_module(self, path, original_changed_time):
         try:
-            pickle_changed_time = self._index[self.py_version][path]
+            pickle_changed_time = self._index[path]
         except KeyError:
             return None
         if original_changed_time is not None \
@@ -238,11 +263,12 @@ class _ModulePickling(object):
         return parser_cache_item.parser
 
     def save_module(self, path, parser_cache_item):
+        self.__index = None
         try:
-            files = self._index[self.py_version]
+            files = self._index
         except KeyError:
             files = {}
-            self._index[self.py_version] = files
+            self._index = files
 
         with open(self._get_hashed_path(path), 'wb') as f:
             pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL)
@@ -255,9 +281,16 @@ class _ModulePickling(object):
         if self.__index is None:
             try:
                 with open(self._get_path('index.json')) as f:
-                    self.__index = json.load(f)
+                    data = json.load(f)
             except IOError:
                 self.__index = {}
+            else:
+                # 0 means version is not defined (= always delete cache):
+                if data.get('version', 0) != self.version:
+                    self.delete_cache()
+                    self.__index = {}
+                else:
+                    self.__index = data['index']
         return self.__index
 
     def _remove_old_modules(self):
@@ -268,18 +301,25 @@ class _ModulePickling(object):
             self._index  # reload index
 
     def _flush_index(self):
+        data = {'version': self.version, 'index': self._index}
         with open(self._get_path('index.json'), 'w') as f:
-            json.dump(self._index, f)
+            json.dump(data, f)
         self.__index = None
 
+    def delete_cache(self):
+        shutil.rmtree(self._cache_directory())
+
     def _get_hashed_path(self, path):
-        return self._get_path('%s_%s.pkl' % (self.py_version, hash(path)))
+        return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest())
 
     def _get_path(self, file):
-        dir = settings.cache_directory
+        dir = self._cache_directory()
         if not os.path.exists(dir):
             os.makedirs(dir)
-        return dir + os.path.sep + file
+        return os.path.join(dir, file)
+
+    def _cache_directory(self):
+        return os.path.join(settings.cache_directory, self.py_tag)
 
 
 # is a singleton

jedi/common.py

@@ -1,10 +1,11 @@
 """ A universal module with functions / classes without dependencies. """
+import sys
 import contextlib
-import tokenize
+import functools
+import tokenizer as tokenize
 
-from _compatibility import next
-import debug
-import settings
+from jedi._compatibility import next, reraise
+from jedi import settings
 
 FLOWS = ['if', 'else', 'elif', 'while', 'with', 'try', 'except', 'finally']
 
@@ -16,23 +17,47 @@ class MultiLevelStopIteration(Exception):
     pass
 
 
-class MultiLevelAttributeError(Exception):
+class UncaughtAttributeError(Exception):
     """
     Important, because `__getattr__` and `hasattr` catch AttributeErrors
     implicitly. This is really evil (mainly because of `__getattr__`).
     `hasattr` in Python 2 is even more evil, because it catches ALL exceptions.
-    Therefore this class has to be a `BaseException` and not an `Exception`.
-    But because I rewrote hasattr, we can now switch back to `Exception`.
+    Therefore this class originally had to be derived from `BaseException`
+    instead of `Exception`.  But because I removed relevant `hasattr` from
+    the code base, we can now switch back to `Exception`.
 
     :param base: return values of sys.exc_info().
     """
-    def __init__(self, base=None):
-        self.base = base
 
-    def __str__(self):
-        import traceback
-        tb = traceback.format_exception(*self.base)
-        return 'Original:\n\n' + ''.join(tb)
+
+def rethrow_uncaught(func):
+    """
+    Re-throw uncaught `AttributeError`.
+
+    Usage:  Put ``@rethrow_uncaught`` in front of the function
+    which does **not** suppose to raise `AttributeError`.
+
+    AttributeError is easily get caught by `hasattr` and another
+    ``except AttributeError`` clause.  This becomes problem when you use
+    a lot of "dynamic" attributes (e.g., using ``@property``) because you
+    can't distinguish if the property does not exist for real or some code
+    inside of the "dynamic" attribute through that error.  In a well
+    written code, such error should not exist but getting there is very
+    difficult.  This decorator is to help us getting there by changing
+    `AttributeError` to `UncaughtAttributeError` to avoid unexpected catch.
+    This helps us noticing bugs earlier and facilitates debugging.
+
+    .. note:: Treating StopIteration here is easy.
+              Add that feature when needed.
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwds):
+        try:
+            return func(*args, **kwds)
+        except AttributeError:
+            exc_info = sys.exc_info()
+            reraise(UncaughtAttributeError(exc_info[1]), exc_info[2])
+    return wrapper
 
 
 class PushBackIterator(object):
@@ -84,29 +109,10 @@ class NoErrorTokenizer(object):
     def __next__(self):
         if self.closed:
             raise MultiLevelStopIteration()
-        try:
-            self.last_previous = self.previous
-            self.previous = self.current
-            self.current = next(self.gen)
-        except tokenize.TokenError:
-            # We just ignore this error, I try to handle it earlier - as
-            # good as possible
-            debug.warning('parentheses not closed error')
-            return self.__next__()
-        except IndentationError:
-            # This is an error, that tokenize may produce, because the code
-            # is not indented as it should. Here it just ignores this line
-            # and restarts the parser.
-            # (This is a rather unlikely error message, for normal code,
-            # tokenize seems to be pretty tolerant)
-            debug.warning('indentation error on line %s, ignoring it' %
-                                                        self.current[2][0])
-            # add the starting line of the last position
-            self.offset = self.current[2]
-            self.gen = PushBackIterator(tokenize.generate_tokens(
-                                                                self.readline))
-            return self.__next__()
 
+        self.last_previous = self.previous
+        self.previous = self.current
+        self.current = next(self.gen)
         c = list(self.current)
 
         if c[0] == tokenize.ENDMARKER:
@@ -187,3 +193,13 @@ def indent_block(text, indention='    '):
         text = text[:-1]
     lines = text.split('\n')
     return '\n'.join(map(lambda s: indention + s, lines)) + temp
+
+
+@contextlib.contextmanager
+def ignored(*exceptions):
+    """Context manager that ignores all of the specified exceptions. This will
+    be in the standard library starting with Python 3.4."""
+    try:
+        yield
+    except exceptions:
+        pass

jedi/docstrings.py

@@ -16,10 +16,10 @@ annotations.
 
 import re
 
-import cache
+from jedi import cache
+from jedi import parsing
 import evaluate
 import evaluate_representation as er
-import parsing
 
 DOCSTRING_PARAM_PATTERNS = [
     r'\s*:type\s+%s:\s*([^\n]+)',  # Sphinx

jedi/dynamic.py

@@ -55,16 +55,17 @@ from __future__ import with_statement
 
 import os
 
-import cache
-import parsing_representation as pr
-import evaluate_representation as er
-import modules
-import evaluate
-import settings
-import debug
-import imports
+from jedi import cache
+from jedi import parsing_representation as pr
+from jedi import modules
+from jedi import settings
+from jedi import common
+from jedi import debug
+from jedi import fast_parser
 import api_classes
-import fast_parser
+import evaluate
+import imports
+import evaluate_representation as er
 
 # This is something like the sys.path, but only for searching params. It means
 # that this is the order in which Jedi searches params.
@@ -487,10 +488,8 @@ def related_name_add_import_modules(definitions, search_name):
     for d in definitions:
         if isinstance(d.parent, pr.Import):
             s = imports.ImportPath(d.parent, direct_resolve=True)
-            try:
+            with common.ignored(IndexError):
                 new.add(s.follow(is_goto=True)[0])
-            except IndexError:
-                pass
     return set(definitions) | new
 
 

jedi/evaluate.py

@@ -68,21 +68,22 @@ backtracking algorithm.
 
 .. todo:: nonlocal statement, needed or can be ignored? (py3k)
 """
-from _compatibility import next, hasattr, is_py3k, unicode, utf8
+from __future__ import with_statement
 
 import sys
 import itertools
 
-import common
-import cache
-import parsing_representation as pr
+from jedi._compatibility import next, hasattr, is_py3k, unicode, reraise
+from jedi import common
+from jedi import cache
+from jedi import parsing_representation as pr
+from jedi import debug
 import evaluate_representation as er
-import debug
+import recursion
+import docstrings
 import builtin
 import imports
-import recursion
 import dynamic
-import docstrings
 
 
 def get_defined_names_for_position(scope, position=None, start_scope=None):
@@ -179,7 +180,7 @@ def get_names_of_scope(scope, position=None, star_search=True,
                     yield scope, get_defined_names_for_position(scope,
                                                     position, in_func_scope)
             except StopIteration:
-                raise common.MultiLevelStopIteration('StopIteration raised')
+                reraise(common.MultiLevelStopIteration, sys.exc_info()[2])
         if scope.isinstance(pr.ForFlow) and scope.is_list_comp:
             # is a list comprehension
             yield scope, scope.get_set_vars(is_internal_call=True)
@@ -433,11 +434,9 @@ def find_name(scope, name_str, position=None, search_global=False,
             if isinstance(scope, (er.Instance, er.Class)) \
                                 and hasattr(r, 'get_descriptor_return'):
                 # handle descriptors
-                try:
+                with common.ignored(KeyError):
                     res_new += r.get_descriptor_return(scope)
                     continue
-                except KeyError:
-                    pass
             res_new.append(r)
         return res_new
 
@@ -466,19 +465,15 @@ def check_getattr(inst, name_str):
     # str is important to lose the NamePart!
     module = builtin.Builtin.scope
     name = pr.Call(module, str(name_str), pr.Call.STRING, (0, 0), inst)
-    try:
+    with common.ignored(KeyError):
         result = inst.execute_subscope_by_name('__getattr__', [name])
-    except KeyError:
-        pass
     if not result:
         # this is a little bit special. `__getattribute__` is executed
         # before anything else. But: I know no use case, where this
         # could be practical and the jedi would return wrong types. If
         # you ever have something, let me know!
-        try:
+        with common.ignored(KeyError):
             result = inst.execute_subscope_by_name('__getattribute__', [name])
-        except KeyError:
-            pass
     return result
 
 
@@ -540,10 +535,8 @@ def assign_tuples(tup, results, seek_name):
                 debug.warning("invalid tuple lookup %s of result %s in %s"
                                     % (tup, results, seek_name))
             else:
-                try:
+                with common.ignored(IndexError):
                     types += func(index)
-                except IndexError:
-                    pass
         return types
 
     result = []
@@ -585,12 +578,7 @@ def follow_statement(stmt, seek_name=None):
     commands = stmt.get_commands()
     debug.dbg('calls: %s' % commands)
 
-    try:
-        result = follow_call_list(commands)
-    except AttributeError:
-        # This is so evil! But necessary to propagate errors. The attribute
-        # errors here must not be catched, because they shouldn't exist.
-        raise common.MultiLevelAttributeError(sys.exc_info())
+    result = follow_call_list(commands)
 
     # Assignment checking is only important if the statement defines multiple
     # variables.
@@ -602,6 +590,7 @@ def follow_statement(stmt, seek_name=None):
     return set(result)
 
 
+@common.rethrow_uncaught
 def follow_call_list(call_list, follow_array=False):
     """
     `call_list` can be either `pr.Array` or `list of list`.
@@ -656,11 +645,9 @@ def follow_call_list(call_list, follow_array=False):
                             call = next(calls_iterator)
                         except StopIteration:
                             break
-                        try:
+                        with common.ignored(AttributeError):
                             if str(call.name) == 'else':
                                 break
-                        except AttributeError:
-                            pass
                     continue
                 result += follow_call(call)
             elif call == '*':

jedi/evaluate_representation.py

@@ -9,22 +9,23 @@ instantiated. This class represents these cases.
 So, why is there also a ``Class`` class here? Well, there are decorators and
 they change classes in Python 3.
 """
-import sys
+from __future__ import with_statement
+
 import copy
 import itertools
 
-from _compatibility import property, use_metaclass, next, hasattr
-import parsing_representation as pr
-import imports
+from jedi._compatibility import property, use_metaclass, next, hasattr
+from jedi import parsing_representation as pr
+from jedi import cache
+from jedi import helpers
+from jedi import debug
+from jedi import common
+import recursion
 import docstrings
-import cache
+import imports
+import evaluate
 import builtin
 import dynamic
-import helpers
-import recursion
-import debug
-import evaluate
-import common
 
 
 class DecoratorNotFound(LookupError):
@@ -62,10 +63,8 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)):
         else:
             # need to execute the __init__ function, because the dynamic param
             # searching needs it.
-            try:
+            with common.ignored(KeyError):
                 self.execute_subscope_by_name('__init__', self.var_args)
-            except KeyError:
-                pass
         # Generated instances are classes that are just generated by self
         # (No var_args) used.
         self.is_generated = False
@@ -668,6 +667,7 @@ class Execution(Executable):
         """
         return self.get_params() + pr.Scope.get_set_vars(self)
 
+    @common.rethrow_uncaught
     def copy_properties(self, prop):
         """
         Literally copies a property of a Function. Copying is very expensive,
@@ -675,22 +675,19 @@ class Execution(Executable):
         objects can be used for the executions, as if they were in the
         execution.
         """
-        try:
-            # Copy all these lists into this local function.
-            attr = getattr(self.base, prop)
-            objects = []
-            for element in attr:
-                if element is None:
-                    copied = element
-                else:
-                    copied = helpers.fast_parent_copy(element)
-                    copied.parent = self._scope_copy(copied.parent)
-                    if isinstance(copied, pr.Function):
-                        copied = Function(copied)
-                objects.append(copied)
-            return objects
-        except AttributeError:
-            raise common.MultiLevelAttributeError(sys.exc_info())
+        # Copy all these lists into this local function.
+        attr = getattr(self.base, prop)
+        objects = []
+        for element in attr:
+            if element is None:
+                copied = element
+            else:
+                copied = helpers.fast_parent_copy(element)
+                copied.parent = self._scope_copy(copied.parent)
+                if isinstance(copied, pr.Function):
+                    copied = Function(copied)
+            objects.append(copied)
+        return objects
 
     def __getattr__(self, name):
         if name not in ['start_pos', 'end_pos', 'imports', '_sub_module']:
@@ -698,21 +695,19 @@ class Execution(Executable):
         return getattr(self.base, name)
 
     @cache.memoize_default()
+    @common.rethrow_uncaught
     def _scope_copy(self, scope):
-        try:
-            """ Copies a scope (e.g. if) in an execution """
-            # TODO method uses different scopes than the subscopes property.
+        """ Copies a scope (e.g. if) in an execution """
+        # TODO method uses different scopes than the subscopes property.
 
-            # just check the start_pos, sometimes it's difficult with closures
-            # to compare the scopes directly.
-            if scope.start_pos == self.start_pos:
-                return self
-            else:
-                copied = helpers.fast_parent_copy(scope)
-                copied.parent = self._scope_copy(copied.parent)
-                return copied
-        except AttributeError:
-            raise common.MultiLevelAttributeError(sys.exc_info())
+        # just check the start_pos, sometimes it's difficult with closures
+        # to compare the scopes directly.
+        if scope.start_pos == self.start_pos:
+            return self
+        else:
+            copied = helpers.fast_parent_copy(scope)
+            copied.parent = self._scope_copy(copied.parent)
+            return copied
 
     @property
     @cache.memoize_default()
@@ -774,9 +769,13 @@ class Generator(use_metaclass(cache.CachedMetaClass, pr.Base)):
         debug.warning('Tried to get array access on a generator', self)
         return []
 
-    @property
-    def parent(self):
-        return self.func.parent
+    def __getattr__(self, name):
+        if name not in ['start_pos', 'end_pos', 'parent', 'get_imports',
+                'asserts', 'doc', 'docstr', 'get_parent_until', 'get_code',
+                'subscopes']:
+            raise AttributeError("Accessing %s of %s is not allowed."
+                                    % (self, name))
+        return getattr(self.func, name)
 
     def __repr__(self):
         return "<%s of %s>" % (type(self).__name__, self.func)
@@ -805,10 +804,8 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)):
                 if isinstance(index, Instance) \
                             and str(index.name) in ['int', 'str'] \
                             and len(index.var_args) == 1:
-                    try:
+                    with common.ignored(KeyError, IndexError):
                         return self.get_exact_index_types(index.var_args[0])
-                    except (KeyError, IndexError):
-                        pass
 
         result = list(self._follow_values(self._array.values))
         result += dynamic.check_array_additions(self)

jedi/fast_parser.py

@@ -6,11 +6,11 @@ finished (and still not working as I want), I won't document it any further.
 import re
 import operator
 
-from _compatibility import use_metaclass, reduce, property
-import settings
-import parsing
-import parsing_representation as pr
-import cache
+from jedi._compatibility import use_metaclass, reduce, property
+from jedi import settings
+from jedi import parsing
+from jedi import parsing_representation as pr
+from jedi import cache
 import common
 
 

jedi/helpers.py

@@ -1,6 +1,9 @@
+from __future__ import with_statement
+
 import copy
 
-import parsing_representation as pr
+from jedi import common
+from jedi import parsing_representation as pr
 
 
 def fast_parent_copy(obj):
@@ -21,13 +24,11 @@ def fast_parent_copy(obj):
 
         before = ()
         for cls in new_obj.__class__.__mro__:
-            try:
+            with common.ignored(AttributeError):
                 if before == cls.__slots__:
                     continue
                 before = cls.__slots__
                 items += [(n, getattr(new_obj, n)) for n in before]
-            except AttributeError:
-                pass
 
         for key, value in items:
             # replace parent (first try _parent and then parent)
@@ -35,10 +36,8 @@ def fast_parent_copy(obj):
                 if key == 'parent' and '_parent' in items:
                     # parent can be a property
                     continue
-                try:
+                with common.ignored(KeyError):
                     setattr(new_obj, key, new_elements[value])
-                except KeyError:
-                    pass
             elif key in ['parent_function', 'use_as_parent', '_sub_module']:
                 continue
             elif isinstance(value, list):

jedi/imports.py

@@ -5,28 +5,27 @@ any actual importing done. This module is about finding modules in the
 filesystem. This can be quite tricky sometimes, because Python imports are not
 always that simple.
 
-Currently the import process uses ``imp`` to find modules. In the future, it's
-a goal to use ``importlib`` for this purpose. There's a `pull request
-<https://github.com/davidhalter/jedi/pull/109>`_ for that.
+This module uses imp for python up to 3.2 and importlib for python 3.3 on; the
+correct implementation is delegated to _compatibility.
 
 This module also supports import autocompletion, which means to complete
 statements like ``from datetim`` (curser at the end would return ``datetime``).
 """
-
 from __future__ import with_statement
 
 import os
 import pkgutil
-import imp
 import sys
-
-import builtin
-import modules
-import debug
-import parsing_representation as pr
-import evaluate
 import itertools
-import cache
+
+from jedi._compatibility import find_module
+from jedi import modules
+from jedi import common
+from jedi import debug
+from jedi import parsing_representation as pr
+from jedi import cache
+import builtin
+import evaluate
 
 # for debugging purposes only
 imports_processed = 0
@@ -123,11 +122,9 @@ class ImportPath(pr.Base):
 
                     if self.import_stmt.relative_count:
                         rel_path = self.get_relative_path() + '/__init__.py'
-                        try:
+                        with common.ignored(IOError):
                             m = modules.Module(rel_path)
                             names += m.parser.module.get_defined_names()
-                        except IOError:
-                            pass
             else:
                 if on_import_stmt and isinstance(scope, pr.Module) \
                                         and scope.path.endswith('__init__.py'):
@@ -238,20 +235,22 @@ class ImportPath(pr.Base):
 
             global imports_processed
             imports_processed += 1
+            importing = None
             if path is not None:
-                return imp.find_module(string, [path])
+                importing = find_module(string, [path])
             else:
                 debug.dbg('search_module', string, self.file_path)
                 # Override the sys.path. It works only good that way.
                 # Injecting the path directly into `find_module` did not work.
                 sys.path, temp = sys_path_mod, sys.path
                 try:
-                    i = imp.find_module(string)
+                    importing = find_module(string)
                 except ImportError:
                     sys.path = temp
                     raise
                 sys.path = temp
-                return i
+            
+            return importing
 
         if self.file_path:
             sys_path_mod = list(self.sys_path_with_modifications())
@@ -259,6 +258,9 @@ class ImportPath(pr.Base):
         else:
             sys_path_mod = list(modules.get_sys_path())
 
+        def module_not_found():
+            raise ModuleNotFound('The module you searched has not been found')
+
         current_namespace = (None, None, None)
         # now execute those paths
         rest = []
@@ -270,19 +272,19 @@ class ImportPath(pr.Base):
                                 and len(self.import_path) == 1:
                     # follow `from . import some_variable`
                     rel_path = self.get_relative_path()
-                    try:
+                    with common.ignored(ImportError):
                         current_namespace = follow_str(rel_path, '__init__')
-                    except ImportError:
-                        pass
                 if current_namespace[1]:
                     rest = self.import_path[i:]
                 else:
-                    raise ModuleNotFound(
-                            'The module you searched has not been found')
+                    module_not_found()
+
+        if current_namespace == (None, None, False):
+            module_not_found()
 
         sys_path_mod.pop(0)  # TODO why is this here?
         path = current_namespace[1]
-        is_package_directory = current_namespace[2][2] == imp.PKG_DIRECTORY
+        is_package_directory = current_namespace[2]
 
         f = None
         if is_package_directory or current_namespace[0]:

jedi/keywords.py

@@ -1,9 +1,12 @@
-import keyword
-
-from _compatibility import is_py3k
-import builtin
+from __future__ import with_statement
 
 import pydoc
+import keyword
+
+from jedi._compatibility import is_py3k
+from jedi import common
+import builtin
+
 try:
     from pydoc_data import topics as pydoc_topics
 except ImportError:
@@ -63,12 +66,10 @@ def imitate_pydoc(string):
     # with unicode strings)
     string = str(string)
     h = pydoc.help
-    try:
+    with common.ignored(KeyError):
         # try to access symbols
         string = h.symbols[string]
         string, _, related = string.partition(' ')
-    except KeyError:
-        pass
 
     get_target = lambda s: h.topics.get(s, h.keywords.get(s))
     while isinstance(string, str):

jedi/modules.py

@@ -15,19 +15,19 @@ Apart from those classes there's a ``sys.path`` fetching function, as well as
 """
 from __future__ import with_statement
 
-from _compatibility import exec_function, unicode, is_py25, literal_eval
-
 import re
-import tokenize
+import tokenizer as tokenize
 import sys
 import os
 
-import cache
-import parsing
-import parsing_representation as pr
-import fast_parser
-import debug
-import settings
+from jedi._compatibility import exec_function, unicode, is_py25, literal_eval
+from jedi import cache
+from jedi import parsing
+from jedi import parsing_representation as pr
+from jedi import fast_parser
+from jedi import debug
+from jedi import settings
+from jedi import common
 
 
 class CachedModule(object):
@@ -108,11 +108,9 @@ class ModuleWithCursor(Module):
     def parser(self):
         """ get the parser lazy """
         if not self._parser:
-            try:
+            with common.ignored(KeyError):
                 parser = cache.parser_cache[self.path].parser
                 cache.invalidate_star_import_cache(parser.module)
-            except KeyError:
-                pass
             # Call the parser already here, because it will be used anyways.
             # Also, the position is here important (which will not be used by
             # default), therefore fill the cache here.
@@ -279,9 +277,8 @@ def get_sys_path():
             venv, 'lib', 'python%d.%d' % sys.version_info[:2], 'site-packages')
         sys_path.insert(0, p)
 
-    p = sys.path[1:]
-    check_virtual_env(p)
-    return p
+    check_virtual_env(sys.path)
+    return [p for p in sys.path if p != ""]
 
 
 @cache.memoize_default([])
@@ -350,10 +347,8 @@ def sys_path_with_modifications(module):
         return []  # support for modules without a path is intentionally bad.
 
     curdir = os.path.abspath(os.curdir)
-    try:
+    with common.ignored(OSError):
         os.chdir(os.path.dirname(module.path))
-    except OSError:
-        pass
 
     result = check_module(module)
     result += detect_django_path(module.path)
@@ -374,12 +369,10 @@ def detect_django_path(module_path):
         else:
             module_path = new
 
-        try:
+        with common.ignored(IOError):
             with open(module_path + os.path.sep + 'manage.py'):
                 debug.dbg('Found django path: %s' % module_path)
                 result.append(module_path)
-        except IOError:
-            pass
     return result
 
 

jedi/parsing.py

@@ -15,14 +15,15 @@ within the statement. This lowers memory usage and cpu time and reduces the
 complexity of the ``Parser`` (there's another parser sitting inside
 ``Statement``, which produces ``Array`` and ``Call``).
 """
-from _compatibility import next, StringIO
+from __future__ import with_statement
 
-import tokenize
+import tokenizer as tokenize
 import keyword
 
-import debug
-import common
-import parsing_representation as pr
+from jedi._compatibility import next, StringIO
+from jedi import debug
+from jedi import common
+from jedi import parsing_representation as pr
 
 
 class ParserError(Exception):
@@ -395,10 +396,11 @@ class Parser(object):
             stmt = stmt_class(self.module, set_vars, used_vars, tok_list,
                               first_pos, self.end_pos)
 
+            stmt.parent = self.top_module
             self._check_user_stmt(stmt)
 
         # Attribute docstring (PEP 257) support
-        try:
+        with common.ignored(IndexError, AttributeError):
             # If string literal is being parsed
             first_tok = stmt.token_list[0]
             if (not stmt.set_vars and
@@ -407,8 +409,6 @@ class Parser(object):
                 first_tok[0] == tokenize.STRING):
                 # ... then set it as a docstring
                 self.scope.statements[-1].add_docstr(first_tok[1])
-        except (IndexError, AttributeError):
-            pass
 
         if tok in always_break + not_first_break:
             self._gen.push_last_back()

jedi/parsing_representation.py

@@ -33,15 +33,16 @@ statements in this scope.  Check this out:
 
 See also :attr:`Scope.subscopes` and :attr:`Scope.statements`.
 """
+from __future__ import with_statement
 
 import os
 import re
-import tokenize
+import tokenizer as tokenize
 
-from _compatibility import next, literal_eval, cleandoc, Python3Method, \
+from jedi._compatibility import next, literal_eval, cleandoc, Python3Method, \
                             encoding, property, unicode, is_py3k
-import common
-import debug
+from jedi import common
+from jedi import debug
 
 
 class Base(object):
@@ -564,8 +565,10 @@ class Flow(Scope):
     @parent.setter
     def parent(self, value):
         self._parent = value
-        if self.next:
+        try:
             self.next.parent = value
+        except AttributeError:
+            return
 
     def get_code(self, first_indent=False, indention='    '):
         stmts = []
@@ -1282,11 +1285,8 @@ class Array(Call):
         inner = []
         for i, stmt in enumerate(self.values):
             s = ''
-            try:
+            with common.ignored(IndexError):
                 key = self.keys[i]
-            except IndexError:
-                pass
-            else:
                 s += key.get_code(new_line=False) + ': '
             s += stmt.get_code(new_line=False)
             inner.append(s)

jedi/recursion.py

@@ -7,11 +7,11 @@ Next to :mod:`cache` this module also makes |jedi| not thread-safe. Why?
 ``ExecutionRecursionDecorator`` uses class variables to count the function
 calls.
 """
-import parsing_representation as pr
+from jedi import parsing_representation as pr
+from jedi import debug
+from jedi import settings
 import evaluate_representation as er
-import debug
 import builtin
-import settings
 
 
 class RecursionDecorator(object):

jedi/refactoring.py

@@ -12,13 +12,14 @@ following functions (sometimes bug-prone):
 - extract variable
 - inline variable
 """
-
 from __future__ import with_statement
 
-import modules
 import difflib
-import helpers
-import parsing_representation as pr
+
+from jedi import common
+from jedi import modules
+from jedi import helpers
+from jedi import parsing_representation as pr
 
 
 class Refactoring(object):
@@ -167,7 +168,7 @@ def inline(script):
     dct = {}
 
     definitions = script.goto()
-    try:
+    with common.ignored(AssertionError):
         assert len(definitions) == 1
         stmt = definitions[0].definition
         related_names = script.related_names()
@@ -201,7 +202,4 @@ def inline(script):
         else:
             new_lines.pop(index)
 
-    except AssertionError:
-        pass
-
     return Refactoring(dct)

jedi/tokenizer.py

@@ -0,0 +1,270 @@
+"""
+This tokenizer has been copied from the ``tokenize.py`` standard library
+tokenizer. The reason was simple: The standanrd library  tokenizer fails
+if the indentation is not right. The fast parser of jedi however requires
+"wrong" indentation.
+
+Basically this is a stripped down version of the standard library module, so
+you can read the documentation there.
+"""
+
+import string
+import re
+from token import *
+from codecs import lookup, BOM_UTF8
+import collections
+cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
+
+namechars = string.ascii_letters + '_'
+
+
+COMMENT = N_TOKENS
+tok_name[COMMENT] = 'COMMENT'
+NL = N_TOKENS + 1
+tok_name[NL] = 'NL'
+ENCODING = N_TOKENS + 2
+tok_name[ENCODING] = 'ENCODING'
+N_TOKENS += 3
+
+class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
+    def __repr__(self):
+        annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
+        return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
+                self._replace(type=annotated_type))
+
+def group(*choices): return '(' + '|'.join(choices) + ')'
+def any(*choices): return group(*choices) + '*'
+def maybe(*choices): return group(*choices) + '?'
+
+# Note: we use unicode matching for names ("\w") but ascii matching for
+# number literals.
+Whitespace = r'[ \f\t]*'
+Comment = r'#[^\r\n]*'
+Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
+Name = r'\w+'
+
+Hexnumber = r'0[xX][0-9a-fA-F]+'
+Binnumber = r'0[bB][01]+'
+Octnumber = r'0[oO][0-7]+'
+Decnumber = r'(?:0+|[1-9][0-9]*)'
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+Exponent = r'[eE][-+]?[0-9]+'
+Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
+Expfloat = r'[0-9]+' + Exponent
+Floatnumber = group(Pointfloat, Expfloat)
+Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
+Number = group(Imagnumber, Floatnumber, Intnumber)
+
+# Tail end of ' string.
+Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
+# Tail end of " string.
+Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
+# Tail end of ''' string.
+Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
+# Tail end of """ string.
+Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
+Triple = group("[bB]?[rR]?'''", '[bB]?[rR]?"""')
+# Single-line ' or " string.
+String = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
+               r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
+
+# Because of leftmost-then-longest match semantics, be sure to put the
+# longest operators first (e.g., if = came before ==, == would get
+# recognized as two instances of =).
+Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
+                 r"//=?", r"->",
+                 r"[+\-*/%&|^=<>]=?",
+                 r"~")
+
+Bracket = '[][(){}]'
+Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
+Funny = group(Operator, Bracket, Special)
+
+PlainToken = group(Number, Funny, String, Name)
+Token = Ignore + PlainToken
+
+# First (or only) line of ' or " string.
+ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
+                group("'", r'\\\r?\n'),
+                r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
+                group('"', r'\\\r?\n'))
+PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
+
+def _compile(expr):
+    return re.compile(expr, re.UNICODE)
+
+tokenprog, pseudoprog, single3prog, double3prog = map(
+    _compile, (Token, PseudoToken, Single3, Double3))
+endprogs = {"'": _compile(Single), '"': _compile(Double),
+            "'''": single3prog, '"""': double3prog,
+            "r'''": single3prog, 'r"""': double3prog,
+            "b'''": single3prog, 'b"""': double3prog,
+            "br'''": single3prog, 'br"""': double3prog,
+            "R'''": single3prog, 'R"""': double3prog,
+            "B'''": single3prog, 'B"""': double3prog,
+            "bR'''": single3prog, 'bR"""': double3prog,
+            "Br'''": single3prog, 'Br"""': double3prog,
+            "BR'''": single3prog, 'BR"""': double3prog,
+            'r': None, 'R': None, 'b': None, 'B': None}
+
+triple_quoted = {}
+for t in ("'''", '"""',
+          "r'''", 'r"""', "R'''", 'R"""',
+          "b'''", 'b"""', "B'''", 'B"""',
+          "br'''", 'br"""', "Br'''", 'Br"""',
+          "bR'''", 'bR"""', "BR'''", 'BR"""'):
+    triple_quoted[t] = t
+single_quoted = {}
+for t in ("'", '"',
+          "r'", 'r"', "R'", 'R"',
+          "b'", 'b"', "B'", 'B"',
+          "br'", 'br"', "Br'", 'Br"',
+          "bR'", 'bR"', "BR'", 'BR"' ):
+    single_quoted[t] = t
+
+del _compile
+
+tabsize = 8
+
+class TokenError(Exception): pass
+
+
+def generate_tokens(readline):
+    lnum = parenlev = continued = 0
+    numchars = '0123456789'
+    contstr, needcont = '', 0
+    contline = None
+    indents = [0]
+
+    while True:             # loop over lines in stream
+        try:
+            line = readline()
+        except StopIteration:
+            line = b''
+
+        lnum += 1
+        pos, max = 0, len(line)
+
+        if contstr:                            # continued string
+            if not line:
+                # multiline string has not been finished
+                break
+            endmatch = endprog.match(line)
+            if endmatch:
+                pos = end = endmatch.end(0)
+                yield TokenInfo(STRING, contstr + line[:end],
+                       strstart, (lnum, end), contline + line)
+                contstr, needcont = '', 0
+                contline = None
+            elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
+                yield TokenInfo(ERRORTOKEN, contstr + line,
+                           strstart, (lnum, len(line)), contline)
+                contstr = ''
+                contline = None
+                continue
+            else:
+                contstr = contstr + line
+                contline = contline + line
+                continue
+
+        elif parenlev == 0 and not continued:  # new statement
+            if not line: break
+            column = 0
+            while pos < max:                   # measure leading whitespace
+                if line[pos] == ' ':
+                    column += 1
+                elif line[pos] == '\t':
+                    column = (column//tabsize + 1)*tabsize
+                elif line[pos] == '\f':
+                    column = 0
+                else:
+                    break
+                pos += 1
+            if pos == max:
+                break
+
+            if line[pos] in '#\r\n':           # skip comments or blank lines
+                if line[pos] == '#':
+                    comment_token = line[pos:].rstrip('\r\n')
+                    nl_pos = pos + len(comment_token)
+                    yield TokenInfo(COMMENT, comment_token,
+                           (lnum, pos), (lnum, pos + len(comment_token)), line)
+                    yield TokenInfo(NL, line[nl_pos:],
+                           (lnum, nl_pos), (lnum, len(line)), line)
+                else:
+                    yield TokenInfo((NL, COMMENT)[line[pos] == '#'], line[pos:],
+                           (lnum, pos), (lnum, len(line)), line)
+                continue
+
+            if column > indents[-1]:           # count indents or dedents
+                indents.append(column)
+                yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+            while column < indents[-1]:
+                indents = indents[:-1]
+                yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
+
+        else:                                  # continued statement
+            if not line:
+                # basically a statement has not been finished here.
+                break
+            continued = 0
+
+        while pos < max:
+            pseudomatch = pseudoprog.match(line, pos)
+            if pseudomatch:                                # scan for tokens
+                start, end = pseudomatch.span(1)
+                spos, epos, pos = (lnum, start), (lnum, end), end
+                token, initial = line[start:end], line[start]
+
+                if (initial in numchars or                  # ordinary number
+                    (initial == '.' and token != '.' and token != '...')):
+                    yield TokenInfo(NUMBER, token, spos, epos, line)
+                elif initial in '\r\n':
+                    yield TokenInfo(NL if parenlev > 0 else NEWLINE,
+                           token, spos, epos, line)
+                elif initial == '#':
+                    assert not token.endswith("\n")
+                    yield TokenInfo(COMMENT, token, spos, epos, line)
+                elif token in triple_quoted:
+                    endprog = endprogs[token]
+                    endmatch = endprog.match(line, pos)
+                    if endmatch:                           # all on one line
+                        pos = endmatch.end(0)
+                        token = line[start:pos]
+                        yield TokenInfo(STRING, token, spos, (lnum, pos), line)
+                    else:
+                        strstart = (lnum, start)           # multiple lines
+                        contstr = line[start:]
+                        contline = line
+                        break
+                elif initial in single_quoted or \
+                    token[:2] in single_quoted or \
+                    token[:3] in single_quoted:
+                    if token[-1] == '\n':                  # continued string
+                        strstart = (lnum, start)
+                        endprog = (endprogs[initial] or endprogs[token[1]] or
+                                   endprogs[token[2]])
+                        contstr, needcont = line[start:], 1
+                        contline = line
+                        break
+                    else:                                  # ordinary string
+                        yield TokenInfo(STRING, token, spos, epos, line)
+                elif initial in namechars:                 # ordinary name
+                    yield TokenInfo(NAME, token, spos, epos, line)
+                elif initial == '\\':                      # continued stmt
+                    continued = 1
+                else:
+                    if initial in '([{':
+                        parenlev += 1
+                    elif initial in ')]}':
+                        parenlev -= 1
+                    yield TokenInfo(OP, token, spos, epos, line)
+            else:
+                yield TokenInfo(ERRORTOKEN, line[pos],
+                           (lnum, pos), (lnum, pos + 1), line)
+                pos += 1
+
+    for indent in indents[1:]:                 # pop remaining indent levels
+        yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
+    yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')